From fcc4886b7442acc1b48c12d54a67da088627c9f5 Mon Sep 17 00:00:00 2001
From: Khalid Shakir <kshakir@broadinstitute.org>
Date: Thu, 5 Jan 2017 16:08:02 -0500
Subject: [PATCH 01/84] Version bump

---
 project/Version.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/project/Version.scala b/project/Version.scala
index 3cd7d5da8..a4dc20d69 100644
--- a/project/Version.scala
+++ b/project/Version.scala
@@ -4,7 +4,7 @@ import sbt._
 
 object Version {
   // Upcoming release, or current if we're on a master / hotfix branch
-  val cromwellVersion = "24"
+  val cromwellVersion = "25"
 
   // Adapted from SbtGit.versionWithGit
   def cromwellVersionWithGit: Seq[Setting[_]] =

From 864ff61c918b3bd7f17416015f1b967af8951e74 Mon Sep 17 00:00:00 2001
From: Khalid Shakir <kshakir@broadinstitute.org>
Date: Thu, 5 Jan 2017 16:09:29 -0500
Subject: [PATCH 02/84] Version bump in CHANGELOG.md

---
 CHANGELOG.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 06b720f39..a12ac4c8c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,9 @@
 # Cromwell Change Log
 
+## 25
+
+
+
 ## 24
 
 * When emitting workflow outputs to the Cromwell log only the first 1000 characters per output will be printed

From 50af6d829c4819976c08cf202a92ac799aad2bbc Mon Sep 17 00:00:00 2001
From: Khalid Shakir <kshakir@broadinstitute.org>
Date: Wed, 28 Dec 2016 20:37:18 -0500
Subject: [PATCH 03/84] Moved files for next edits.

---
 .../backend/standard/StandardValidatedRuntimeAttributesBuilder.scala      | 0
 .../backend/validation}/PrimitiveRuntimeAttributesValidation.scala        | 0
 .../backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala  | 0
 3 files changed, 0 insertions(+), 0 deletions(-)
 rename supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemValidatedRuntimeAttributesBuilder.scala => backend/src/main/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilder.scala (100%)
 rename {supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config => backend/src/main/scala/cromwell/backend/validation}/PrimitiveRuntimeAttributesValidation.scala (100%)
 rename supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemValidatedRuntimeAttributesBuilderSpec.scala => backend/src/test/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala (100%)

diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemValidatedRuntimeAttributesBuilder.scala b/backend/src/main/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilder.scala
similarity index 100%
rename from supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemValidatedRuntimeAttributesBuilder.scala
rename to backend/src/main/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilder.scala
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/PrimitiveRuntimeAttributesValidation.scala b/backend/src/main/scala/cromwell/backend/validation/PrimitiveRuntimeAttributesValidation.scala
similarity index 100%
rename from supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/PrimitiveRuntimeAttributesValidation.scala
rename to backend/src/main/scala/cromwell/backend/validation/PrimitiveRuntimeAttributesValidation.scala
diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemValidatedRuntimeAttributesBuilderSpec.scala b/backend/src/test/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala
similarity index 100%
rename from supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemValidatedRuntimeAttributesBuilderSpec.scala
rename to backend/src/test/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala

From a906fe6cbd5e03d375d54504a7a69fb0672d42b7 Mon Sep 17 00:00:00 2001
From: Khalid Shakir <kshakir@broadinstitute.org>
Date: Fri, 23 Dec 2016 00:34:29 -0500
Subject: [PATCH 04/84] Updates for Standard backends. JES now uses the
 uncustomized sync standard actor. Moved primitive runtime attributes
 validators to standard. Moved the SFS ValidatedRuntimeAttributesBuilder
 (VRAB) to standard. Moved the SFS initialization data to standard. JES and
 Standard backends now use VRAB within their backends via initialization data.
 Primitive runtime attribute validations now handle basic `WdlString`
 conversion to Boolean/Float/Integer. Merged code for continue on return code
 expression validation, from BackendWorkflowInitializationActor and
 ContinueOnReturnCodeValidation. Added a
 BackendWorkflowInitializationActorSpec. Updated
 ContinueOnReturnCodeValidation to also allow boolean strings `"true"` and
 `"false"`. The standard async backend pulls failOnStdErr and
 continueOnReturnCode from those generated by the VRAB. JesRuntimeAttributes
 (JRA) generation now extracts values from the the map build by the the VRAB.
 Realized based on the JRA Spec that the VRAB failure messages need to
 differentiate missing vs. bad. When encountering invalid runtime attributes,
 the validation should (optionally) say what the bad value is (see Jes'
 NoAddressValidation). Plumbed the synchronous actor of the job actor method
 params into the param-traits used by Standard. Moved the SFS path builders
 from the SFS factory to the SFS init actor. New standard cache-hit-copying
 actor trait, plus consolidation of the cache-hit-copying and async actors
 into a new standard helper trait. SharedFileSystemJob.JobIdKey is now
 SharedFileSystemAsyncJobExecutionActor.JobIdKey. WorkflowPaths.toJobPaths now
 takes into account that jobs for sub workflow use a different workflow
 descriptor than the workflow. Merged WorkflowPathsBackendInitializationData
 into StandardInitializationData. New standard finalization trait. Changed JES
 runtime attributes (JRA) to always require docker. Updated JRA spec match new
 required docker. While updating spec, changed JRA spec to match the primitive
 validator messages.

---
 .../BackendWorkflowInitializationActor.scala       |  34 +---
 .../scala/cromwell/backend/io/WorkflowPaths.scala  |  23 ++-
 .../WorkflowPathsBackendInitializationData.scala   |  44 ----
 .../backend/io/WorkflowPathsWithDocker.scala       |  14 +-
 .../standard/StandardAsyncExecutionActor.scala     |  67 ++-----
 .../standard/StandardCacheHitCopyingActor.scala    |  46 +++++
 .../standard/StandardCachingActorHelper.scala      |  89 +++++++++
 .../standard/StandardFinalizationActor.scala       |  41 ++++
 .../standard/StandardInitializationActor.scala     |  70 +++++++
 .../standard/StandardInitializationData.scala      |  21 ++
 .../standard/StandardJobExecutionActorParams.scala |  19 +-
 .../standard/StandardLifecycleActorFactory.scala   | 119 ++++++++++-
 .../standard/StandardSyncExecutionActor.scala      |   1 +
 ...StandardValidatedRuntimeAttributesBuilder.scala | 101 +++-------
 .../ContinueOnReturnCodeValidation.scala           |  49 +++--
 .../backend/validation/CpuValidation.scala         |  39 ++--
 .../backend/validation/DockerValidation.scala      |  27 +--
 .../validation/FailOnStderrValidation.scala        |  41 +---
 .../backend/validation/MemoryValidation.scala      |  27 ++-
 .../PrimitiveRuntimeAttributesValidation.scala     |  63 ++++--
 .../validation/RuntimeAttributesValidation.scala   | 183 +++++++++--------
 .../ValidatedRuntimeAttributesBuilder.scala        |  27 +--
 .../BackendWorkflowInitializationActorSpec.scala   | 204 +++++++++++++++++++
 ...dardValidatedRuntimeAttributesBuilderSpec.scala |  35 ++--
 .../jes/JesAsyncBackendJobExecutionActor.scala     |  49 +----
 .../impl/jes/JesBackendInitializationData.scala    |  10 +-
 .../impl/jes/JesBackendLifecycleActorFactory.scala |  92 +++------
 .../backend/impl/jes/JesCacheHitCopyingActor.scala |  31 +--
 .../backend/impl/jes/JesFinalizationActor.scala    |  42 ++--
 .../backend/impl/jes/JesInitializationActor.scala  |  80 +++-----
 .../impl/jes/JesJobCachingActorHelper.scala        |  47 ++---
 .../backend/impl/jes/JesJobExecutionActor.scala    |  61 ------
 .../backend/impl/jes/JesPipelineInfo.scala         |  11 +-
 .../backend/impl/jes/JesRuntimeAttributes.scala    | 221 +++++++++++----------
 .../backend/impl/jes/JesWorkflowPaths.scala        |  22 +-
 .../backend/impl/jes/io/JesAttachedDisk.scala      |  10 +-
 .../jes/JesAsyncBackendJobExecutionActorSpec.scala |  23 ++-
 .../impl/jes/JesInitializationActorSpec.scala      |  18 +-
 .../impl/jes/JesJobExecutionActorSpec.scala        |  29 +--
 .../impl/jes/JesRuntimeAttributesSpec.scala        |  44 ++--
 .../sfs/config/ConfigAsyncJobExecutionActor.scala  |   2 +-
 .../ConfigBackendLifecycleActorFactory.scala       |  13 +-
 .../impl/sfs/config/ConfigHashingStrategy.scala    |  17 +-
 .../sfs/config/ConfigInitializationActor.scala     |  13 +-
 .../impl/sfs/config/DeclarationValidation.scala    |  26 ++-
 .../SharedFileSystemAsyncJobExecutionActor.scala   |  51 ++---
 ...redFileSystemBackendLifecycleActorFactory.scala |  57 +-----
 .../sfs/SharedFileSystemCacheHitCopyingActor.scala |  24 +--
 .../sfs/SharedFileSystemExpressionFunctions.scala  |  13 +-
 .../sfs/SharedFileSystemInitializationActor.scala  | 101 ++++------
 .../SharedFileSystemJobCachingActorHelper.scala    |  35 +---
 .../sfs/config/ConfigHashingStrategySpec.scala     |   4 +-
 .../SharedFileSystemInitializationActorSpec.scala  |   9 +-
 .../SharedFileSystemJobExecutionActorSpec.scala    |   5 +-
 .../sfs/TestLocalAsyncJobExecutionActor.scala      |  12 +-
 55 files changed, 1345 insertions(+), 1211 deletions(-)
 delete mode 100644 backend/src/main/scala/cromwell/backend/io/WorkflowPathsBackendInitializationData.scala
 create mode 100644 backend/src/main/scala/cromwell/backend/standard/StandardCacheHitCopyingActor.scala
 create mode 100644 backend/src/main/scala/cromwell/backend/standard/StandardCachingActorHelper.scala
 create mode 100644 backend/src/main/scala/cromwell/backend/standard/StandardFinalizationActor.scala
 create mode 100644 backend/src/main/scala/cromwell/backend/standard/StandardInitializationActor.scala
 create mode 100644 backend/src/main/scala/cromwell/backend/standard/StandardInitializationData.scala
 create mode 100644 backend/src/test/scala/cromwell/backend/BackendWorkflowInitializationActorSpec.scala
 delete mode 100644 supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobExecutionActor.scala

diff --git a/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala b/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala
index feaf5720b..626674f19 100644
--- a/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala
+++ b/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala
@@ -4,13 +4,14 @@ import akka.actor.{ActorLogging, ActorRef}
 import akka.event.LoggingReceive
 import cromwell.backend.BackendLifecycleActor._
 import cromwell.backend.BackendWorkflowInitializationActor._
-import wdl4s.expression.PureStandardLibraryFunctions
+import cromwell.backend.validation.ContinueOnReturnCodeValidation
 import cromwell.core.{WorkflowMetadataKeys, WorkflowOptions}
 import cromwell.services.metadata.MetadataService.PutMetadataAction
 import cromwell.services.metadata.{MetadataEvent, MetadataKey, MetadataValue}
-import wdl4s.types._
-import wdl4s.values.{WdlArray, WdlBoolean, WdlInteger, WdlString, WdlValue}
 import wdl4s._
+import wdl4s.expression.PureStandardLibraryFunctions
+import wdl4s.types._
+import wdl4s.values.WdlValue
 
 import scala.concurrent.Future
 import scala.util.{Failure, Success, Try}
@@ -34,7 +35,7 @@ object BackendWorkflowInitializationActor {
   * Workflow-level actor for executing, recovering and aborting jobs.
   */
 trait BackendWorkflowInitializationActor extends BackendWorkflowLifecycleActor with ActorLogging {
-  val serviceRegistryActor: ActorRef
+  def serviceRegistryActor: ActorRef
 
   def calls: Set[TaskCall]
 
@@ -67,26 +68,7 @@ trait BackendWorkflowInitializationActor extends BackendWorkflowLifecycleActor w
     * return `true` in both cases.
     */
   protected def continueOnReturnCodePredicate(valueRequired: Boolean)(wdlExpressionMaybe: Option[WdlValue]): Boolean = {
-    def isInteger(s: String) = s.forall(_.isDigit) && !s.isEmpty
-
-    def validateValue(wdlValue: WdlValue) = wdlValue match {
-      case WdlInteger(_) => true
-      case WdlString(_) => isInteger(wdlValue.valueString)
-      case WdlArray(WdlArrayType(WdlIntegerType), _) => true
-      case WdlArray(WdlArrayType(WdlStringType), elements) => elements forall { x => isInteger(x.valueString) }
-      case WdlBoolean(_) => true
-      case _ => false
-    }
-
-    wdlExpressionMaybe match {
-      case None => !valueRequired
-      case Some(wdlExpression: WdlExpression) =>
-        wdlExpression.evaluate(NoLookup, PureStandardLibraryFunctions) match {
-          case Success(wdlValue) => validateValue(wdlValue)
-          case Failure(throwable) => true // If we can't evaluate it, we'll let it pass for now...
-        }
-      case Some(wdlValue) => validateValue(wdlValue)
-    }
+    ContinueOnReturnCodeValidation.default.validateOptionalExpression(wdlExpressionMaybe)
   }
 
   protected def runtimeAttributeValidators: Map[String, Option[WdlValue] => Boolean]
@@ -94,7 +76,7 @@ trait BackendWorkflowInitializationActor extends BackendWorkflowLifecycleActor w
   // FIXME: If a workflow executes jobs using multiple backends,
   // each backend will try to write its own workflow root and override any previous one.
   // They should be structured differently or at least be prefixed by the backend name
-  protected def publishWorkflowRoot(workflowRoot: String) = {
+  protected def publishWorkflowRoot(workflowRoot: String): Unit = {
     serviceRegistryActor ! PutMetadataAction(MetadataEvent(MetadataKey(workflowDescriptor.id, None, WorkflowMetadataKeys.WorkflowRoot), MetadataValue(workflowRoot)))
   }
 
@@ -167,7 +149,7 @@ trait BackendWorkflowInitializationActor extends BackendWorkflowLifecycleActor w
   /**
     * Our predefined sequence to run during preStart
     */
-  final def initSequence() = for {
+  final def initSequence(): Future[InitializationSuccess] = for {
     _ <- validateRuntimeAttributes
     _ <- validate()
     data <- beforeAll()
diff --git a/backend/src/main/scala/cromwell/backend/io/WorkflowPaths.scala b/backend/src/main/scala/cromwell/backend/io/WorkflowPaths.scala
index a342d443e..b1955c9ec 100644
--- a/backend/src/main/scala/cromwell/backend/io/WorkflowPaths.scala
+++ b/backend/src/main/scala/cromwell/backend/io/WorkflowPaths.scala
@@ -3,7 +3,7 @@ package cromwell.backend.io
 import java.nio.file.Path
 
 import com.typesafe.config.Config
-import cromwell.backend.{BackendJobDescriptorKey, BackendWorkflowDescriptor}
+import cromwell.backend.{BackendJobDescriptor, BackendJobDescriptorKey, BackendWorkflowDescriptor}
 import cromwell.core.WorkflowOptions.FinalCallLogsDir
 import cromwell.core.path.{DefaultPathBuilder, PathFactory}
 import net.ceedubs.ficus.Ficus._
@@ -17,11 +17,11 @@ object WorkflowPaths {
 trait WorkflowPaths extends PathFactory {
   def workflowDescriptor: BackendWorkflowDescriptor
   def config: Config
-  
+
   protected lazy val executionRootString: String = config.as[Option[String]]("root").getOrElse("cromwell-executions")
-  
+
   def getPath(url: String): Try[Path] = Try(PathFactory.buildPath(url, pathBuilders))
-  
+
   // Rebuild potential intermediate call directories in case of a sub workflow
   protected def workflowPathBuilder(root: Path): Path = {
     workflowDescriptor.breadCrumbs.foldLeft(root)((acc, breadCrumb) => {
@@ -34,5 +34,18 @@ trait WorkflowPaths extends PathFactory {
   lazy val finalCallLogsPath: Option[Path] =
     workflowDescriptor.getWorkflowOption(FinalCallLogsDir) map getPath map { _.get }
 
-  def toJobPaths(jobKey: BackendJobDescriptorKey): JobPaths
+  def toJobPaths(jobDescriptor: BackendJobDescriptor): JobPaths = {
+    toJobPaths(jobDescriptor.key, jobDescriptor.workflowDescriptor)
+  }
+
+  /**
+    * Creates job paths using the key and workflow descriptor.
+    *
+    * NOTE: For sub workflows, the jobWorkflowDescriptor will be different than the WorkflowPaths.workflowDescriptor.
+    *
+    * @param jobKey                The key for the job.
+    * @param jobWorkflowDescriptor The workflow descriptor for the job.
+    * @return The paths for the job.
+    */
+  def toJobPaths(jobKey: BackendJobDescriptorKey, jobWorkflowDescriptor: BackendWorkflowDescriptor): JobPaths
 }
diff --git a/backend/src/main/scala/cromwell/backend/io/WorkflowPathsBackendInitializationData.scala b/backend/src/main/scala/cromwell/backend/io/WorkflowPathsBackendInitializationData.scala
deleted file mode 100644
index b0861d6bb..000000000
--- a/backend/src/main/scala/cromwell/backend/io/WorkflowPathsBackendInitializationData.scala
+++ /dev/null
@@ -1,44 +0,0 @@
-package cromwell.backend.io
-
-import cromwell.backend.BackendInitializationData
-import cromwell.core.path.PathBuilder
-
-/**
-  * Extension of backend initialization data that also provides a `WorkflowPaths`, and by proxy its `List[FileSystem]`.
-  *
-  * This class consolidates code used by the legacy Local backend and the Shared File System (SFS) backend, that need to
-  * localize data from a collection of different file systems (local and GCS for now).
-  *
-  * NOTE: The JES backend uses a fork of the `WorkflowPaths`. As `JesWorkflowPaths` is not a subclass, the
-  * initialization data for JES does not use this trait nor the companion object. It is not clear if JES needs the
-  * functionality, as it cannot localize from generic file systems, only GCS with different authentication modes.
-  *
-  * Meanwhile the local and SFS backends need access to the functionality in workflow paths. Their initialization data
-  * will therefore be a subset of this trait.
-  *
-  * The initialization data will be generated in the intialization actor as class extending this trait.
-  *
-  * From there, the initialization actor will pass the data back to the backend factory, wrapped in an Option, with its
-  * type downcast to an `Option[BackendInitializationData]`.
-  *
-  * The factory then sends a copy of the intialization data down to each execution actor created for every `wdl4s.Call`.
-  *
-  * Each instance of the execution actor will unpack the initialization data, obtaining the workflow paths using
-  * `WorkflowPathsBackendInitializationData.workflowPaths(Option[BackendInitializationData]): WorkflowPaths`
-  *
-  * However, the execution instances are most interested in actually getting to the file systems embedded within the
-  * workflow paths. That functionality is provided via `WorkflowPathsBackendInitializationData.workflowPaths`
-  */
-trait WorkflowPathsBackendInitializationData extends BackendInitializationData {
-  def workflowPaths: WorkflowPaths
-}
-
-object WorkflowPathsBackendInitializationData {
-  def workflowPaths(initializationDataOption: Option[BackendInitializationData]): WorkflowPaths = {
-    BackendInitializationData.as[WorkflowPathsBackendInitializationData](initializationDataOption).workflowPaths
-  }
-
-  def pathBuilders(initializationDataOption: Option[BackendInitializationData]): List[PathBuilder] = {
-    workflowPaths(initializationDataOption).pathBuilders
-  }
-}
diff --git a/backend/src/main/scala/cromwell/backend/io/WorkflowPathsWithDocker.scala b/backend/src/main/scala/cromwell/backend/io/WorkflowPathsWithDocker.scala
index c10e66972..26e6ce95d 100644
--- a/backend/src/main/scala/cromwell/backend/io/WorkflowPathsWithDocker.scala
+++ b/backend/src/main/scala/cromwell/backend/io/WorkflowPathsWithDocker.scala
@@ -1,16 +1,20 @@
 package cromwell.backend.io
 
-import java.nio.file.Paths
+import java.nio.file.{Path, Paths}
 
 import com.typesafe.config.Config
 import cromwell.backend.{BackendJobDescriptorKey, BackendWorkflowDescriptor}
 import cromwell.core.path.PathBuilder
 
 object WorkflowPathsWithDocker {
-  val DockerRoot = Paths.get("/root")
+  val DockerRoot: Path = Paths.get("/root")
 }
 
 class WorkflowPathsWithDocker(val workflowDescriptor: BackendWorkflowDescriptor, val config: Config, val pathBuilders: List[PathBuilder] = WorkflowPaths.DefaultPathBuilders) extends WorkflowPaths {
-  val dockerWorkflowRoot = workflowPathBuilder(WorkflowPathsWithDocker.DockerRoot)
-  override def toJobPaths(jobKey: BackendJobDescriptorKey): JobPaths = new JobPathsWithDocker(jobKey, workflowDescriptor, config, pathBuilders)
-}
\ No newline at end of file
+  val dockerWorkflowRoot: Path = workflowPathBuilder(WorkflowPathsWithDocker.DockerRoot)
+
+  override def toJobPaths(jobKey: BackendJobDescriptorKey,
+                          jobWorkflowDescriptor: BackendWorkflowDescriptor): JobPathsWithDocker = {
+    new JobPathsWithDocker(jobKey, jobWorkflowDescriptor, config, pathBuilders)
+  }
+}
diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala b/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala
index daabf40a1..a88f07fdb 100644
--- a/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala
+++ b/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala
@@ -1,7 +1,5 @@
 package cromwell.backend.standard
 
-import java.nio.file.Path
-
 import akka.actor.{Actor, ActorLogging, ActorRef}
 import akka.event.LoggingReceive
 import better.files.File
@@ -9,7 +7,7 @@ import cromwell.backend.BackendJobExecutionActor.{AbortedResponse, BackendJobExe
 import cromwell.backend.BackendLifecycleActor.AbortJobCommand
 import cromwell.backend.async.AsyncBackendJobExecutionActor.{ExecutionMode, JobId, Recover}
 import cromwell.backend.async.{AbortedExecutionHandle, AsyncBackendJobExecutionActor, ExecutionHandle, FailedNonRetryableExecutionHandle, PendingExecutionHandle, SuccessfulExecutionHandle}
-import cromwell.backend.validation.{ContinueOnReturnCode, ContinueOnReturnCodeFlag}
+import cromwell.backend.validation._
 import cromwell.backend.wdl.Command
 import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendJobDescriptor, BackendJobLifecycleActor}
 import cromwell.services.keyvalue.KeyValueServiceActor._
@@ -31,7 +29,7 @@ import scala.util.{Failure, Success, Try}
   * NOTE: Unlike the parent trait `AsyncBackendJobExecutionActor`, this trait is subject to even more frequent updates
   * as the common behavior among the backends adjusts in unison.
   */
-trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor {
+trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with StandardCachingActorHelper {
   this: Actor with ActorLogging with BackendJobLifecycleActor =>
 
   val SIGTERM = 143
@@ -48,26 +46,23 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor {
     PendingExecutionHandle[StandardAsyncJob, StandardAsyncRunInfo, StandardAsyncRunStatus]
 
   /** Standard set of parameters passed to the backend. */
-  val standardParams: StandardAsyncExecutionActorParams
-
-  override lazy val jobDescriptor: BackendJobDescriptor = standardParams.jobDescriptor
+  def standardParams: StandardAsyncExecutionActorParams
 
   override lazy val configurationDescriptor: BackendConfigurationDescriptor = standardParams.configurationDescriptor
 
   override lazy val completionPromise: Promise[BackendJobExecutionResponse] = standardParams.completionPromise
 
   /** Backend initialization data created by the a factory initializer. */
-  lazy val backendInitializationDataOption: Option[BackendInitializationData] =
+  override lazy val backendInitializationDataOption: Option[BackendInitializationData] =
     standardParams.backendInitializationDataOption
 
-  /** Typed backend initialization. */
-  def backendInitializationDataAs[A <: BackendInitializationData]: A =
-    BackendInitializationData.as[A](backendInitializationDataOption)
+  /** @see [[StandardAsyncExecutionActorParams.serviceRegistryActor]] */
+  override lazy val serviceRegistryActor: ActorRef = standardParams.serviceRegistryActor
 
-  /** @see [[StandardJobExecutionActorParams.serviceRegistryActor]] */
-  lazy val serviceRegistryActor: ActorRef = standardParams.serviceRegistryActor
+  /** @see [[StandardAsyncExecutionActorParams.jobDescriptor]] */
+  override lazy val jobDescriptor: BackendJobDescriptor = standardParams.jobDescriptor
 
-  /** @see [[StandardJobExecutionActorParams.jobIdKey]] */
+  /** @see [[StandardAsyncExecutionActorParams.jobIdKey]] */
   def jobIdKey: String = standardParams.jobIdKey
 
   /** @see [[Command.instantiate]] */
@@ -91,48 +86,16 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor {
     *
     * @return True if a non-empty `remoteStdErrPath` should fail the job.
     */
-  def failOnStdErr: Boolean = false
-
-  /**
-    * Returns the path to the standard error output of the job. Only needs to be implemented if `failOnStdErr` is
-    * returning `true`.
-    *
-    * @return The path to the standard error output.
-    */
-  def remoteStdErrPath: Path = {
-    throw new NotImplementedError(s"failOnStdErr returned true but remote path not implemented by $getClass")
-  }
-
-  /**
-    * Returns the path to the return code output of the job. Must be implemented unless `returnCodeContents` is
-    * overridden not to use this method.
-    *
-    * @return The path to the return code output.
-    */
-  def remoteReturnCodePath: Path = {
-    throw new NotImplementedError(s"remoteReturnCodePath returned true but remote path not implemented by $getClass")
-  }
-
-  /**
-    * Returns the contents of the return code file.
-    *
-    * @return The contents of the return code file.
-    */
-  def returnCodeContents: String = File(remoteReturnCodePath).contentAsString
+  lazy val failOnStdErr: Boolean = RuntimeAttributesValidation.extract(
+    FailOnStderrValidation.instance, validatedRuntimeAttributes)
 
   /**
     * Returns the behavior for continuing on the return code, obtained by converting `returnCodeContents` to an Int.
     *
     * @return the behavior for continuing on the return code.
     */
-  def continueOnReturnCode: ContinueOnReturnCode = ContinueOnReturnCodeFlag(false)
-
-  /**
-    * Returns the metadata key values to store before executing a job.
-    *
-    * @return the metadata key values to store before executing a job.
-    */
-  def startMetadataKeyValues: Map[String, Any] = Map.empty
+  lazy val continueOnReturnCode: ContinueOnReturnCode = RuntimeAttributesValidation.extract(
+    ContinueOnReturnCodeValidation.instance, validatedRuntimeAttributes)
 
   /**
     * Execute the job specified in the params. Should return a `StandardAsyncPendingExecutionHandle`, or a
@@ -410,8 +373,8 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor {
     try {
       if (isSuccess(status)) {
 
-        lazy val stderrLength: Long = File(remoteStdErrPath).size
-        lazy val returnCode: Try[Int] = Try(returnCodeContents).map(_.trim.toInt)
+        lazy val stderrLength: Long = File(jobPaths.stderr).size
+        lazy val returnCode: Try[Int] = Try(File(jobPaths.returnCode).contentAsString).map(_.trim.toInt)
         status match {
           case _ if failOnStdErr && stderrLength.intValue > 0 =>
             // returnCode will be None if it couldn't be downloaded/parsed, which will yield a null in the DB
diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardCacheHitCopyingActor.scala b/backend/src/main/scala/cromwell/backend/standard/StandardCacheHitCopyingActor.scala
new file mode 100644
index 000000000..7f5531e94
--- /dev/null
+++ b/backend/src/main/scala/cromwell/backend/standard/StandardCacheHitCopyingActor.scala
@@ -0,0 +1,46 @@
+package cromwell.backend.standard
+
+import java.nio.file.Path
+
+import akka.actor.ActorRef
+import cromwell.backend.callcaching.CacheHitDuplicating
+import cromwell.backend.{BackendCacheHitCopyingActor, BackendConfigurationDescriptor, BackendInitializationData, BackendJobDescriptor}
+
+/**
+  * Trait of parameters passed to a StandardCacheHitCopyingActor.
+  */
+trait StandardCacheHitCopyingActorParams {
+  def jobDescriptor: BackendJobDescriptor
+
+  def backendInitializationDataOption: Option[BackendInitializationData]
+
+  def serviceRegistryActor: ActorRef
+
+  def configurationDescriptor: BackendConfigurationDescriptor
+}
+
+/** A default implementation of the cache hit copying params. */
+case class DefaultStandardCacheHitCopyingActorParams
+(
+  override val jobDescriptor: BackendJobDescriptor,
+  override val backendInitializationDataOption: Option[BackendInitializationData],
+  override val serviceRegistryActor: ActorRef,
+  override val configurationDescriptor: BackendConfigurationDescriptor
+) extends StandardCacheHitCopyingActorParams
+
+/**
+  * Standard implementation of a BackendCacheHitCopyingActor.
+  */
+trait StandardCacheHitCopyingActor extends BackendCacheHitCopyingActor with CacheHitDuplicating
+  with StandardCachingActorHelper {
+
+  def standardParams: StandardCacheHitCopyingActorParams
+
+  override lazy val jobDescriptor: BackendJobDescriptor = standardParams.jobDescriptor
+  override lazy val backendInitializationDataOption: Option[BackendInitializationData] =
+    standardParams.backendInitializationDataOption
+  override lazy val serviceRegistryActor: ActorRef = standardParams.serviceRegistryActor
+  override lazy val configurationDescriptor: BackendConfigurationDescriptor = standardParams.configurationDescriptor
+  override lazy val destinationCallRootPath: Path = jobPaths.callRoot
+  override lazy val destinationJobDetritusPaths: Map[String, Path] = jobPaths.detritusPaths
+}
diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardCachingActorHelper.scala b/backend/src/main/scala/cromwell/backend/standard/StandardCachingActorHelper.scala
new file mode 100644
index 000000000..db93cc1b5
--- /dev/null
+++ b/backend/src/main/scala/cromwell/backend/standard/StandardCachingActorHelper.scala
@@ -0,0 +1,89 @@
+package cromwell.backend.standard
+
+import java.nio.file.Path
+
+import akka.actor.{Actor, ActorRef}
+import cromwell.backend._
+import cromwell.backend.callcaching.JobCachingActorHelper
+import cromwell.backend.io.{JobPaths, WorkflowPaths}
+import cromwell.backend.validation.{RuntimeAttributesValidation, ValidatedRuntimeAttributes}
+import cromwell.core.logging.JobLogging
+import wdl4s.TaskCall
+
+import scala.util.Try
+
+/**
+  * Extends the JobCachingActorHelper with standard implementations.
+  *
+  * Like the JobCachingActorHelper, this trait should be extended by a backend, and then __that__ trait should be mixed
+  * into a async job execution actor, and cache hit copying actor.
+  */
+trait StandardCachingActorHelper extends JobCachingActorHelper {
+  this: Actor with JobLogging =>
+
+  def backendInitializationDataOption: Option[BackendInitializationData]
+
+  /** Typed backend initialization. */
+  def backendInitializationDataAs[A <: BackendInitializationData]: A =
+    BackendInitializationData.as[A](backendInitializationDataOption)
+
+  /**
+    * Returns the service registry actor. Both the StandardAsyncExecutorActor and StandardCacheHitCopyingActor traits
+    * implement this method.
+    *
+    * @return Paths to the job.
+    */
+  def serviceRegistryActor: ActorRef
+
+  // So... JobPaths doesn't extend WorkflowPaths, but does contain a self-type
+  lazy val workflowPaths: WorkflowPaths = jobPaths.asInstanceOf[WorkflowPaths]
+
+  def getPath(str: String): Try[Path] = workflowPaths.getPath(str)
+
+  /**
+    * The workflow descriptor for this job. NOTE: This may be different than the workflow descriptor created in the
+    * workflow initialization data. For example, sub workflows use a different workflow descriptor.
+    */
+  lazy val workflowDescriptor: BackendWorkflowDescriptor = jobDescriptor.workflowDescriptor
+
+  lazy val call: TaskCall = jobDescriptor.key.call
+
+  lazy val standardInitializationData: StandardInitializationData = BackendInitializationData.
+    as[StandardInitializationData](backendInitializationDataOption)
+
+  lazy val validatedRuntimeAttributes: ValidatedRuntimeAttributes = {
+    val builder = standardInitializationData.runtimeAttributesBuilder
+    builder.build(jobDescriptor.runtimeAttributes, jobLogger)
+  }
+
+  /**
+    * Returns the paths to the job.
+    *
+    * @return Paths to the job.
+    */
+  lazy val jobPaths: JobPaths = standardInitializationData.workflowPaths.toJobPaths(jobDescriptor)
+
+  /**
+    * Returns the metadata key values to store before executing a job.
+    *
+    * @return the metadata key values to store before executing a job.
+    */
+  def startMetadataKeyValues: Map[String, Any] = {
+    val runtimeAttributesMetadata = RuntimeAttributesValidation.toMetadataStrings(validatedRuntimeAttributes) map {
+      case (key, value) => (s"runtimeAttributes:$key", value)
+    }
+
+    val fileMetadata = jobPaths.metadataPaths
+
+    val otherMetadata = Map("cache:allowResultReuse" -> true)
+
+    runtimeAttributesMetadata ++ fileMetadata ++ otherMetadata ++ nonStandardMetadata
+  }
+
+  /**
+    * Returns any custom medatata for the backend.
+    *
+    * @return any custom metadata for the backend.
+    */
+  protected def nonStandardMetadata: Map[String, Any] = Map.empty
+}
diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardFinalizationActor.scala b/backend/src/main/scala/cromwell/backend/standard/StandardFinalizationActor.scala
new file mode 100644
index 000000000..19500a90d
--- /dev/null
+++ b/backend/src/main/scala/cromwell/backend/standard/StandardFinalizationActor.scala
@@ -0,0 +1,41 @@
+package cromwell.backend.standard
+
+import cromwell.backend._
+import cromwell.core.CallOutputs
+import wdl4s.TaskCall
+
+trait StandardFinalizationActorParams {
+  def workflowDescriptor: BackendWorkflowDescriptor
+
+  def calls: Set[TaskCall]
+
+  def jobExecutionMap: JobExecutionMap
+
+  def workflowOutputs: CallOutputs
+
+  def initializationDataOption: Option[BackendInitializationData]
+
+  def configurationDescriptor: BackendConfigurationDescriptor
+}
+
+case class DefaultStandardFinalizationActorParams
+(
+  workflowDescriptor: BackendWorkflowDescriptor,
+  calls: Set[TaskCall],
+  jobExecutionMap: JobExecutionMap,
+  workflowOutputs: CallOutputs,
+  initializationDataOption: Option[BackendInitializationData],
+  configurationDescriptor: BackendConfigurationDescriptor
+) extends StandardFinalizationActorParams
+
+trait StandardFinalizationActor extends BackendWorkflowFinalizationActor {
+
+  def standardParams: StandardFinalizationActorParams
+
+  override lazy val workflowDescriptor: BackendWorkflowDescriptor = standardParams.workflowDescriptor
+  override lazy val calls: Set[TaskCall] = standardParams.calls
+  lazy val initializationDataOption: Option[BackendInitializationData] = standardParams.initializationDataOption
+  lazy val jobExecutionMap: JobExecutionMap = standardParams.jobExecutionMap
+  lazy val workflowOutputs: CallOutputs = standardParams.workflowOutputs
+  override lazy val configurationDescriptor: BackendConfigurationDescriptor = standardParams.configurationDescriptor
+}
diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardInitializationActor.scala b/backend/src/main/scala/cromwell/backend/standard/StandardInitializationActor.scala
new file mode 100644
index 000000000..1fbfc92d1
--- /dev/null
+++ b/backend/src/main/scala/cromwell/backend/standard/StandardInitializationActor.scala
@@ -0,0 +1,70 @@
+package cromwell.backend.standard
+
+import akka.actor.ActorRef
+import cromwell.backend.validation.RuntimeAttributesDefault
+import cromwell.backend.{BackendConfigurationDescriptor, BackendWorkflowDescriptor, BackendWorkflowInitializationActor}
+import cromwell.core.WorkflowOptions
+import wdl4s.TaskCall
+import wdl4s.values.WdlValue
+
+import scala.concurrent.Future
+import scala.util.Try
+
+trait StandardInitializationActorParams {
+  def workflowDescriptor: BackendWorkflowDescriptor
+
+  def calls: Set[TaskCall]
+
+  def serviceRegistryActor: ActorRef
+
+  def configurationDescriptor: BackendConfigurationDescriptor
+}
+
+case class DefaultInitializationActorParams
+(
+  workflowDescriptor: BackendWorkflowDescriptor,
+  calls: Set[TaskCall],
+  serviceRegistryActor: ActorRef,
+  configurationDescriptor: BackendConfigurationDescriptor
+) extends StandardInitializationActorParams
+
+trait StandardInitializationActor extends BackendWorkflowInitializationActor {
+
+  def standardParams: StandardInitializationActorParams
+
+  override lazy val serviceRegistryActor: ActorRef = standardParams.serviceRegistryActor
+
+  override lazy val calls: Set[TaskCall] = standardParams.calls
+
+  def runtimeAttributesBuilder: StandardValidatedRuntimeAttributesBuilder =
+    StandardValidatedRuntimeAttributesBuilder.default
+
+  override protected def runtimeAttributeValidators: Map[String, (Option[WdlValue]) => Boolean] = {
+    runtimeAttributesBuilder.validatorMap
+  }
+
+  override protected def coerceDefaultRuntimeAttributes(options: WorkflowOptions): Try[Map[String, WdlValue]] = {
+    RuntimeAttributesDefault.workflowOptionsDefault(options, runtimeAttributesBuilder.coercionMap)
+  }
+
+  override def validate(): Future[Unit] = {
+    Future.fromTry(Try {
+      calls foreach { call =>
+        val runtimeAttributeKeys = call.task.runtimeAttributes.attrs.keys.toList
+        val notSupportedAttributes = runtimeAttributesBuilder.unsupportedKeys(runtimeAttributeKeys).toList
+
+        if (notSupportedAttributes.nonEmpty) {
+          val notSupportedAttrString = notSupportedAttributes mkString ", "
+          workflowLogger.warn(
+            s"Key/s [$notSupportedAttrString] is/are not supported by backend. " +
+              s"Unsupported attributes will not be part of job executions.")
+        }
+      }
+    })
+  }
+
+  override protected def workflowDescriptor: BackendWorkflowDescriptor = standardParams.workflowDescriptor
+
+  override protected def configurationDescriptor: BackendConfigurationDescriptor =
+    standardParams.configurationDescriptor
+}
diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardInitializationData.scala b/backend/src/main/scala/cromwell/backend/standard/StandardInitializationData.scala
new file mode 100644
index 000000000..e9cdb2f35
--- /dev/null
+++ b/backend/src/main/scala/cromwell/backend/standard/StandardInitializationData.scala
@@ -0,0 +1,21 @@
+package cromwell.backend.standard
+
+import cromwell.backend.BackendInitializationData
+import cromwell.backend.io.WorkflowPaths
+import cromwell.core.path.PathBuilder
+
+class StandardInitializationData
+(
+  val workflowPaths: WorkflowPaths,
+  val runtimeAttributesBuilder: StandardValidatedRuntimeAttributesBuilder
+) extends BackendInitializationData
+
+object StandardInitializationData {
+  def workflowPaths(initializationDataOption: Option[BackendInitializationData]): WorkflowPaths = {
+    BackendInitializationData.as[StandardInitializationData](initializationDataOption).workflowPaths
+  }
+
+  def pathBuilders(initializationDataOption: Option[BackendInitializationData]): List[PathBuilder] = {
+    workflowPaths(initializationDataOption).pathBuilders
+  }
+}
diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardJobExecutionActorParams.scala b/backend/src/main/scala/cromwell/backend/standard/StandardJobExecutionActorParams.scala
index 9242aa601..1261119d9 100644
--- a/backend/src/main/scala/cromwell/backend/standard/StandardJobExecutionActorParams.scala
+++ b/backend/src/main/scala/cromwell/backend/standard/StandardJobExecutionActorParams.scala
@@ -12,19 +12,22 @@ import scala.language.existentials
   */
 trait StandardJobExecutionActorParams {
   /** The service registry actor for key/value and metadata. */
-  val serviceRegistryActor: ActorRef
+  def serviceRegistryActor: ActorRef
 
   /** The descriptor of this job. */
-  val jobDescriptor: BackendJobDescriptor
+  def jobDescriptor: BackendJobDescriptor
 
   /** The global and backend configuration. */
-  val configurationDescriptor: BackendConfigurationDescriptor
+  def configurationDescriptor: BackendConfigurationDescriptor
 
   /** Any backend initialization data. */
-  val backendInitializationDataOption: Option[BackendInitializationData]
+  def backendInitializationDataOption: Option[BackendInitializationData]
 
   /** The key for this job. */
-  val jobIdKey: String
+  def jobIdKey: String
+
+  /** The singleton actor. */
+  def backendSingletonActorOption: Option[ActorRef]
 }
 
 /**
@@ -36,7 +39,7 @@ trait StandardSyncExecutionActorParams extends StandardJobExecutionActorParams {
     *
     * @see [[StandardSyncExecutionActor]]
     */
-  val asyncJobExecutionActorClass: Class[_ <: StandardAsyncExecutionActor]
+  def asyncJobExecutionActorClass: Class[_ <: StandardAsyncExecutionActor]
 }
 
 /** A default implementation of the sync params. */
@@ -47,6 +50,7 @@ case class DefaultStandardSyncExecutionActorParams
   override val jobDescriptor: BackendJobDescriptor,
   override val configurationDescriptor: BackendConfigurationDescriptor,
   override val backendInitializationDataOption: Option[BackendInitializationData],
+  override val backendSingletonActorOption: Option[ActorRef],
   override val asyncJobExecutionActorClass: Class[_ <: StandardAsyncExecutionActor]
 ) extends StandardSyncExecutionActorParams
 
@@ -55,7 +59,7 @@ case class DefaultStandardSyncExecutionActorParams
   */
 trait StandardAsyncExecutionActorParams extends StandardJobExecutionActorParams {
   /** The promise that will be completed when the async run is complete. */
-  val completionPromise: Promise[BackendJobExecutionResponse]
+  def completionPromise: Promise[BackendJobExecutionResponse]
 }
 
 /** A default implementation of the async params. */
@@ -66,5 +70,6 @@ case class DefaultStandardAsyncExecutionActorParams
   override val jobDescriptor: BackendJobDescriptor,
   override val configurationDescriptor: BackendConfigurationDescriptor,
   override val backendInitializationDataOption: Option[BackendInitializationData],
+  override val backendSingletonActorOption: Option[ActorRef],
   override val completionPromise: Promise[BackendJobExecutionResponse]
 ) extends StandardAsyncExecutionActorParams
diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardLifecycleActorFactory.scala b/backend/src/main/scala/cromwell/backend/standard/StandardLifecycleActorFactory.scala
index 6a0f9324b..aa7e57356 100644
--- a/backend/src/main/scala/cromwell/backend/standard/StandardLifecycleActorFactory.scala
+++ b/backend/src/main/scala/cromwell/backend/standard/StandardLifecycleActorFactory.scala
@@ -1,8 +1,13 @@
 package cromwell.backend.standard
 
+import java.nio.file.Path
+
 import akka.actor.{ActorRef, Props}
+import com.typesafe.config.Config
 import cromwell.backend._
-import cromwell.core.Dispatcher
+import cromwell.core.{CallOutputs, Dispatcher}
+import cromwell.core.Dispatcher.BackendDispatcher
+import wdl4s.TaskCall
 
 /**
   * May be extended for using the standard sync/async backend pattern.
@@ -18,11 +23,40 @@ trait StandardLifecycleActorFactory extends BackendLifecycleActorFactory {
   def configurationDescriptor: BackendConfigurationDescriptor
 
   /**
-    * Returns the main engine for async execution.
+    * Returns the initialization class.
+    *
+    * @return the initialization class.
+    */
+  def initializationActorClass: Class[_ <: StandardInitializationActor]
+
+  /**
+    * Returns the synchronous executor class. By default using the standard sync executor should be sufficient for most
+    * implementations.
     *
-    * @return the main engine for async execution.
+    * @return the synchronous executor class.
     */
-  def asyncJobExecutionActorClass: Class[_ <: StandardAsyncExecutionActor]
+  def syncExecutionActorClass: Class[_ <: StandardSyncExecutionActor] = classOf[StandardSyncExecutionActor]
+
+  /**
+    * Returns the asynchronous executor class.
+    *
+    * @return the asynchronous executor class.
+    */
+  def asyncExecutionActorClass: Class[_ <: StandardAsyncExecutionActor]
+
+  /**
+    * Returns the cache hit copying class.
+    *
+    * @return the cache hit copying class.
+    */
+  def standardCacheHitCopyingActorOption: Option[Class[_ <: StandardCacheHitCopyingActor]] = None
+
+  /**
+    * Returns the finalization class.
+    *
+    * @return the finalization class.
+    */
+  def finalizationActorClassOption: Option[Class[_ <: StandardFinalizationActor]] = None
 
   /**
     * Returns the key to use for storing and looking up the job id.
@@ -31,12 +65,83 @@ trait StandardLifecycleActorFactory extends BackendLifecycleActorFactory {
     */
   def jobIdKey: String
 
+  override def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[TaskCall],
+                                                serviceRegistryActor: ActorRef): Option[Props] = {
+    val params = workflowInitializationActorParams(workflowDescriptor, calls, serviceRegistryActor)
+    val props = Props(initializationActorClass, params).withDispatcher(Dispatcher.BackendDispatcher)
+    Option(props)
+  }
+
+  def workflowInitializationActorParams(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[TaskCall],
+                                        serviceRegistryActor: ActorRef): StandardInitializationActorParams = {
+    DefaultInitializationActorParams(workflowDescriptor, calls, serviceRegistryActor, configurationDescriptor)
+  }
+
   override def jobExecutionActorProps(jobDescriptor: BackendJobDescriptor,
                                       initializationDataOption: Option[BackendInitializationData],
                                       serviceRegistryActor: ActorRef,
-                                      backendSingletonActor: Option[ActorRef]): Props = {
-    val params = DefaultStandardSyncExecutionActorParams(jobIdKey, serviceRegistryActor,
-      jobDescriptor, configurationDescriptor, initializationDataOption, asyncJobExecutionActorClass)
+                                      backendSingletonActorOption: Option[ActorRef]): Props = {
+    val params = jobExecutionActorParams(jobDescriptor, initializationDataOption, serviceRegistryActor,
+      backendSingletonActorOption)
     Props(new StandardSyncExecutionActor(params)).withDispatcher(Dispatcher.BackendDispatcher)
   }
+
+  def jobExecutionActorParams(jobDescriptor: BackendJobDescriptor,
+                              initializationDataOption: Option[BackendInitializationData],
+                              serviceRegistryActor: ActorRef,
+                              backendSingletonActorOption: Option[ActorRef]): StandardSyncExecutionActorParams = {
+    DefaultStandardSyncExecutionActorParams(jobIdKey, serviceRegistryActor, jobDescriptor, configurationDescriptor,
+      initializationDataOption, backendSingletonActorOption, asyncExecutionActorClass)
+  }
+
+  override def cacheHitCopyingActorProps:
+  Option[(BackendJobDescriptor, Option[BackendInitializationData], ActorRef) => Props] = {
+    standardCacheHitCopyingActorOption map {
+      standardCacheHitCopyingActor => cacheHitCopyingActorInner(standardCacheHitCopyingActor) _
+    }
+  }
+
+  def cacheHitCopyingActorInner(standardCacheHitCopyingActor: Class[_ <: StandardCacheHitCopyingActor])
+                               (jobDescriptor: BackendJobDescriptor,
+                                initializationDataOption: Option[BackendInitializationData],
+                                serviceRegistryActor: ActorRef): Props = {
+    val params = cacheHitCopyingActorParams(jobDescriptor, initializationDataOption, serviceRegistryActor)
+    Props(standardCacheHitCopyingActor, params).withDispatcher(BackendDispatcher)
+  }
+
+  def cacheHitCopyingActorParams(jobDescriptor: BackendJobDescriptor,
+                                 initializationDataOption: Option[BackendInitializationData],
+                                 serviceRegistryActor: ActorRef): StandardCacheHitCopyingActorParams = {
+    DefaultStandardCacheHitCopyingActorParams(
+      jobDescriptor, initializationDataOption, serviceRegistryActor, configurationDescriptor)
+  }
+
+  override def workflowFinalizationActorProps(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[TaskCall],
+                                              jobExecutionMap: JobExecutionMap, workflowOutputs: CallOutputs,
+                                              initializationData: Option[BackendInitializationData]): Option[Props] = {
+    finalizationActorClassOption map { finalizationActorClass =>
+      val params = workflowFinalizationActorParams(workflowDescriptor, calls, jobExecutionMap, workflowOutputs,
+        initializationData)
+      Props(finalizationActorClass, params).withDispatcher(BackendDispatcher)
+    }
+  }
+
+  def workflowFinalizationActorParams(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[TaskCall],
+                                      jobExecutionMap: JobExecutionMap, workflowOutputs: CallOutputs,
+                                      initializationDataOption: Option[BackendInitializationData]):
+  StandardFinalizationActorParams = {
+    DefaultStandardFinalizationActorParams(workflowDescriptor, calls, jobExecutionMap, workflowOutputs,
+      initializationDataOption, configurationDescriptor)
+  }
+
+  override def getExecutionRootPath(workflowDescriptor: BackendWorkflowDescriptor, backendConfig: Config,
+                                    initializationData: Option[BackendInitializationData]): Path = {
+    initializationData.get.asInstanceOf[StandardInitializationData].workflowPaths.executionRoot
+  }
+
+  override def getWorkflowExecutionRootPath(workflowDescriptor: BackendWorkflowDescriptor, backendConfig: Config,
+                                            initializationData: Option[BackendInitializationData]): Path = {
+    initializationData.get.asInstanceOf[StandardInitializationData].workflowPaths.workflowRoot
+  }
+
 }
diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardSyncExecutionActor.scala b/backend/src/main/scala/cromwell/backend/standard/StandardSyncExecutionActor.scala
index 52bba0153..472fffa0f 100644
--- a/backend/src/main/scala/cromwell/backend/standard/StandardSyncExecutionActor.scala
+++ b/backend/src/main/scala/cromwell/backend/standard/StandardSyncExecutionActor.scala
@@ -97,6 +97,7 @@ class StandardSyncExecutionActor(val standardParams: StandardSyncExecutionActorP
       standardParams.jobDescriptor,
       standardParams.configurationDescriptor,
       standardParams.backendInitializationDataOption,
+      standardParams.backendSingletonActorOption,
       completionPromise
     )
   }
diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilder.scala b/backend/src/main/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilder.scala
index 55be9e663..20a649e00 100644
--- a/backend/src/main/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilder.scala
+++ b/backend/src/main/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilder.scala
@@ -1,105 +1,50 @@
-package cromwell.backend.sfs
+package cromwell.backend.standard
 
 import cromwell.backend.validation._
 
 /**
-  * Validates a collection of runtime attributes for a Shared File System (SFS) Backend.
+  * Validates a collection of runtime attributes for a Standard Backend.
   *
-  * There are always three collections of runtime attributes, two of which are required for the SFS base class:
+  * There are two collections of runtime attributes, the first that are required for the standard async execution, and
+  * custom validations that the backend sub class may specify.
   *
-  * Required for the SFS to be able to run properly:
-  * 1) The set of runtime attributes that are absolutely required, or the validation fails.
-  * 2) _Extra_ validations that will run for the SFS.
+  * Currently the required validations are:
+  * - `ContinueOnReturnCodeValidation.default`
+  * - `FailOnStderrValidation.default`
   *
-  * Lastly for the sub classes of the SFS, there are:
-  * 3) Custom validations that the backend sub class may specify.
-  *
-  * 3), the custom validations, are always set by the sub class, via calls to `withValidation()`.
-  *
-  * For 1) and 2) the biggest difference is when it comes to docker support in a backend.
-  *
-  * For a backend that does __not__ support docker, the default, 1) and 2) will contain:
-  *
-  * 1) required         = ContinueOnReturnCodeValidation.default, FailOnStderrValidation.default
-  * 2) unsupportedExtra = DockerValidation.optional
-  *
-  * Suppose the above validation runs a WDL with runtime attributes shows up with:
-  *
-  * {{{
-  *   runtimeAttributes {
-  *     continueOnReturnCode: 14
-  *     # failOnStdErr not specified
-  *     docker: "ubuntu"
-  *   }
-  * }}}
-  *
-  * This will cause a validation warning to print out that docker is unsupported, because the docker validation isn't
-  * present in the required validations. If/when the SFS backend asks what the value of the optional docker is, it will
-  * receive `None`, as the validation _is not_ listed in the required validations.
-  *
-  * There is an additional interesting thing about having the docker validation still running under an "unsupported
-  * extra validation". Say an invalid docker were to be specified as a WdlArray. The extra validation __would__ catch
-  * the error, even after the warning had been printed stating that docker is not supported by the backend.
-  *
-  * Meanwhile, even though failOnStdErr is not specified, the `FailOnStderrValidation.default` will return its default
-  * value. And of course, the `ContinueOnReturnCodeValidation.default` returns the specified, and valid, runtime
-  * attribute `ContinueOnReturnCodeSet(14)`.
-  *
-  * Now--
-  *
-  * Suppose the `withDockerSupport(true)` has been invoked on the builder. The required and unsupported runtime
-  * attributes will then look like:
-  *
-  * 1) required         = ContinueOnReturnCodeValidation.default, FailOnStderrValidation.default,
-  *                       DockerValidation.optional
-  * 2) unsupportedExtra = __empty__
-  *
-  * With the same WDL above, this builder does NOT print a warning, because docker __is__ supported. When the SFS asks
-  * for the optional docker element, it receives `Some("ubuntu")`, as the `DockerValidation.optional` __is__ listed in
-  * the required validations.
-  *
-  * `ContinueOnReturnCodeValidation.default` and `FailOnStderrValidation.default` still operate as in the previous
-  * example.
-  *
-  * What happens when there is no runtime attribute for docker? Easy, the validation for docker is always optional!
-  * In either case of running a builder via `withDockerSupport(true)` or `withDockerSupport(false)`, if the docker
-  * runtime attribute is not specified, the `SharedFileSystemValidatedRuntimeAttributesBuilder` will return a
-  * `None` value.
+  * NOTE: The required runtime attributes may be moved to the engine in the future.
   */
-object SharedFileSystemValidatedRuntimeAttributesBuilder {
+object StandardValidatedRuntimeAttributesBuilder {
 
-  private case class SharedFileSystemValidatedRuntimeAttributesBuilderImpl
+  private case class StandardValidatedRuntimeAttributesBuilderImpl
   (
     override val requiredValidations: Seq[RuntimeAttributesValidation[_]],
     override val customValidations: Seq[RuntimeAttributesValidation[_]]
-  ) extends SharedFileSystemValidatedRuntimeAttributesBuilder
+  ) extends StandardValidatedRuntimeAttributesBuilder
 
   /**
-    * `default` returns the default set of attributes required to run an SFS:
+    * `default` returns the default set of attributes required to run a standard backend:
     * - `ContinueOnReturnCodeValidation.default`
     * - `FailOnStderrValidation.default`
     *
     * Additional runtime attribute validations may be added by calling `withValidation` on the default.
-    *
-    * The SFS will also _always_ validate using the `DockerValidation`, but will end up warning the user that the
-    * runtime attribute is unsupported by the backend implementation unless `withDockerSupport(true)` is called.
     */
-  lazy val default: SharedFileSystemValidatedRuntimeAttributesBuilder = {
+  lazy val default: StandardValidatedRuntimeAttributesBuilder = {
     val required = Seq(ContinueOnReturnCodeValidation.default, FailOnStderrValidation.default)
     val custom = Seq.empty
-    SharedFileSystemValidatedRuntimeAttributesBuilderImpl(custom, required)
+    StandardValidatedRuntimeAttributesBuilderImpl(custom, required)
   }
 
-  private def withValidations(builder: SharedFileSystemValidatedRuntimeAttributesBuilder,
+  private def withValidations(builder: StandardValidatedRuntimeAttributesBuilder,
                               customValidations: Seq[RuntimeAttributesValidation[_]]):
-  SharedFileSystemValidatedRuntimeAttributesBuilder = {
+  StandardValidatedRuntimeAttributesBuilder = {
     val required = builder.requiredValidations
     val custom = builder.customValidations ++ customValidations
-    SharedFileSystemValidatedRuntimeAttributesBuilderImpl(custom, required)
+    StandardValidatedRuntimeAttributesBuilderImpl(custom, required)
   }
 }
 
-sealed trait SharedFileSystemValidatedRuntimeAttributesBuilder extends ValidatedRuntimeAttributesBuilder {
+sealed trait StandardValidatedRuntimeAttributesBuilder extends ValidatedRuntimeAttributesBuilder {
   /**
     * Returns a new builder with the additional validation(s).
     *
@@ -107,14 +52,14 @@ sealed trait SharedFileSystemValidatedRuntimeAttributesBuilder extends Validated
     * @return New builder with the validation.
     */
   final def withValidation(validation: RuntimeAttributesValidation[_]*):
-  SharedFileSystemValidatedRuntimeAttributesBuilder = {
-    SharedFileSystemValidatedRuntimeAttributesBuilder.withValidations(this, validation)
+  StandardValidatedRuntimeAttributesBuilder = {
+    StandardValidatedRuntimeAttributesBuilder.withValidations(this, validation)
   }
 
-  /** Returns on the supported validations, those required for the SFS, plus custom addons for the subclass. */
+  /** Returns all the validations, those required for the standard backend, plus custom addons for the subclass. */
   override final lazy val validations: Seq[RuntimeAttributesValidation[_]] = requiredValidations ++ customValidations
 
-  private[sfs] def requiredValidations: Seq[RuntimeAttributesValidation[_]]
+  private[standard] def requiredValidations: Seq[RuntimeAttributesValidation[_]]
 
-  private[sfs] def customValidations: Seq[RuntimeAttributesValidation[_]]
+  private[standard] def customValidations: Seq[RuntimeAttributesValidation[_]]
 }
diff --git a/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala b/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala
index db0cacaf0..d0ca49339 100644
--- a/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala
+++ b/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala
@@ -6,8 +6,8 @@ import cats.syntax.traverse._
 import cats.syntax.validated._
 import cromwell.backend.validation.RuntimeAttributesValidation._
 import lenthall.validation.ErrorOr._
-import wdl4s.types.{WdlArrayType, WdlIntegerType, WdlStringType}
-import wdl4s.values.{WdlArray, WdlBoolean, WdlInteger, WdlString}
+import wdl4s.types.{WdlArrayType, WdlIntegerType, WdlStringType, WdlType}
+import wdl4s.values.{WdlArray, WdlBoolean, WdlInteger, WdlString, WdlValue}
 
 import scala.util.Try
 
@@ -23,45 +23,44 @@ import scala.util.Try
   * found.
   */
 object ContinueOnReturnCodeValidation {
-  val key = RuntimeAttributesKeys.ContinueOnReturnCodeKey
-
-  lazy val instance = new ContinueOnReturnCodeValidation
-
-  lazy val default = instance.withDefault(WdlInteger(0))
-
-  lazy val optional = default.optional
-
-  private[validation] val missingMessage =
-    s"Expecting $key runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]"
+  lazy val instance: RuntimeAttributesValidation[ContinueOnReturnCode] = new ContinueOnReturnCodeValidation
+  lazy val default: RuntimeAttributesValidation[ContinueOnReturnCode] = instance.withDefault(WdlInteger(0))
+  lazy val optional: OptionalRuntimeAttributesValidation[ContinueOnReturnCode] = default.optional
 }
 
 class ContinueOnReturnCodeValidation extends RuntimeAttributesValidation[ContinueOnReturnCode] {
 
-  import ContinueOnReturnCodeValidation._
+  override def key: String = RuntimeAttributesKeys.ContinueOnReturnCodeKey
 
-  override def key = RuntimeAttributesKeys.ContinueOnReturnCodeKey
+  override def coercion: Set[WdlType] = ContinueOnReturnCode.validWdlTypes
 
-  override def coercion = ContinueOnReturnCode.validWdlTypes
-
-  override def validateValue = {
+  override def validateValue: PartialFunction[WdlValue, ErrorOr[ContinueOnReturnCode]] = {
     case WdlBoolean(value) => ContinueOnReturnCodeFlag(value).validNel
     case WdlString(value) if Try(value.toBoolean).isSuccess => ContinueOnReturnCodeFlag(value.toBoolean).validNel
+    case WdlString(value) if Try(value.toInt).isSuccess => ContinueOnReturnCodeSet(Set(value.toInt)).validNel
     case WdlInteger(value) => ContinueOnReturnCodeSet(Set(value)).validNel
-    case WdlArray(wdlType, seq) =>
+    case value@WdlArray(_, seq) =>
       val errorOrInts: ErrorOr[List[Int]] = (seq.toList map validateInt).sequence[ErrorOr, Int]
       errorOrInts match {
         case Valid(ints) => ContinueOnReturnCodeSet(ints.toSet).validNel
-        case Invalid(_) => failureWithMessage
+        case Invalid(_) => invalidValueFailure(value)
       }
   }
 
-  override def validateExpression = {
-    case _: WdlBoolean => true
+  override def validateExpression: PartialFunction[WdlValue, Boolean] = {
+    case WdlBoolean(_) => true
+    case WdlString(value) if Try(value.toInt).isSuccess => true
     case WdlString(value) if Try(value.toBoolean).isSuccess => true
-    case _: WdlInteger => true
-    case WdlArray(WdlArrayType(WdlStringType), elements) => elements.forall(validateInt(_).isValid)
-    case WdlArray(WdlArrayType(WdlIntegerType), elements) => elements.forall(validateInt(_).isValid)
+    case WdlInteger(_) => true
+    case WdlArray(WdlArrayType(WdlStringType), elements) =>
+      elements forall {
+        value => Try(value.valueString.toInt).isSuccess
+      }
+    case WdlArray(WdlArrayType(WdlIntegerType), _) => true
   }
 
-  override protected def failureMessage = missingMessage
+  override protected def missingValueMessage: String = s"Expecting $key" +
+    " runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]"
+
+  override protected def usedInCallCaching: Boolean = true
 }
diff --git a/backend/src/main/scala/cromwell/backend/validation/CpuValidation.scala b/backend/src/main/scala/cromwell/backend/validation/CpuValidation.scala
index 43303b6cb..5a5d09730 100644
--- a/backend/src/main/scala/cromwell/backend/validation/CpuValidation.scala
+++ b/backend/src/main/scala/cromwell/backend/validation/CpuValidation.scala
@@ -1,8 +1,9 @@
 package cromwell.backend.validation
 
 import cats.syntax.validated._
+import lenthall.validation.ErrorOr.ErrorOr
 import wdl4s.types.WdlIntegerType
-import wdl4s.values.WdlInteger
+import wdl4s.values.{WdlInteger, WdlValue}
 
 /**
   * Validates the "cpu" runtime attribute an Integer greater than 0, returning the value as an `Int`.
@@ -15,36 +16,22 @@ import wdl4s.values.WdlInteger
   * found.
   */
 object CpuValidation extends {
-  val key = RuntimeAttributesKeys.CpuKey
-
-  lazy val instance = new CpuValidation
-
-  lazy val default = instance.withDefault(WdlInteger(1))
-
-  lazy val optional = default.optional
-
-  private[validation] val missingMessage = s"Expecting $key runtime attribute to be an Integer"
-  private[validation] val wrongAmountMsg = s"Expecting $key runtime attribute value greater than 0"
+  lazy val instance: RuntimeAttributesValidation[Int] = new CpuValidation
+  lazy val default: RuntimeAttributesValidation[Int] = instance.withDefault(WdlInteger(1))
+  lazy val optional: OptionalRuntimeAttributesValidation[Int] = default.optional
 }
 
-class CpuValidation extends RuntimeAttributesValidation[Int] {
-
-  import CpuValidation._
-
-  override def key = RuntimeAttributesKeys.CpuKey
-
-  override def coercion = Seq(WdlIntegerType)
-
-  override protected def validateValue = {
+class CpuValidation extends IntRuntimeAttributesValidation(RuntimeAttributesKeys.CpuKey) {
+  override protected def validateValue: PartialFunction[WdlValue, ErrorOr[Int]] = {
     case wdlValue if WdlIntegerType.coerceRawValue(wdlValue).isSuccess =>
       WdlIntegerType.coerceRawValue(wdlValue).get match {
-        case WdlInteger(value) => if (value.toInt <= 0) wrongAmountMsg.invalidNel else value.toInt.validNel
+        case WdlInteger(value) =>
+          if (value.toInt <= 0)
+            s"Expecting $key runtime attribute value greater than 0".invalidNel
+          else
+            value.toInt.validNel
       }
   }
 
-  override def validateExpression = {
-    case wdlValue if WdlIntegerType.coerceRawValue(wdlValue).isSuccess => true
-  }
-
-  override protected def failureMessage = missingMessage
+  override protected def missingValueMessage: String = s"Expecting $key runtime attribute to be an Integer"
 }
diff --git a/backend/src/main/scala/cromwell/backend/validation/DockerValidation.scala b/backend/src/main/scala/cromwell/backend/validation/DockerValidation.scala
index 12e090fcf..f793d1397 100644
--- a/backend/src/main/scala/cromwell/backend/validation/DockerValidation.scala
+++ b/backend/src/main/scala/cromwell/backend/validation/DockerValidation.scala
@@ -1,8 +1,8 @@
 package cromwell.backend.validation
 
 import cats.syntax.validated._
-import wdl4s.types.WdlStringType
-import wdl4s.values.WdlString
+import lenthall.validation.ErrorOr.ErrorOr
+import wdl4s.values.{WdlString, WdlValue}
 
 /**
   * Validates the "docker" runtime attribute as a String, returning it as `String`.
@@ -13,26 +13,19 @@ import wdl4s.values.WdlString
   * if present, or `None` if not found.
   */
 object DockerValidation {
-  val key = RuntimeAttributesKeys.DockerKey
-
-  lazy val instance = new DockerValidation
-
-  lazy val optional = instance.optional
-
-  private[validation] val missingMessage = s"Expecting $key runtime attribute to be a String"
+  lazy val instance: RuntimeAttributesValidation[String] = new DockerValidation
+  lazy val optional: OptionalRuntimeAttributesValidation[String] = instance.optional
 }
 
-class DockerValidation extends RuntimeAttributesValidation[String] {
+class DockerValidation extends StringRuntimeAttributesValidation(RuntimeAttributesKeys.DockerKey) {
+  override protected def usedInCallCaching: Boolean = true
 
-  import DockerValidation._
+  override protected def missingValueMessage: String = "Can't find an attribute value for key docker"
 
-  override def key = RuntimeAttributesKeys.DockerKey
+  override protected def invalidValueMessage(value: WdlValue): String = super.missingValueMessage
 
-  override def coercion = Seq(WdlStringType)
-
-  override protected def validateValue = {
+  // NOTE: Docker's current test specs don't like WdlInteger, etc. auto converted to WdlString.
+  override protected def validateValue: PartialFunction[WdlValue, ErrorOr[String]] = {
     case WdlString(value) => value.validNel
   }
-
-  override protected def failureMessage = missingMessage
 }
diff --git a/backend/src/main/scala/cromwell/backend/validation/FailOnStderrValidation.scala b/backend/src/main/scala/cromwell/backend/validation/FailOnStderrValidation.scala
index 6c75a40cf..6a7ea6ee2 100644
--- a/backend/src/main/scala/cromwell/backend/validation/FailOnStderrValidation.scala
+++ b/backend/src/main/scala/cromwell/backend/validation/FailOnStderrValidation.scala
@@ -1,10 +1,6 @@
 package cromwell.backend.validation
 
-import cats.syntax.validated._
-import wdl4s.types.{WdlBooleanType, WdlStringType}
-import wdl4s.values.{WdlBoolean, WdlString}
-
-import scala.util.Try
+import wdl4s.values.WdlBoolean
 
 /**
   * Validates the "failOnStderr" runtime attribute as a Boolean or a String 'true' or 'false', returning the value as a
@@ -15,35 +11,14 @@ import scala.util.Try
   * The default returns `false` when no attribute is specified.
   */
 object FailOnStderrValidation {
-  val key = RuntimeAttributesKeys.FailOnStderrKey
-
-  lazy val instance = new FailOnStderrValidation
-
-  lazy val default = instance.withDefault(WdlBoolean(false))
-
-  lazy val optional = default.optional
-
-  private[validation] val missingMessage =
-    s"Expecting $key runtime attribute to be a Boolean or a String with values of 'true' or 'false'"
+  lazy val instance: RuntimeAttributesValidation[Boolean] = new FailOnStderrValidation
+  lazy val default: RuntimeAttributesValidation[Boolean] = instance.withDefault(WdlBoolean(false))
+  lazy val optional: OptionalRuntimeAttributesValidation[Boolean] = default.optional
 }
 
-class FailOnStderrValidation extends RuntimeAttributesValidation[Boolean] {
+class FailOnStderrValidation extends BooleanRuntimeAttributesValidation(RuntimeAttributesKeys.FailOnStderrKey) {
+  override protected def usedInCallCaching: Boolean = true
 
-  import FailOnStderrValidation._
-
-  override def key = RuntimeAttributesKeys.FailOnStderrKey
-
-  override def coercion = Seq(WdlBooleanType, WdlStringType)
-
-  override protected def validateValue = {
-    case WdlBoolean(value) => value.validNel
-    case WdlString(value) if Try(value.toBoolean).isSuccess => value.toBoolean.validNel
-  }
-
-  override def validateExpression = {
-    case _: WdlBoolean => true
-    case WdlString(value) if Try(value.toBoolean).isSuccess => true
-  }
-
-  override protected def failureMessage = missingMessage
+  override protected def missingValueMessage: String =
+    s"Expecting $key runtime attribute to be a Boolean or a String with values of 'true' or 'false'"
 }
diff --git a/backend/src/main/scala/cromwell/backend/validation/MemoryValidation.scala b/backend/src/main/scala/cromwell/backend/validation/MemoryValidation.scala
index 1e1c807cf..d493e357e 100644
--- a/backend/src/main/scala/cromwell/backend/validation/MemoryValidation.scala
+++ b/backend/src/main/scala/cromwell/backend/validation/MemoryValidation.scala
@@ -5,7 +5,7 @@ import cromwell.backend.MemorySize
 import lenthall.validation.ErrorOr._
 import wdl4s.parser.MemoryUnit
 import wdl4s.types.{WdlIntegerType, WdlStringType}
-import wdl4s.values.{WdlInteger, WdlString}
+import wdl4s.values.{WdlInteger, WdlString, WdlValue}
 
 /**
   * Validates the "memory" runtime attribute as an Integer or String with format '8 GB', returning the value as a
@@ -19,18 +19,17 @@ import wdl4s.values.{WdlInteger, WdlString}
   * `withDefaultMemory` can be used to create a memory validation that defaults to a particular memory size.
   */
 object MemoryValidation {
-  val key = RuntimeAttributesKeys.MemoryKey
+  lazy val instance: RuntimeAttributesValidation[MemorySize] = new MemoryValidation
+  lazy val optional: OptionalRuntimeAttributesValidation[MemorySize] = instance.optional
 
-  lazy val instance = new MemoryValidation
+  def withDefaultMemory(memorySize: MemorySize): RuntimeAttributesValidation[MemorySize] =
+    instance.withDefault(WdlInteger(memorySize.bytes.toInt))
 
-  lazy val optional = instance.optional
-
-  def withDefaultMemory(memorySize: MemorySize) = instance.withDefault(WdlInteger(memorySize.bytes.toInt))
-
-  private[validation] val missingMessage =
-    s"Expecting $key runtime attribute to be an Integer or String with format '8 GB'"
-  private[validation] val wrongAmountFormat = s"Expecting $key runtime attribute value greater than 0 but got %s"
-  private[validation] val missingFormat = s"$missingMessage. Exception: %s"
+  private[validation] val wrongAmountFormat =
+    s"Expecting ${RuntimeAttributesKeys.MemoryKey} runtime attribute value greater than 0 but got %s"
+  private[validation] val wrongTypeFormat =
+    s"Expecting ${RuntimeAttributesKeys.MemoryKey} runtime attribute to be an Integer or String with format '8 GB'." +
+      s" Exception: %s"
 
   private[validation] def validateMemoryString(wdlString: WdlString): ErrorOr[MemorySize] =
     validateMemoryString(wdlString.value)
@@ -42,7 +41,7 @@ object MemoryValidation {
       case scala.util.Success(memorySize: MemorySize) =>
         wrongAmountFormat.format(memorySize.amount).invalidNel
       case scala.util.Failure(throwable) =>
-        missingFormat.format(throwable.getMessage).invalidNel
+        wrongTypeFormat.format(throwable.getMessage).invalidNel
     }
   }
 
@@ -65,10 +64,10 @@ class MemoryValidation extends RuntimeAttributesValidation[MemorySize] {
 
   override def coercion = Seq(WdlIntegerType, WdlStringType)
 
-  override protected def validateValue = {
+  override protected def validateValue: PartialFunction[WdlValue, ErrorOr[MemorySize]] = {
     case WdlInteger(value) => MemoryValidation.validateMemoryInteger(value)
     case WdlString(value) => MemoryValidation.validateMemoryString(value)
   }
 
-  override def failureMessage = missingMessage
+  override def missingValueMessage: String = wrongTypeFormat.format("Not supported WDL type value")
 }
diff --git a/backend/src/main/scala/cromwell/backend/validation/PrimitiveRuntimeAttributesValidation.scala b/backend/src/main/scala/cromwell/backend/validation/PrimitiveRuntimeAttributesValidation.scala
index 16fed1f3d..aa07afcdc 100644
--- a/backend/src/main/scala/cromwell/backend/validation/PrimitiveRuntimeAttributesValidation.scala
+++ b/backend/src/main/scala/cromwell/backend/validation/PrimitiveRuntimeAttributesValidation.scala
@@ -1,51 +1,74 @@
-package cromwell.backend.impl.sfs.config
+package cromwell.backend.validation
 
 import cats.syntax.validated._
-import cromwell.backend.validation.RuntimeAttributesValidation
+import lenthall.validation.ErrorOr.ErrorOr
 import wdl4s.types._
-import wdl4s.values.{WdlBoolean, WdlFloat, WdlInteger, WdlString}
+import wdl4s.values.{WdlBoolean, WdlFloat, WdlInteger, WdlPrimitive, WdlString, WdlValue}
 
 /**
   * Validates one of the wdl primitive types: Boolean, Float, Integer, or String. WdlFile is not supported.
   *
   * @tparam A The type of validated runtime attribute.
+  * @tparam B The type of coerced WdlValue.
   */
-sealed trait PrimitiveRuntimeAttributesValidation[A] extends RuntimeAttributesValidation[A] {
+sealed trait PrimitiveRuntimeAttributesValidation[A, B <: WdlPrimitive] extends RuntimeAttributesValidation[A] {
   val wdlType: WdlPrimitiveType
 
   override def coercion = Seq(wdlType)
+
+  override protected def validateExpression: PartialFunction[WdlValue, Boolean] = {
+    case wdlValue if wdlType.coerceRawValue(wdlValue).isSuccess => true
+  }
+
+  override protected def missingValueMessage: String =
+    s"Expecting $key runtime attribute to be $typeString"
+
+  protected def typeString = s"a ${wdlType.toWdlString}"
+
+  override protected def validateValue: PartialFunction[WdlValue, ErrorOr[A]] = {
+    case value if wdlType.coerceRawValue(value).isSuccess =>
+      validateCoercedValue(wdlType.coerceRawValue(value).get.asInstanceOf[B])
+    case value if wdlType.coerceRawValue(value.valueString).isSuccess =>
+      /*
+      NOTE: This case statement handles WdlString("true") coercing to WdlBoolean(true).
+      For some reason "true" as String is coercable... but not the WdlString.
+       */
+      validateCoercedValue(wdlType.coerceRawValue(value.valueString).get.asInstanceOf[B])
+  }
+
+  protected def validateCoercedValue(wdlValue: B): ErrorOr[A]
 }
 
 class BooleanRuntimeAttributesValidation(override val key: String) extends
-  PrimitiveRuntimeAttributesValidation[Boolean] {
+  PrimitiveRuntimeAttributesValidation[Boolean, WdlBoolean] {
 
   override val wdlType = WdlBooleanType
 
-  override protected def validateValue = {
-    case WdlBoolean(value) => value.validNel
-  }
+  override protected def validateCoercedValue(wdlValue: WdlBoolean): ErrorOr[Boolean] = wdlValue.value.validNel
 }
 
-class FloatRuntimeAttributesValidation(override val key: String) extends PrimitiveRuntimeAttributesValidation[Double] {
+class FloatRuntimeAttributesValidation(override val key: String) extends
+  PrimitiveRuntimeAttributesValidation[Double, WdlFloat] {
+
   override val wdlType = WdlFloatType
 
-  override protected def validateValue = {
-    case WdlFloat(value) => value.validNel
-  }
+  override protected def validateCoercedValue(wdlValue: WdlFloat): ErrorOr[Double] = wdlValue.value.validNel
 }
 
-class IntRuntimeAttributesValidation(override val key: String) extends PrimitiveRuntimeAttributesValidation[Int] {
+class IntRuntimeAttributesValidation(override val key: String) extends
+  PrimitiveRuntimeAttributesValidation[Int, WdlInteger] {
+
   override val wdlType = WdlIntegerType
 
-  override protected def validateValue = {
-    case WdlInteger(value) => value.toInt.validNel
-  }
+  override protected def validateCoercedValue(wdlValue: WdlInteger): ErrorOr[Int] = wdlValue.value.toInt.validNel
+
+  override protected def typeString: String = "an Integer"
 }
 
-class StringRuntimeAttributesValidation(override val key: String) extends PrimitiveRuntimeAttributesValidation[String] {
+class StringRuntimeAttributesValidation(override val key: String) extends
+  PrimitiveRuntimeAttributesValidation[String, WdlString] {
+
   override val wdlType = WdlStringType
 
-  override protected def validateValue = {
-    case WdlString(value) => value.validNel
-  }
+  override protected def validateCoercedValue(wdlValue: WdlString): ErrorOr[String] = wdlValue.value.validNel
 }
diff --git a/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala b/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala
index 1adbc254f..322fe0e43 100644
--- a/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala
+++ b/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala
@@ -1,54 +1,51 @@
 package cromwell.backend.validation
 
+import cats.data.{NonEmptyList, Validated}
 import cats.syntax.validated._
-import wdl4s.expression.PureStandardLibraryFunctions
 import cromwell.backend.{MemorySize, RuntimeAttributeDefinition}
 import lenthall.validation.ErrorOr._
 import org.slf4j.Logger
-import wdl4s.WdlExpression
-import wdl4s.WdlExpression._
+import wdl4s.expression.PureStandardLibraryFunctions
 import wdl4s.types.{WdlBooleanType, WdlIntegerType, WdlType}
 import wdl4s.values._
+import wdl4s.{NoLookup, WdlExpression}
 
 import scala.util.{Failure, Success}
 
 object RuntimeAttributesValidation {
 
-  def warnUnrecognized(actual: Set[String], expected: Set[String], logger: Logger) = {
+  def warnUnrecognized(actual: Set[String], expected: Set[String], logger: Logger): Unit = {
     val unrecognized = actual.diff(expected).mkString(", ")
     if (unrecognized.nonEmpty) logger.warn(s"Unrecognized runtime attribute keys: $unrecognized")
   }
 
   def validateDocker(docker: Option[WdlValue], onMissingKey: => ErrorOr[Option[String]]): ErrorOr[Option[String]] = {
-    validateWithValidation(docker, DockerValidation.optional, onMissingKey, DockerValidation.missingMessage)
+    validateWithValidation(docker, DockerValidation.optional, onMissingKey)
   }
 
   def validateFailOnStderr(value: Option[WdlValue], onMissingKey: => ErrorOr[Boolean]): ErrorOr[Boolean] = {
-    validateWithValidation(value, FailOnStderrValidation.default, onMissingKey, FailOnStderrValidation.missingMessage)
+    validateWithValidation(value, FailOnStderrValidation.default, onMissingKey)
   }
 
   def validateContinueOnReturnCode(value: Option[WdlValue],
                                    onMissingKey: => ErrorOr[ContinueOnReturnCode]): ErrorOr[ContinueOnReturnCode] = {
-    validateWithValidation(value, ContinueOnReturnCodeValidation.default, onMissingKey,
-      ContinueOnReturnCodeValidation.missingMessage)
+    validateWithValidation(value, ContinueOnReturnCodeValidation.default, onMissingKey)
   }
 
   def validateMemory(value: Option[WdlValue], onMissingKey: => ErrorOr[MemorySize]): ErrorOr[MemorySize] = {
-    validateWithValidation(value, MemoryValidation.instance, onMissingKey,
-      MemoryValidation.missingFormat.format("Not supported WDL type value"))
+    validateWithValidation(value, MemoryValidation.instance, onMissingKey)
   }
 
   def validateCpu(cpu: Option[WdlValue], onMissingKey: => ErrorOr[Int]): ErrorOr[Int] = {
-    validateWithValidation(cpu, CpuValidation.default, onMissingKey, CpuValidation.missingMessage)
+    validateWithValidation(cpu, CpuValidation.default, onMissingKey)
   }
 
   private def validateWithValidation[T](valueOption: Option[WdlValue],
                                         validation: RuntimeAttributesValidation[T],
-                                        onMissingValue: => ErrorOr[T],
-                                        missingValidationMessage: String): ErrorOr[T] = {
+                                        onMissingValue: => ErrorOr[T]): ErrorOr[T] = {
     valueOption match {
       case Some(value) =>
-        validation.validateValue.applyOrElse(value, (_: Any) => missingValidationMessage.invalidNel)
+        validation.validateValue.applyOrElse(value, (_: Any) => validation.invalidValueFailure(value))
       case None => onMissingValue
     }
   }
@@ -78,46 +75,62 @@ object RuntimeAttributesValidation {
   def withDefault[ValidatedType](validation: RuntimeAttributesValidation[ValidatedType],
                                  default: WdlValue): RuntimeAttributesValidation[ValidatedType] = {
     new RuntimeAttributesValidation[ValidatedType] {
-      override def key = validation.key
+      override def key: String = validation.key
+
+      override def coercion: Traversable[WdlType] = validation.coercion
 
-      override def coercion = validation.coercion
+      override protected def validateValue: PartialFunction[WdlValue, ErrorOr[ValidatedType]] =
+        validation.validateValuePackagePrivate
 
-      override protected def validateValue = validation.validateValuePackagePrivate
+      override protected def validateExpression: PartialFunction[WdlValue, Boolean] =
+        validation.validateExpressionPackagePrivate
 
-      override protected def validateExpression = validation.validateExpressionPackagePrivate
+      override protected def invalidValueMessage(value: WdlValue): String =
+        validation.invalidValueMessagePackagePrivate(value)
 
-      override def staticDefaultOption = Option(default)
+      override protected def missingValueMessage: String = validation.missingValueMessage
 
-      override protected def failureMessage = validation.failureMessagePackagePrivate
+      override protected def usedInCallCaching: Boolean = validation.usedInCallCachingPackagePrivate
+
+      override protected def staticDefaultOption = Option(default)
     }
   }
 
   def optional[ValidatedType](validation: RuntimeAttributesValidation[ValidatedType]):
   OptionalRuntimeAttributesValidation[ValidatedType] = {
     new OptionalRuntimeAttributesValidation[ValidatedType] {
-      override def key = validation.key
+      override def key: String = validation.key
+
+      override def coercion: Traversable[WdlType] = validation.coercion
 
-      override def coercion = validation.coercion
+      override protected def validateOption: PartialFunction[WdlValue, ErrorOr[ValidatedType]] =
+        validation.validateValuePackagePrivate
 
-      override protected def validateOption = validation.validateValuePackagePrivate
+      override protected def validateExpression: PartialFunction[WdlValue, Boolean] =
+        validation.validateExpressionPackagePrivate
 
-      override protected def validateExpression = validation.validateExpressionPackagePrivate
+      override protected def invalidValueMessage(value: WdlValue): String =
+        validation.invalidValueMessagePackagePrivate(value)
 
-      override protected def failureMessage = validation.failureMessagePackagePrivate
+      override protected def missingValueMessage: String = validation.missingValueMessage
+
+      override protected def usedInCallCaching: Boolean = validation.usedInCallCachingPackagePrivate
     }
   }
 
   /**
-    * Returns the value from the attributes, unpacking options.
+    * Returns the value from the attributes, unpacking options, and converting them to string values suitable for
+    * storage in metadata.
     *
     * @param validatedRuntimeAttributes The values to search.
     * @return The keys and extracted values.
     */
-  def extract(validatedRuntimeAttributes: ValidatedRuntimeAttributes): Map[String, Any] = {
+  def toMetadataStrings(validatedRuntimeAttributes: ValidatedRuntimeAttributes): Map[String, String] = {
     val attributeOptions: Map[String, Option[Any]] = validatedRuntimeAttributes.attributes.mapValues(unpackOption)
 
-    val attributes = attributeOptions collect {
-      case (name, Some(value)) => (name, value)
+    val attributes: Map[String, String] = attributeOptions collect {
+      case (name, Some(values: Traversable[_])) => (name, values.mkString(","))
+      case (name, Some(value)) => (name, value.toString)
     }
 
     attributes
@@ -195,23 +208,6 @@ object RuntimeAttributesValidation {
       case _ => Option(value.asInstanceOf[A])
     }
   }
-
-  /**
-    * Converts a RuntimeAttributesValidation to a RuntimeAttributeDefinition.
-    *
-    * @param validation RuntimeAttributesValidation
-    * @return RuntimeAttributeDefinition
-    */
-  def toRuntimeAttributeDefinition(validation: RuntimeAttributesValidation[_]): RuntimeAttributeDefinition = {
-    val name = validation.key
-    val default = validation.staticDefaultOption
-    import cromwell.backend.validation.RuntimeAttributesKeys._
-    val usedInCallCaching = name match {
-      case DockerKey | ContinueOnReturnCodeKey | FailOnStderrKey => true
-      case _ => false
-    }
-    RuntimeAttributeDefinition(name, default, usedInCallCaching)
-  }
 }
 
 /**
@@ -247,7 +243,7 @@ trait RuntimeAttributesValidation[ValidatedType] {
     *
     * @return the value for when there is no wdl value.
     */
-  protected def validateNone: ErrorOr[ValidatedType] = failureWithMessage
+  protected def validateNone: ErrorOr[ValidatedType] = missingValueFailure
 
   /**
     * Returns true if the value can be validated.
@@ -268,31 +264,50 @@ trait RuntimeAttributesValidation[ValidatedType] {
     *
     * @return the optional default value when no other is specified.
     */
-  def staticDefaultOption: Option[WdlValue] = None
+  protected def staticDefaultOption: Option[WdlValue] = None
 
   /**
     * Returns message to return when a value is invalid.
     *
+    * By default returns the missingValueMessage.
+    *
     * @return Message to return when a value is invalid.
     */
-  protected def failureMessage: String = s"Expecting $key runtime attribute to be a type in $coercion"
+  protected def invalidValueMessage(value: WdlValue): String = missingValueMessage
+
+  /**
+    * Utility method to wrap the invalidValueMessage in an ErrorOr.
+    *
+    * @return Wrapped invalidValueMessage.
+    */
+  protected final def invalidValueFailure(value: WdlValue): ErrorOr[ValidatedType] =
+    invalidValueMessage(value).invalidNel
 
   /**
-    * Utility method to wrap the failureMessage in an ErrorOr.
+    * Returns message to return when a value is missing.
     *
-    * @return Wrapped failureMessage.
+    * @return Message to return when a value is missing.
     */
-  protected final lazy val failureWithMessage: ErrorOr[ValidatedType] = failureMessage.invalidNel
+  protected def missingValueMessage: String = s"Expecting $key runtime attribute to be a type in $coercion"
+
+  /**
+    * Utility method to wrap the missingValueMessage in an ErrorOr.
+    *
+    * @return Wrapped missingValueMessage.
+    */
+  protected final lazy val missingValueFailure: ErrorOr[ValidatedType] = missingValueMessage.invalidNel
 
   /**
     * Runs this validation on the value matching key.
     *
+    * NOTE: The values passed to this method should already be evaluated instances of WdlValue, and not WdlExpression.
+    *
     * @param values The full set of values.
     * @return The error or valid value for this key.
     */
   def validate(values: Map[String, WdlValue]): ErrorOr[ValidatedType] = {
     values.get(key) match {
-      case Some(value) => validateValue.applyOrElse(value, (_: Any) => failureWithMessage)
+      case Some(value) => validateValue.applyOrElse(value, (_: Any) => invalidValueFailure(value))
       case None => validateNone
     }
   }
@@ -308,6 +323,9 @@ trait RuntimeAttributesValidation[ValidatedType] {
     *
     * With our `key` as the key in the map, one can return this function as the value in the map.
     *
+    * NOTE: If there is an attempt lookup a value within a WdlExpression, or a WdlExpression fails to evaluate for any
+    * reason, this method will simply return true.
+    *
     * @param wdlExpressionMaybe The optional expression.
     * @return True if the expression may be evaluated.
     */
@@ -315,36 +333,28 @@ trait RuntimeAttributesValidation[ValidatedType] {
     wdlExpressionMaybe match {
       case None => staticDefaultOption.isDefined || validateNone.isValid
       case Some(wdlExpression: WdlExpression) =>
-        /*
-        TODO: BUG:
-
-        Using `wdl4s.NoLookup` with the following options causes the following exception:
-
-        command:
-          sbt -J-Dbackend.default=SGE 'run run test_wdl/test.wdl - test_wdl/test.default.options'
-
-        options:
-          { "default_runtime_attributes": { "sge_queue": "fromoptions" } }
-
-        exception:
-          java.lang.RuntimeException: Expression evaluation failed due to java.lang.UnsupportedOperationException:
-          No identifiers should be looked up: fromoptions: WdlExpression(<string:1:1 identifier "ZnJvbW9wdGlvbnM=">)
-
-        Not sure why yet the string "fromoptions" is being converted to an expression and not a WdlString.
-
-        For now, if something tries to "lookup" a value, convert it to a WdlString.
-         */
-        val wdlStringLookup: ScopedLookupFunction = (value: String) => WdlString(value)
-        wdlExpression.evaluate(wdlStringLookup, PureStandardLibraryFunctions) match {
+        wdlExpression.evaluate(NoLookup, PureStandardLibraryFunctions) match {
           case Success(wdlValue) => validateExpression.applyOrElse(wdlValue, (_: Any) => false)
-          case Failure(throwable) =>
-            throw new RuntimeException(s"Expression evaluation failed due to $throwable: $wdlExpression", throwable)
+          case Failure(_) => true // If we can't evaluate it, we'll let it pass for now...
         }
       case Some(wdlValue) => validateExpression.applyOrElse(wdlValue, (_: Any) => false)
     }
   }
 
   /**
+    * Used to convert this instance to a `RuntimeAttributeDefinition`.
+    *
+    * @see [[RuntimeAttributeDefinition.usedInCallCaching]]
+    * @return Value for [[RuntimeAttributeDefinition.usedInCallCaching]].
+    */
+  protected def usedInCallCaching: Boolean = false
+
+  /**
+    * Returns this as an instance of a runtime attribute definition.
+    */
+  final lazy val runtimeAttributeDefinition = RuntimeAttributeDefinition(key, staticDefaultOption, usedInCallCaching)
+
+  /**
     * Returns an optional version of this validation.
     */
   final lazy val optional: OptionalRuntimeAttributesValidation[ValidatedType] =
@@ -356,7 +366,8 @@ trait RuntimeAttributesValidation[ValidatedType] {
     * @param wdlValue The default wdl value.
     * @return The new version of this validation.
     */
-  final def withDefault(wdlValue: WdlValue) = RuntimeAttributesValidation.withDefault(this, wdlValue)
+  final def withDefault(wdlValue: WdlValue): RuntimeAttributesValidation[ValidatedType] =
+    RuntimeAttributesValidation.withDefault(this, wdlValue)
 
   /*
   Methods below provide aliases to expose protected methods to the package.
@@ -365,11 +376,15 @@ trait RuntimeAttributesValidation[ValidatedType] {
   access the protected values, except the `validation` package that uses these back doors.
    */
 
-  private[validation] lazy val validateValuePackagePrivate = validateValue
+  private[validation] final lazy val validateValuePackagePrivate = validateValue
+
+  private[validation] final lazy val validateExpressionPackagePrivate = validateExpression
 
-  private[validation] lazy val validateExpressionPackagePrivate = validateExpression
+  private[validation] final def invalidValueMessagePackagePrivate(value: WdlValue) = invalidValueMessage(value)
 
-  private[validation] lazy val failureMessagePackagePrivate = failureMessage
+  private[validation] final lazy val missingValueMessagePackagePrivate = missingValueMessage
+
+  private[validation] final lazy val usedInCallCachingPackagePrivate = usedInCallCaching
 }
 
 /**
@@ -389,10 +404,12 @@ trait OptionalRuntimeAttributesValidation[ValidatedType] extends RuntimeAttribut
   protected def validateOption: PartialFunction[WdlValue, ErrorOr[ValidatedType]]
 
   override final protected lazy val validateValue = new PartialFunction[WdlValue, ErrorOr[Option[ValidatedType]]] {
-    override def isDefinedAt(wdlValue: WdlValue) = validateOption.isDefinedAt(wdlValue)
+    override def isDefinedAt(wdlValue: WdlValue): Boolean = validateOption.isDefinedAt(wdlValue)
 
-    override def apply(wdlValue: WdlValue) = validateOption.apply(wdlValue).map(Option.apply)
+    override def apply(wdlValue: WdlValue): Validated[NonEmptyList[String], Option[ValidatedType]] = {
+      validateOption.apply(wdlValue).map(Option.apply)
+    }
   }
 
-  override final protected lazy val validateNone = None.validNel[String]
+  override final protected lazy val validateNone: ErrorOr[None.type] = None.validNel[String]
 }
diff --git a/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala b/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala
index ad3c2cb57..ed3981817 100644
--- a/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala
+++ b/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala
@@ -6,6 +6,7 @@ import cromwell.backend.RuntimeAttributeDefinition
 import lenthall.exception.MessageAggregation
 import lenthall.validation.ErrorOr._
 import org.slf4j.Logger
+import wdl4s.types.WdlType
 import wdl4s.values.WdlValue
 
 final case class ValidatedRuntimeAttributes(attributes: Map[String, Any])
@@ -29,20 +30,23 @@ trait ValidatedRuntimeAttributesBuilder {
   /**
     * Returns a mapping of the validations: RuntimeAttributesValidation each converted to a RuntimeAttributeDefinition.
     */
-  final lazy val definitions: Seq[RuntimeAttributeDefinition] = {
-    validations map RuntimeAttributesValidation.toRuntimeAttributeDefinition
+  final lazy val definitions: Seq[RuntimeAttributeDefinition] = validations.map(_.runtimeAttributeDefinition)
+
+  /**
+    * Returns validators suitable for BackendWorkflowInitializationActor.runtimeAttributeValidators.
+    */
+  final lazy val validatorMap: Map[String, (Option[WdlValue]) => Boolean] = {
+    validations.map(validation =>
+      validation.key -> validation.validateOptionalExpression _
+    ).toMap
   }
 
   /**
-    * Returns the additional validations that should be used during value parsing.
-    *
-    * For example, sometimes docker might not be supported, BUT we want to still validate the value if specified.
-    *
-    * In that case, return the validation here.
-    *
-    * @return the additional validations that should be used during value parsing.
+    * Returns a map of coercions suitable for RuntimeAttributesDefault.workflowOptionsDefault.
     */
-  protected def unsupportedExtraValidations: Seq[OptionalRuntimeAttributesValidation[_]] = Seq.empty
+  final lazy val coercionMap: Map[String, Traversable[WdlType]] = {
+    validations.map(validation => validation.key -> validation.coercion).toMap
+  }
 
   def unsupportedKeys(keys: Seq[String]): Seq[String] = keys.diff(validationKeys)
 
@@ -63,9 +67,8 @@ trait ValidatedRuntimeAttributesBuilder {
   }
 
   private def validate(values: Map[String, WdlValue]): ErrorOr[ValidatedRuntimeAttributes] = {
-    val validationsForValues = validations ++ unsupportedExtraValidations
     val listOfKeysToErrorOrAnys: List[(String, ErrorOr[Any])] =
-      validationsForValues.map(validation => validation.key -> validation.validate(values)).toList
+      validations.map(validation => validation.key -> validation.validate(values)).toList
 
     val listOfErrorOrKeysToAnys: List[ErrorOr[(String, Any)]] = listOfKeysToErrorOrAnys map {
       case (key, errorOrAny) => errorOrAny map { any => (key, any) }
diff --git a/backend/src/test/scala/cromwell/backend/BackendWorkflowInitializationActorSpec.scala b/backend/src/test/scala/cromwell/backend/BackendWorkflowInitializationActorSpec.scala
new file mode 100644
index 000000000..82e62d641
--- /dev/null
+++ b/backend/src/test/scala/cromwell/backend/BackendWorkflowInitializationActorSpec.scala
@@ -0,0 +1,204 @@
+package cromwell.backend
+
+import akka.actor.ActorRef
+import akka.testkit.TestActorRef
+import cromwell.backend.validation.{ContinueOnReturnCodeFlag, ContinueOnReturnCodeSet, ContinueOnReturnCodeValidation, RuntimeAttributesKeys}
+import cromwell.core.{TestKitSuite, WorkflowOptions}
+import org.scalatest.prop.TableDrivenPropertyChecks
+import org.scalatest.{FlatSpecLike, Matchers}
+import wdl4s.types._
+import wdl4s.values.{WdlArray, WdlBoolean, WdlFloat, WdlInteger, WdlString, WdlValue}
+import wdl4s.{TaskCall, WdlExpression}
+
+import scala.concurrent.Future
+import scala.util.Try
+
+class BackendWorkflowInitializationActorSpec extends TestKitSuite("BackendWorkflowInitializationActorSpec")
+  with FlatSpecLike with Matchers with TableDrivenPropertyChecks {
+
+  behavior of "BackendWorkflowInitializationActorSpec"
+
+  val testPredicateBackendWorkflowInitializationActorRef:
+    TestActorRef[TestPredicateBackendWorkflowInitializationActor] =
+    TestActorRef[TestPredicateBackendWorkflowInitializationActor]
+
+  val testPredicateBackendWorkflowInitializationActor:
+    TestPredicateBackendWorkflowInitializationActor =
+    testPredicateBackendWorkflowInitializationActorRef.underlyingActor
+
+  val testContinueOnReturnCode: (Option[WdlValue]) => Boolean = {
+    testPredicateBackendWorkflowInitializationActor.continueOnReturnCodePredicate(valueRequired = false)
+  }
+
+  it should "continueOnReturnCodePredicate" in {
+    testContinueOnReturnCode(None) should be(true)
+    ContinueOnReturnCodeValidation.default.validateOptionalExpression(None) should be(true)
+
+    val booleanRows = Table(
+      "value",
+      true,
+      false
+    )
+
+    val integerRows = Table(
+      "value",
+      -1,
+      0,
+      1,
+      1024
+    )
+
+    val expressionRows = Table(
+      "expression",
+      "read_int(\"bad file\")"
+    )
+
+    val invalidWdlValueRows = Table(
+      "wdlValue",
+      WdlString(""),
+      WdlString("z"),
+      WdlFloat(0.0D),
+      WdlArray(WdlArrayType(WdlBooleanType), Seq(WdlBoolean(true))),
+      WdlArray(WdlArrayType(WdlFloatType), Seq(WdlFloat(0.0D)))
+    )
+
+    forAll(booleanRows) { value =>
+      val wdlValue = WdlBoolean(value)
+      val result = true
+      testContinueOnReturnCode(Option(wdlValue)) should be(result)
+      ContinueOnReturnCodeValidation.default.validateOptionalExpression(Option(wdlValue)) should be(result)
+      val valid =
+        ContinueOnReturnCodeValidation.default.validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> wdlValue))
+      valid.isValid should be(result)
+      valid.toEither.right.get should be(ContinueOnReturnCodeFlag(value))
+    }
+
+    forAll(booleanRows) { value =>
+      val wdlValue = WdlString(value.toString)
+      val result = true
+      testContinueOnReturnCode(Option(wdlValue)) should be(result)
+      ContinueOnReturnCodeValidation.default.validateOptionalExpression(Option(wdlValue)) should be(result)
+      val valid =
+        ContinueOnReturnCodeValidation.default.validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> wdlValue))
+      valid.isValid should be(result)
+      valid.toEither.right.get should be(ContinueOnReturnCodeFlag(value))
+    }
+
+    forAll(booleanRows) { value =>
+      val wdlValue = WdlExpression.fromString(value.toString)
+      val result = true
+      testContinueOnReturnCode(Option(wdlValue)) should be(result)
+      ContinueOnReturnCodeValidation.default.validateOptionalExpression(Option(wdlValue)) should be(result)
+      // NOTE: expressions are never valid to validate
+    }
+
+    forAll(integerRows) { value =>
+      val wdlValue = WdlInteger(value)
+      val result = true
+      testContinueOnReturnCode(Option(wdlValue)) should be(result)
+      ContinueOnReturnCodeValidation.default.validateOptionalExpression(Option(wdlValue)) should be(result)
+      val valid =
+        ContinueOnReturnCodeValidation.default.validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> wdlValue))
+      valid.isValid should be(result)
+      valid.toEither.right.get should be(ContinueOnReturnCodeSet(Set(value)))
+    }
+
+    forAll(integerRows) { value =>
+      val wdlValue = WdlString(value.toString)
+      val result = true
+      testContinueOnReturnCode(Option(wdlValue)) should be(result)
+      ContinueOnReturnCodeValidation.default.validateOptionalExpression(Option(wdlValue)) should be(result)
+      val valid =
+        ContinueOnReturnCodeValidation.default.validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> wdlValue))
+      valid.isValid should be(result)
+      valid.toEither.right.get should be(ContinueOnReturnCodeSet(Set(value)))
+    }
+
+    forAll(integerRows) { value =>
+      val wdlValue = WdlExpression.fromString(value.toString)
+      val result = true
+      testContinueOnReturnCode(Option(wdlValue)) should be(result)
+      ContinueOnReturnCodeValidation.default.validateOptionalExpression(Option(wdlValue)) should be(result)
+      // NOTE: expressions are never valid to validate
+    }
+
+    forAll(integerRows) { value =>
+      val wdlValue = WdlArray(WdlArrayType(WdlIntegerType), Seq(WdlInteger(value)))
+      val result = true
+      testContinueOnReturnCode(Option(wdlValue)) should be(result)
+      ContinueOnReturnCodeValidation.default.validateOptionalExpression(Option(wdlValue)) should be(result)
+      val valid =
+        ContinueOnReturnCodeValidation.default.validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> wdlValue))
+      valid.isValid should be(result)
+      valid.toEither.right.get should be(ContinueOnReturnCodeSet(Set(value)))
+    }
+
+    forAll(integerRows) { value =>
+      val wdlValue = WdlArray(WdlArrayType(WdlStringType), Seq(WdlString(value.toString)))
+      val result = true
+      testContinueOnReturnCode(Option(wdlValue)) should be(result)
+      ContinueOnReturnCodeValidation.default.validateOptionalExpression(Option(wdlValue)) should be(result)
+      val valid =
+        ContinueOnReturnCodeValidation.default.validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> wdlValue))
+      valid.isValid should be(result)
+      valid.toEither.right.get should be(ContinueOnReturnCodeSet(Set(value)))
+    }
+
+    forAll(integerRows) { value =>
+      val wdlValue = WdlArray(WdlArrayType(WdlExpressionType), Seq(WdlExpression.fromString(value.toString)))
+      val result = false
+      testContinueOnReturnCode(Option(wdlValue)) should be(result)
+      ContinueOnReturnCodeValidation.default.validateOptionalExpression(Option(wdlValue)) should be(result)
+      // NOTE: expressions are never valid to validate
+    }
+
+    forAll(expressionRows) { expression =>
+      val wdlValue = WdlExpression.fromString(expression)
+      val result = true
+      testContinueOnReturnCode(Option(wdlValue)) should be(result)
+      ContinueOnReturnCodeValidation.default.validateOptionalExpression(Option(wdlValue)) should be(result)
+      // NOTE: expressions are never valid to validate
+    }
+
+    forAll(invalidWdlValueRows) { wdlValue =>
+      val result = false
+      testContinueOnReturnCode(Option(wdlValue)) should be(result)
+      ContinueOnReturnCodeValidation.default.validateOptionalExpression(Option(wdlValue)) should be(result)
+      val valid =
+        ContinueOnReturnCodeValidation.default.validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> wdlValue))
+      valid.isValid should be(result)
+      valid.toEither.left.get.toList should contain theSameElementsAs List(
+        "Expecting continueOnReturnCode runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]"
+      )
+    }
+
+  }
+
+}
+
+class TestPredicateBackendWorkflowInitializationActor extends BackendWorkflowInitializationActor {
+  override val serviceRegistryActor: ActorRef = context.system.deadLetters
+
+  override def calls: Set[TaskCall] = throw new NotImplementedError("calls")
+
+  override protected def runtimeAttributeValidators: Map[String, (Option[WdlValue]) => Boolean] =
+    throw new NotImplementedError("runtimeAttributeValidators")
+
+  override protected def coerceDefaultRuntimeAttributes(options: WorkflowOptions): Try[Map[String, WdlValue]] =
+    throw new NotImplementedError("coerceDefaultRuntimeAttributes")
+
+  override def beforeAll(): Future[Option[BackendInitializationData]] = throw new NotImplementedError("beforeAll")
+
+  override def validate(): Future[Unit] = throw new NotImplementedError("validate")
+
+  override protected def workflowDescriptor: BackendWorkflowDescriptor =
+    throw new NotImplementedError("workflowDescriptor")
+
+  override protected def configurationDescriptor: BackendConfigurationDescriptor =
+    throw new NotImplementedError("configurationDescriptor")
+
+  override def continueOnReturnCodePredicate(valueRequired: Boolean)
+                                            (wdlExpressionMaybe: Option[WdlValue]): Boolean = {
+    super.continueOnReturnCodePredicate(valueRequired)(wdlExpressionMaybe)
+  }
+}
diff --git a/backend/src/test/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala b/backend/src/test/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala
index c257ffa4d..b116aae72 100644
--- a/backend/src/test/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala
+++ b/backend/src/test/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala
@@ -1,4 +1,4 @@
-package cromwell.backend.sfs
+package cromwell.backend.standard
 
 import cromwell.backend.RuntimeAttributeDefinition
 import cromwell.backend.validation.RuntimeAttributesKeys._
@@ -10,9 +10,9 @@ import org.specs2.mock.Mockito
 import spray.json.{JsArray, JsBoolean, JsNumber, JsObject, JsValue}
 import wdl4s.values.{WdlBoolean, WdlInteger, WdlString, WdlValue}
 
-class SharedFileSystemValidatedRuntimeAttributesBuilderSpec extends WordSpecLike with Matchers with Mockito {
+class StandardValidatedRuntimeAttributesBuilderSpec extends WordSpecLike with Matchers with Mockito {
 
-  val HelloWorld =
+  val HelloWorld: String =
     s"""
       |task hello {
       |  String addressee = "you"
@@ -37,7 +37,7 @@ class SharedFileSystemValidatedRuntimeAttributesBuilderSpec extends WordSpecLike
     FailOnStderrKey -> false,
     ContinueOnReturnCodeKey -> ContinueOnReturnCodeSet(Set(0)))
 
-  def workflowOptionsWithDefaultRuntimeAttributes(defaults: Map[String, JsValue]) = {
+  def workflowOptionsWithDefaultRuntimeAttributes(defaults: Map[String, JsValue]): WorkflowOptions = {
     WorkflowOptions(JsObject(Map("default_runtime_attributes" -> JsObject(defaults))))
   }
 
@@ -75,7 +75,18 @@ class SharedFileSystemValidatedRuntimeAttributesBuilderSpec extends WordSpecLike
       message should include("Unrecognized runtime attribute keys: docker")
     }
 
-    "log a warning and fail to validate an invalid Docker entry" in {
+    "log a warning and validate an invalid Docker entry" in {
+      /*
+      NOTE: The behavior used to be: when present, a "docker" runtime attribute would always be validated to ensure that
+      the value of the runtime attribute was a String-- even if the actual runtime attribute was unsupported by the
+      backend! NOW, if the backend doesn't support docker, and the runtime attributes contain say as an invalid integer,
+      say `{ docker: 1 }`, the validation will now succeed. If we want to change this, the git history contains a
+      concept of validated-yet-still-unsupported attributes. One could use this behavior to validate other standard
+      attributes such as CPU, Memory, etc.-- checking their syntax even when they are unsupported by the current
+      backend.
+
+      https://github.com/broadinstitute/cromwell/blob/a4a952de33f6d1ef646be51d298c3d613a8cce5f/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemValidatedRuntimeAttributesBuilder.scala
+       */
       val expectedRuntimeAttributes = defaultRuntimeAttributes
       val runtimeAttributes = Map("docker" -> WdlInteger(1))
       val mockWarnings = new MockWarnings
@@ -129,7 +140,7 @@ class SharedFileSystemValidatedRuntimeAttributesBuilderSpec extends WordSpecLike
       */
     class MockWarnings() {
       var warnings: Seq[String] = Seq.empty
-      val mockLogger = mock[Logger]
+      val mockLogger: Logger = mock[Logger]
       mockLogger.warn(anyString).answers { result =>
         result match {
           case message: String =>
@@ -139,8 +150,8 @@ class SharedFileSystemValidatedRuntimeAttributesBuilderSpec extends WordSpecLike
     }
   }
 
-  val defaultLogger = LoggerFactory.getLogger(classOf[SharedFileSystemValidatedRuntimeAttributesBuilderSpec])
-  val emptyWorkflowOptions = WorkflowOptions.fromMap(Map.empty).get
+  val defaultLogger: Logger = LoggerFactory.getLogger(classOf[StandardValidatedRuntimeAttributesBuilderSpec])
+  val emptyWorkflowOptions: WorkflowOptions = WorkflowOptions.fromMap(Map.empty).get
 
   private def assertRuntimeAttributesSuccessfulCreation(runtimeAttributes: Map[String, WdlValue],
                                                         expectedRuntimeAttributes: Map[String, Any],
@@ -149,9 +160,9 @@ class SharedFileSystemValidatedRuntimeAttributesBuilderSpec extends WordSpecLike
                                                         logger: Logger = defaultLogger): Unit = {
 
     val builder = if (includeDockerSupport) {
-      SharedFileSystemValidatedRuntimeAttributesBuilder.default.withValidation(DockerValidation.optional)
+      StandardValidatedRuntimeAttributesBuilder.default.withValidation(DockerValidation.optional)
     } else {
-      SharedFileSystemValidatedRuntimeAttributesBuilder.default
+      StandardValidatedRuntimeAttributesBuilder.default
     }
     val runtimeAttributeDefinitions = builder.definitions.toSet
     val addDefaultsToAttributes = RuntimeAttributeDefinition.addDefaultsToAttributes(runtimeAttributeDefinitions, workflowOptions) _
@@ -178,9 +189,9 @@ class SharedFileSystemValidatedRuntimeAttributesBuilderSpec extends WordSpecLike
                                                     logger: Logger = defaultLogger): Unit = {
     val thrown = the[RuntimeException] thrownBy {
       val builder = if (supportsDocker) {
-        SharedFileSystemValidatedRuntimeAttributesBuilder.default.withValidation(DockerValidation.optional)
+        StandardValidatedRuntimeAttributesBuilder.default.withValidation(DockerValidation.optional)
       } else {
-        SharedFileSystemValidatedRuntimeAttributesBuilder.default
+        StandardValidatedRuntimeAttributesBuilder.default
       }
       val runtimeAttributeDefinitions = builder.definitions.toSet
       val addDefaultsToAttributes = RuntimeAttributeDefinition.addDefaultsToAttributes(runtimeAttributeDefinitions, workflowOptions) _
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala
index 447310711..dc24e4c3d 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala
@@ -3,21 +3,18 @@ package cromwell.backend.impl.jes
 import java.net.SocketTimeoutException
 import java.nio.file.{Path, Paths}
 
-import akka.actor.{Actor, ActorLogging, ActorRef}
+import akka.actor.ActorRef
 import better.files._
 import com.google.api.client.googleapis.json.GoogleJsonResponseException
 import com.google.cloud.storage.contrib.nio.CloudStoragePath
-import cromwell.backend.BackendJobExecutionActor.BackendJobExecutionResponse
 import cromwell.backend._
 import cromwell.backend.async.{AbortedExecutionHandle, ExecutionHandle, FailedNonRetryableExecutionHandle, FailedRetryableExecutionHandle, PendingExecutionHandle, SuccessfulExecutionHandle}
 import cromwell.backend.impl.jes.RunStatus.TerminalRunStatus
 import cromwell.backend.impl.jes.io._
 import cromwell.backend.impl.jes.statuspolling.JesPollingActorClient
 import cromwell.backend.standard.{StandardAsyncExecutionActor, StandardAsyncExecutionActorParams, StandardAsyncJob}
-import cromwell.backend.validation.ContinueOnReturnCode
 import cromwell.backend.wdl.OutputEvaluator
 import cromwell.core._
-import cromwell.core.logging.JobLogging
 import cromwell.core.path.PathFactory._
 import cromwell.core.path.PathImplicits._
 import cromwell.core.path.proxy.PathProxy
@@ -27,25 +24,13 @@ import wdl4s.expression.{NoFunctions, WdlFunctions}
 import wdl4s.values._
 
 import scala.concurrent.duration._
-import scala.concurrent.{ExecutionContext, Future, Promise}
+import scala.concurrent.{ExecutionContext, Future}
 import scala.language.postfixOps
 import scala.util.{Failure, Success, Try}
 
-case class JesAsyncExecutionActorParams
-(
-  override val jobDescriptor: BackendJobDescriptor,
-  jesConfiguration: JesConfiguration,
-  jesBackendInitializationData: JesBackendInitializationData,
-  override val serviceRegistryActor: ActorRef,
-  jesBackendSingletonActorOption: Option[ActorRef],
-  override val completionPromise: Promise[BackendJobExecutionResponse]
-) extends StandardAsyncExecutionActorParams {
-  override val jobIdKey: String = JesJobExecutionActor.JesOperationIdKey
-  override val configurationDescriptor: BackendConfigurationDescriptor = jesConfiguration.configurationDescriptor
-  override val backendInitializationDataOption: Option[BackendInitializationData] = Option(jesBackendInitializationData)
-}
-
 object JesAsyncBackendJobExecutionActor {
+  val JesOperationIdKey = "__jes_operation_id"
+
   object WorkflowOptionKeys {
     val MonitoringScript = "monitoring_script"
     val GoogleProject = "google_project"
@@ -58,20 +43,14 @@ object JesAsyncBackendJobExecutionActor {
   private val ExtraConfigParamName = "__extra_config_gcs_path"
 }
 
-class JesAsyncBackendJobExecutionActor(val jesParams: JesAsyncExecutionActorParams)
-  extends Actor with ActorLogging with BackendJobLifecycleActor with StandardAsyncExecutionActor
-    with JesJobCachingActorHelper with JobLogging with JesPollingActorClient {
+class JesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyncExecutionActorParams)
+  extends BackendJobLifecycleActor with StandardAsyncExecutionActor with JesJobCachingActorHelper
+    with JesPollingActorClient {
 
   import JesAsyncBackendJobExecutionActor._
 
-  override val standardParams: StandardAsyncExecutionActorParams = jesParams
-  override val jesConfiguration: JesConfiguration = jesParams.jesConfiguration
-  override val initializationData: JesBackendInitializationData = {
-    backendInitializationDataAs[JesBackendInitializationData]
-  }
-
   val jesBackendSingletonActor: ActorRef =
-    jesParams.jesBackendSingletonActorOption.getOrElse(
+    standardParams.backendSingletonActorOption.getOrElse(
       throw new RuntimeException("JES Backend actor cannot exist without the JES backend singleton actor"))
 
   override type StandardAsyncRunInfo = Run
@@ -86,10 +65,6 @@ class JesAsyncBackendJobExecutionActor(val jesParams: JesAsyncExecutionActorPara
   override lazy val executeOrRecoverBackOff = SimpleExponentialBackoff(
     initialInterval = 3 seconds, maxInterval = 20 seconds, multiplier = 1.1)
 
-  override lazy val workflowDescriptor: BackendWorkflowDescriptor = jobDescriptor.workflowDescriptor
-
-  lazy val call: TaskCall = jobDescriptor.key.call
-
   override lazy val retryable: Boolean = jobDescriptor.key.attempt <= runtimeAttributes.preemptible
   private lazy val cmdInput =
     JesFileInput(ExecParamName, jesCallPaths.script.toRealString, Paths.get(jesCallPaths.scriptFilename), workingDisk)
@@ -230,14 +205,6 @@ class JesAsyncBackendJobExecutionActor(val jesParams: JesAsyncExecutionActorPara
     )
   }
 
-  override lazy val remoteStdErrPath: Path = jesStderrFile
-
-  override lazy val remoteReturnCodePath: Path = returnCodeGcsPath
-
-  override lazy val failOnStdErr: Boolean = runtimeAttributes.failOnStderr
-
-  override lazy val continueOnReturnCode: ContinueOnReturnCode = runtimeAttributes.continueOnReturnCode
-
   override lazy val commandLineFunctions: WdlFunctions[WdlValue] = backendEngineFunctions
 
   override lazy val commandLinePreProcessor: (EvaluatedTaskInputs) => Try[EvaluatedTaskInputs] = mapGcsValues
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendInitializationData.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendInitializationData.scala
index 4ba914c34..0d3d07c31 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendInitializationData.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendInitializationData.scala
@@ -1,7 +1,11 @@
 package cromwell.backend.impl.jes
 
 import com.google.api.services.genomics.Genomics
-import cromwell.backend.BackendInitializationData
+import cromwell.backend.standard.StandardInitializationData
 
-case class JesBackendInitializationData(workflowPaths: JesWorkflowPaths, genomics: Genomics)
-  extends BackendInitializationData
+case class JesBackendInitializationData
+(
+  override val workflowPaths: JesWorkflowPaths,
+  jesConfiguration: JesConfiguration,
+  genomics: Genomics
+) extends StandardInitializationData(workflowPaths, JesRuntimeAttributes.runtimeAttributesBuilder)
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala
index f61769379..00b5bfc96 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala
@@ -1,82 +1,61 @@
 package cromwell.backend.impl.jes
 
-import java.nio.file.Path
-
-import akka.actor.{ActorRef, Props}
-import com.typesafe.config.Config
+import akka.actor.ActorRef
 import cromwell.backend._
 import cromwell.backend.callcaching.FileHashingActor.FileHashingFunction
 import cromwell.backend.impl.jes.callcaching.JesBackendFileHashing
-import cromwell.backend.validation.RuntimeAttributesKeys
+import cromwell.backend.standard._
 import cromwell.core.CallOutputs
-import cromwell.core.Dispatcher.BackendDispatcher
 import wdl4s.TaskCall
 import wdl4s.expression.WdlStandardLibraryFunctions
 
-
 case class JesBackendLifecycleActorFactory(name: String, configurationDescriptor: BackendConfigurationDescriptor)
-  extends BackendLifecycleActorFactory {
+  extends StandardLifecycleActorFactory {
   import JesBackendLifecycleActorFactory._
 
-  val jesConfiguration = new JesConfiguration(configurationDescriptor)
+  override def initializationActorClass: Class[_ <: StandardInitializationActor] = classOf[JesInitializationActor]
 
-  override def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor,
-                                                calls: Set[TaskCall],
-                                                serviceRegistryActor: ActorRef): Option[Props] = {
-    Option(JesInitializationActor.props(workflowDescriptor, calls, jesConfiguration, serviceRegistryActor).withDispatcher(BackendDispatcher))
-  }
+  override def asyncExecutionActorClass: Class[_ <: StandardAsyncExecutionActor] =
+    classOf[JesAsyncBackendJobExecutionActor]
 
-  override def jobExecutionActorProps(jobDescriptor: BackendJobDescriptor,
-                                      initializationData: Option[BackendInitializationData],
-                                      serviceRegistryActor: ActorRef,
-                                      backendSingletonActor: Option[ActorRef]): Props = {
-    // The `JesInitializationActor` will only return a non-`Empty` `JesBackendInitializationData` from a successful `beforeAll`
-    // invocation, so the `get` here is safe.
-    JesJobExecutionActor.props(jobDescriptor, jesConfiguration, initializationData.toJes.get, serviceRegistryActor, backendSingletonActor).withDispatcher(BackendDispatcher)
-  }
+  override def standardCacheHitCopyingActorOption: Option[Class[_ <: StandardCacheHitCopyingActor]] =
+    Option(classOf[JesCacheHitCopyingActor])
 
-  override def cacheHitCopyingActorProps = Option(cacheHitCopyingActorInner _)
+  override def finalizationActorClassOption: Option[Class[_ <: StandardFinalizationActor]] =
+    Option(classOf[JesFinalizationActor])
 
-  def cacheHitCopyingActorInner(jobDescriptor: BackendJobDescriptor,
-                                         initializationData: Option[BackendInitializationData],
-                                         serviceRegistryActor: ActorRef): Props = {
-    // The `JesInitializationActor` will only return a non-`Empty` `JesBackendInitializationData` from a successful `beforeAll`
-    // invocation, so the `get` here is safe.
-    JesCacheHitCopyingActor.props(jobDescriptor, jesConfiguration, initializationData.toJes.get, serviceRegistryActor).withDispatcher(BackendDispatcher)
+  override def jobIdKey: String = JesAsyncBackendJobExecutionActor.JesOperationIdKey
+
+  val jesConfiguration = new JesConfiguration(configurationDescriptor)
+
+  override def workflowInitializationActorParams(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[TaskCall],
+                                                 serviceRegistryActor: ActorRef): StandardInitializationActorParams = {
+    JesInitializationActorParams(workflowDescriptor, calls, jesConfiguration, serviceRegistryActor)
   }
 
-  override def workflowFinalizationActorProps(workflowDescriptor: BackendWorkflowDescriptor,
-                                              calls: Set[TaskCall],
-                                              jobExecutionMap: JobExecutionMap,
-                                              workflowOutputs: CallOutputs,
-                                              initializationData: Option[BackendInitializationData]) = {
+  override def workflowFinalizationActorParams(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[TaskCall],
+                                              jobExecutionMap: JobExecutionMap, workflowOutputs: CallOutputs,
+                                              initializationDataOption: Option[BackendInitializationData]):
+  StandardFinalizationActorParams = {
     // The `JesInitializationActor` will only return a non-`Empty` `JesBackendInitializationData` from a successful `beforeAll`
     // invocation.  HOWEVER, the finalization actor is created regardless of whether workflow initialization was successful
     // or not.  So the finalization actor must be able to handle an empty `JesBackendInitializationData` option, and there is no
     // `.get` on the initialization data as there is with the execution or cache hit copying actor methods.
-    Option(JesFinalizationActor.props(workflowDescriptor, calls, jesConfiguration, jobExecutionMap, workflowOutputs, initializationData.toJes).withDispatcher(BackendDispatcher))
+    JesFinalizationActorParams(workflowDescriptor, calls, jesConfiguration, jobExecutionMap, workflowOutputs,
+      initializationDataOption)
   }
 
-  override def runtimeAttributeDefinitions(initializationDataOption: Option[BackendInitializationData]) = staticRuntimeAttributeDefinitions
+  override def runtimeAttributeDefinitions(initializationDataOption: Option[BackendInitializationData]):
+  Set[RuntimeAttributeDefinition] = staticRuntimeAttributeDefinitions
 
   override def expressionLanguageFunctions(workflowDescriptor: BackendWorkflowDescriptor,
                                            jobKey: BackendJobDescriptorKey,
                                            initializationData: Option[BackendInitializationData]): WdlStandardLibraryFunctions = {
 
-    val jesCallPaths = initializationData.toJes.get.workflowPaths.toJobPaths(jobKey)
+    val jesCallPaths = initializationData.toJes.get.workflowPaths.toJobPaths(jobKey, workflowDescriptor)
     new JesExpressionFunctions(List(jesCallPaths.gcsPathBuilder), jesCallPaths.callContext)
   }
 
-  override def getExecutionRootPath(workflowDescriptor: BackendWorkflowDescriptor, backendConfig: Config,
-                                    initializationData: Option[BackendInitializationData]): Path = {
-    initializationData.toJes.get.workflowPaths.executionRoot
-  }
-
-  override def getWorkflowExecutionRootPath(workflowDescriptor: BackendWorkflowDescriptor, backendConfig: Config,
-                                    initializationData: Option[BackendInitializationData]): Path = {
-    initializationData.toJes.get.workflowPaths.workflowRoot
-  }
-
   override def backendSingletonActorProps = Option(JesBackendSingletonActor.props(jesConfiguration.qps))
 
   override lazy val fileHashingFunction: Option[FileHashingFunction] = Option(FileHashingFunction(JesBackendFileHashing.getCrc32c))
@@ -90,21 +69,6 @@ object JesBackendLifecycleActorFactory {
     def toJes: Option[JesBackendInitializationData] = genericInitializationData collectFirst { case d: JesBackendInitializationData => d }
   }
 
-  val staticRuntimeAttributeDefinitions = {
-    import JesRuntimeAttributes._
-    import RuntimeAttributesKeys._
-
-    Set(
-      RuntimeAttributeDefinition(DockerKey, None, usedInCallCaching = true),
-      RuntimeAttributeDefinition(ContinueOnReturnCodeKey, Option(staticDefaults(ContinueOnReturnCodeKey)), usedInCallCaching = true),
-      RuntimeAttributeDefinition(CpuKey, Option(staticDefaults(CpuKey)), usedInCallCaching = false),
-      RuntimeAttributeDefinition(FailOnStderrKey, Option(staticDefaults(FailOnStderrKey)), usedInCallCaching = true),
-      RuntimeAttributeDefinition(MemoryKey, Option(staticDefaults(MemoryKey)), usedInCallCaching = false),
-      RuntimeAttributeDefinition(DisksKey, Option(staticDefaults(DisksKey)), usedInCallCaching = false),
-      RuntimeAttributeDefinition(ZonesKey, Option(staticDefaults(ZonesKey)), usedInCallCaching = false),
-      RuntimeAttributeDefinition(PreemptibleKey, Option(staticDefaults(PreemptibleKey)), usedInCallCaching = false),
-      RuntimeAttributeDefinition(BootDiskSizeKey, Option(staticDefaults(BootDiskSizeKey)), usedInCallCaching = false),
-      RuntimeAttributeDefinition(NoAddressKey, Option(staticDefaults(NoAddressKey)), usedInCallCaching = false)
-    )
-  }
+  val staticRuntimeAttributeDefinitions: Set[RuntimeAttributeDefinition] =
+    JesRuntimeAttributes.runtimeAttributesBuilder.definitions.toSet
 }
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesCacheHitCopyingActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesCacheHitCopyingActor.scala
index b37a9ff31..631eda1b8 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesCacheHitCopyingActor.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesCacheHitCopyingActor.scala
@@ -2,35 +2,10 @@ package cromwell.backend.impl.jes
 
 import java.nio.file.Path
 
-import akka.actor.{ActorRef, Props}
-import cromwell.core.Dispatcher.BackendDispatcher
-import cromwell.backend.callcaching.CacheHitDuplicating
-import cromwell.backend.{BackendCacheHitCopyingActor, BackendConfigurationDescriptor, BackendJobDescriptor, BackendWorkflowDescriptor}
+import cromwell.backend.standard.{StandardCacheHitCopyingActor, StandardCacheHitCopyingActorParams}
 import cromwell.core.path.PathCopier
-import cromwell.core.logging.JobLogging
 
-case class JesCacheHitCopyingActor(override val jobDescriptor: BackendJobDescriptor,
-                                   jesConfiguration: JesConfiguration,
-                                   initializationData: JesBackendInitializationData,
-                                   serviceRegistryActor: ActorRef)
-  extends BackendCacheHitCopyingActor with CacheHitDuplicating with JesJobCachingActorHelper with JobLogging {
+case class JesCacheHitCopyingActor(override val standardParams: StandardCacheHitCopyingActorParams)
+  extends StandardCacheHitCopyingActor with JesJobCachingActorHelper {
   override protected def duplicate(source: Path, destination: Path): Unit = PathCopier.copy(source, destination).get
-
-  override protected lazy val destinationCallRootPath: Path = jesCallPaths.callExecutionRoot
-
-  override protected lazy val destinationJobDetritusPaths: Map[String, Path] = jesCallPaths.detritusPaths
-  
-  override val workflowDescriptor: BackendWorkflowDescriptor = jobDescriptor.workflowDescriptor
-
-  override lazy val configurationDescriptor: BackendConfigurationDescriptor = jesConfiguration.configurationDescriptor
-}
-
-object JesCacheHitCopyingActor {
-
-  def props(jobDescriptor: BackendJobDescriptor,
-            jesConfiguration: JesConfiguration,
-            initializationData: JesBackendInitializationData,
-            serviceRegistryActor: ActorRef): Props = {
-    Props(new JesCacheHitCopyingActor(jobDescriptor, jesConfiguration, initializationData, serviceRegistryActor)).withDispatcher(BackendDispatcher)
-  }
 }
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesFinalizationActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesFinalizationActor.scala
index f8dbf0953..0cf19bc2d 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesFinalizationActor.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesFinalizationActor.scala
@@ -2,12 +2,11 @@ package cromwell.backend.impl.jes
 
 import java.nio.file.Path
 
-import akka.actor.Props
 import better.files._
 import cats.instances.future._
 import cats.syntax.functor._
-import cromwell.core.Dispatcher.BackendDispatcher
-import cromwell.backend.{BackendWorkflowDescriptor, BackendWorkflowFinalizationActor, JobExecutionMap}
+import cromwell.backend._
+import cromwell.backend.standard.{StandardFinalizationActor, StandardFinalizationActorParams}
 import cromwell.core.CallOutputs
 import cromwell.core.Dispatcher.IoDispatcher
 import cromwell.core.path.PathCopier
@@ -16,27 +15,28 @@ import wdl4s.TaskCall
 import scala.concurrent.Future
 import scala.language.postfixOps
 
-object JesFinalizationActor {
-  def props(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[TaskCall], jesConfiguration: JesConfiguration,
-            jobExecutionMap: JobExecutionMap, workflowOutputs: CallOutputs, initializationData: Option[JesBackendInitializationData]) = {
-    Props(new JesFinalizationActor(workflowDescriptor,
-      calls,
-      jesConfiguration,
-      jobExecutionMap,
-      workflowOutputs,
-      initializationData)).withDispatcher(BackendDispatcher)
-  }
+case class JesFinalizationActorParams
+(
+  workflowDescriptor: BackendWorkflowDescriptor,
+  calls: Set[TaskCall],
+  jesConfiguration: JesConfiguration,
+  jobExecutionMap: JobExecutionMap,
+  workflowOutputs: CallOutputs,
+  initializationDataOption: Option[BackendInitializationData]
+) extends StandardFinalizationActorParams {
+  override val configurationDescriptor: BackendConfigurationDescriptor = jesConfiguration.configurationDescriptor
 }
 
-class JesFinalizationActor (override val workflowDescriptor: BackendWorkflowDescriptor,
-                            override val calls: Set[TaskCall],
-                            jesConfiguration: JesConfiguration, jobExecutionMap: JobExecutionMap,
-                            workflowOutputs: CallOutputs,
-                            initializationData: Option[JesBackendInitializationData]) extends BackendWorkflowFinalizationActor {
+class JesFinalizationActor(jesParams: JesFinalizationActorParams)
+  extends StandardFinalizationActor {
+
+  override val standardParams: StandardFinalizationActorParams = jesParams
 
-  override val configurationDescriptor = jesConfiguration.configurationDescriptor
+  lazy val jesConfiguration: JesConfiguration = jesParams.jesConfiguration
 
-  private val workflowPaths = initializationData.map { _.workflowPaths }
+  private val workflowPaths = initializationDataOption.map {
+    _.asInstanceOf[JesBackendInitializationData].workflowPaths
+  }
 
   private val iOExecutionContext = context.system.dispatchers.lookup(IoDispatcher)
 
@@ -77,7 +77,7 @@ class JesFinalizationActor (override val workflowDescriptor: BackendWorkflowDesc
   private lazy val logPaths: Seq[Path] = {
     val allCallPaths = jobExecutionMap flatMap {
       case (backendJobDescriptor, keys) =>
-        keys map { JesWorkflowPaths(backendJobDescriptor, jesConfiguration)(context.system).toJobPaths(_) }
+        keys map { JesJobPaths(_, backendJobDescriptor, jesConfiguration)(context.system) }
     }
 
     allCallPaths.toSeq flatMap { callPaths =>
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesInitializationActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesInitializationActor.scala
index 685d2aec3..20d49fa9a 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesInitializationActor.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesInitializationActor.scala
@@ -6,55 +6,48 @@ import akka.actor.{ActorRef, Props}
 import cats.instances.future._
 import cats.syntax.functor._
 import com.google.api.services.genomics.Genomics
-import cromwell.core.Dispatcher.BackendDispatcher
-import cromwell.backend.impl.jes.JesInitializationActor._
 import cromwell.backend.impl.jes.authentication.{GcsLocalizing, JesAuthInformation}
 import cromwell.backend.impl.jes.io._
-import cromwell.backend.validation.RuntimeAttributesDefault
-import cromwell.backend.validation.RuntimeAttributesKeys._
-import cromwell.backend.{BackendInitializationData, BackendWorkflowDescriptor, BackendWorkflowInitializationActor}
-import cromwell.core.WorkflowOptions
+import cromwell.backend.standard.{StandardInitializationActor, StandardInitializationActorParams, StandardValidatedRuntimeAttributesBuilder}
+import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendWorkflowDescriptor}
+import cromwell.core.Dispatcher.BackendDispatcher
 import cromwell.filesystems.gcs.auth.{ClientSecrets, GoogleAuthMode}
 import spray.json.JsObject
 import wdl4s.TaskCall
-import wdl4s.types.{WdlBooleanType, WdlFloatType, WdlIntegerType, WdlStringType}
-import wdl4s.values.WdlValue
 
 import scala.concurrent.Future
 import scala.language.postfixOps
-import scala.util.Try
 
 object JesInitializationActor {
-  val SupportedKeys = Set(CpuKey, MemoryKey, DockerKey, FailOnStderrKey, ContinueOnReturnCodeKey, JesRuntimeAttributes.ZonesKey,
-    JesRuntimeAttributes.PreemptibleKey, JesRuntimeAttributes.BootDiskSizeKey, JesRuntimeAttributes.DisksKey)
-
+  /* NOTE: Only used by tests */
   def props(workflowDescriptor: BackendWorkflowDescriptor,
             calls: Set[TaskCall],
             jesConfiguration: JesConfiguration,
-            serviceRegistryActor: ActorRef): Props =
-    Props(new JesInitializationActor(workflowDescriptor, calls, jesConfiguration, serviceRegistryActor: ActorRef)).withDispatcher(BackendDispatcher)
+            serviceRegistryActor: ActorRef): Props = {
+    val params = JesInitializationActorParams(workflowDescriptor, calls, jesConfiguration, serviceRegistryActor)
+    Props(new JesInitializationActor(params)).withDispatcher(BackendDispatcher)
+  }
 }
 
-class JesInitializationActor(override val workflowDescriptor: BackendWorkflowDescriptor,
-                             override val calls: Set[TaskCall],
-                             private[jes] val jesConfiguration: JesConfiguration,
-                             override val serviceRegistryActor: ActorRef)
-  extends BackendWorkflowInitializationActor {
+case class JesInitializationActorParams
+(
+  workflowDescriptor: BackendWorkflowDescriptor,
+  calls: Set[TaskCall],
+  jesConfiguration: JesConfiguration,
+  serviceRegistryActor: ActorRef
+) extends StandardInitializationActorParams {
+  override val configurationDescriptor: BackendConfigurationDescriptor = jesConfiguration.configurationDescriptor
+}
+
+class JesInitializationActor(jesParams: JesInitializationActorParams)
+  extends StandardInitializationActor {
 
-  override protected def runtimeAttributeValidators: Map[String, (Option[WdlValue]) => Boolean] = Map(
-    CpuKey -> wdlTypePredicate(valueRequired = false, WdlIntegerType.isCoerceableFrom),
-    MemoryKey -> wdlTypePredicate(valueRequired = false, WdlStringType.isCoerceableFrom),
-    DockerKey -> wdlTypePredicate(valueRequired = true, WdlStringType.isCoerceableFrom),
-    FailOnStderrKey -> wdlTypePredicate(valueRequired = false, WdlBooleanType.isCoerceableFrom),
-    ContinueOnReturnCodeKey -> continueOnReturnCodePredicate(valueRequired = false),
-    JesRuntimeAttributes.PreemptibleKey -> wdlTypePredicate(valueRequired = false, WdlIntegerType.isCoerceableFrom),
-    JesRuntimeAttributes.BootDiskSizeKey -> wdlTypePredicate(valueRequired = false, WdlFloatType.isCoerceableFrom),
+  override val standardParams: StandardInitializationActorParams = jesParams
 
-    // TODO (eventually): make these more appropriate pre-checks
-    JesRuntimeAttributes.ZonesKey -> wdlTypePredicate(valueRequired = false, WdlStringType.isCoerceableFrom),
-    JesRuntimeAttributes.DisksKey -> wdlTypePredicate(valueRequired = false, WdlStringType.isCoerceableFrom))
+  override lazy val runtimeAttributesBuilder: StandardValidatedRuntimeAttributesBuilder =
+    JesRuntimeAttributes.runtimeAttributesBuilder
 
-  override val configurationDescriptor = jesConfiguration.configurationDescriptor
+  private val jesConfiguration = jesParams.jesConfiguration
 
   private[jes] lazy val refreshTokenAuth: Option[JesAuthInformation] = {
     for {
@@ -63,10 +56,6 @@ class JesInitializationActor(override val workflowDescriptor: BackendWorkflowDes
     } yield GcsLocalizing(clientSecrets, token)
   }
 
-  override protected def coerceDefaultRuntimeAttributes(options: WorkflowOptions): Try[Map[String, WdlValue]] = {
-    RuntimeAttributesDefault.workflowOptionsDefault(options, JesRuntimeAttributes.coercionMap)
-  }
-
   /**
     * A call which happens before anything else runs
     */
@@ -81,7 +70,7 @@ class JesInitializationActor(override val workflowDescriptor: BackendWorkflowDes
       workflowPaths = new JesWorkflowPaths(workflowDescriptor, jesConfiguration)(context.system)
       _ <- if (jesConfiguration.needAuthFileUpload) writeAuthenticationFile(workflowPaths) else Future.successful(())
       _ = publishWorkflowRoot(workflowPaths.workflowRoot.toString)
-    } yield Option(JesBackendInitializationData(workflowPaths, genomics))
+    } yield Option(JesBackendInitializationData(workflowPaths, jesConfiguration, genomics))
   }
 
   private def writeAuthenticationFile(workflowPath: JesWorkflowPaths): Future[Unit] = {
@@ -94,7 +83,7 @@ class JesInitializationActor(override val workflowDescriptor: BackendWorkflowDes
     } getOrElse Future.successful(())
   }
 
-  def generateAuthJson(authInformation: Option[JesAuthInformation]*) = {
+  def generateAuthJson(authInformation: Option[JesAuthInformation]*): Option[String] = {
     authInformation.flatten map { _.toMap } match {
       case Nil => None
       case jsons =>
@@ -102,21 +91,4 @@ class JesInitializationActor(override val workflowDescriptor: BackendWorkflowDes
         Option(JsObject("auths" -> JsObject(authsValues)).prettyPrint)
     }
   }
-
-  /**
-    * Validate that this WorkflowBackendActor can run all of the calls that it's been assigned
-    */
-  override def validate(): Future[Unit] = {
-    Future {
-      calls foreach { call =>
-        val runtimeAttributes = call.task.runtimeAttributes.attrs
-        val notSupportedAttributes = runtimeAttributes filterKeys { !SupportedKeys.contains(_) }
-
-        if (notSupportedAttributes.nonEmpty) {
-          val notSupportedAttrString = notSupportedAttributes.keys mkString ", "
-          workflowLogger.warn(s"Key/s [$notSupportedAttrString] is/are not supported by JesBackend. Unsupported attributes will not be part of jobs executions.")
-        }
-      }
-    }
-  }
 }
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobCachingActorHelper.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobCachingActorHelper.scala
index f24551d71..ad699f36b 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobCachingActorHelper.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobCachingActorHelper.scala
@@ -2,17 +2,14 @@ package cromwell.backend.impl.jes
 
 import java.nio.file.Path
 
-import akka.actor.{Actor, ActorRef}
+import akka.actor.Actor
 import better.files._
-import cromwell.backend.BackendWorkflowDescriptor
-import cromwell.backend.callcaching.JobCachingActorHelper
 import cromwell.backend.impl.jes.io.{JesAttachedDisk, JesWorkingDisk}
+import cromwell.backend.standard.StandardCachingActorHelper
 import cromwell.core.logging.JobLogging
 import cromwell.core.path.PathImplicits._
 
-import scala.util.Try
-
-trait JesJobCachingActorHelper extends JobCachingActorHelper {
+trait JesJobCachingActorHelper extends StandardCachingActorHelper {
   this: Actor with JobLogging =>
 
   val ExecParamName = "exec"
@@ -21,27 +18,15 @@ trait JesJobCachingActorHelper extends JobCachingActorHelper {
   val JesMonitoringScript: Path = JesWorkingDisk.MountPoint.resolve("monitoring.sh")
   val JesMonitoringLogFile: Path = JesWorkingDisk.MountPoint.resolve("monitoring.log")
 
-  def jesConfiguration: JesConfiguration
-
-  def initializationData: JesBackendInitializationData
-
-  def serviceRegistryActor: ActorRef
-  
-  def workflowDescriptor: BackendWorkflowDescriptor
+  lazy val initializationData: JesBackendInitializationData = {
+    backendInitializationDataAs[JesBackendInitializationData]
+  }
 
-  def getPath(str: String): Try[Path] = jesCallPaths.getPath(str)
+  lazy val jesConfiguration: JesConfiguration = initializationData.jesConfiguration
 
-  lazy val jesCallPaths: JesJobPaths = {
-    val workflowPaths = if (workflowDescriptor.breadCrumbs.isEmpty) {
-      initializationData.workflowPaths
-    } else {
-      new JesWorkflowPaths(workflowDescriptor, jesConfiguration)(context.system)
-    }
-    
-    workflowPaths.toJobPaths(jobDescriptor.key)
-  }
+  lazy val jesCallPaths: JesJobPaths = jobPaths.asInstanceOf[JesJobPaths]
 
-  lazy val runtimeAttributes = JesRuntimeAttributes(jobDescriptor.runtimeAttributes, jobLogger)
+  lazy val runtimeAttributes = JesRuntimeAttributes(validatedRuntimeAttributes)
 
   lazy val retryable: Boolean = jobDescriptor.key.attempt <= runtimeAttributes.preemptible
   lazy val workingDisk: JesAttachedDisk = runtimeAttributes.disks.find(_.name == JesWorkingDisk.Name).get
@@ -68,20 +53,12 @@ trait JesJobCachingActorHelper extends JobCachingActorHelper {
     defaultMonitoringOutputPath.toString, File(JesMonitoringLogFile).path, workingDisk)
   }
 
-  // Implements CacheHitDuplicating.startMetadataKeyValues
-  def startMetadataKeyValues: Map[String, Any] = {
-    val runtimeAttributesMetadata: Map[String, Any] = runtimeAttributes.asMap map {
-      case (key, value) => s"runtimeAttributes:$key" -> value
-    }
-    
-    val otherMetadata: Map[String, Any] = Map(
+  override protected def nonStandardMetadata: Map[String, Any] = {
+    Map(
       JesMetadataKeys.GoogleProject -> jesAttributes.project,
       JesMetadataKeys.ExecutionBucket -> jesAttributes.executionBucket,
       JesMetadataKeys.EndpointUrl -> jesAttributes.endpointUrl,
-      "preemptible" -> preemptible,
-      "cache:allowResultReuse" -> true
+      "preemptible" -> preemptible
     )
-
-    runtimeAttributesMetadata ++ jesCallPaths.metadataPaths ++ otherMetadata
   }
 }
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobExecutionActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobExecutionActor.scala
deleted file mode 100644
index 44d85cbfd..000000000
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobExecutionActor.scala
+++ /dev/null
@@ -1,61 +0,0 @@
-package cromwell.backend.impl.jes
-
-import akka.actor.{ActorRef, Props}
-import cromwell.backend._
-import cromwell.backend.standard.{StandardAsyncExecutionActor, StandardSyncExecutionActor, StandardSyncExecutionActorParams}
-import cromwell.core.Dispatcher.BackendDispatcher
-
-/** A default implementation of the sync params. */
-case class JesSyncExecutionActorParams
-(
-  override val jobDescriptor: BackendJobDescriptor,
-  jesConfiguration: JesConfiguration,
-  jesBackendInitializationData: JesBackendInitializationData,
-  override val serviceRegistryActor: ActorRef,
-  jesBackendSingletonActorOption: Option[ActorRef]
-) extends StandardSyncExecutionActorParams {
-  override val jobIdKey: String = JesJobExecutionActor.JesOperationIdKey
-  override val asyncJobExecutionActorClass: Class[_ <: StandardAsyncExecutionActor] = classOf[Nothing]
-  override val configurationDescriptor: BackendConfigurationDescriptor = jesConfiguration.configurationDescriptor
-  override val backendInitializationDataOption: Option[BackendInitializationData] = Option(jesBackendInitializationData)
-}
-
-object JesJobExecutionActor {
-  def props(jobDescriptor: BackendJobDescriptor,
-            jesWorkflowInfo: JesConfiguration,
-            initializationData: JesBackendInitializationData,
-            serviceRegistryActor: ActorRef,
-            jesBackendSingletonActor: Option[ActorRef]): Props = {
-    val params = JesSyncExecutionActorParams(
-      jobDescriptor,
-      jesWorkflowInfo,
-      initializationData,
-      serviceRegistryActor,
-      jesBackendSingletonActor)
-    Props(new JesJobExecutionActor(params)).withDispatcher(BackendDispatcher)
-  }
-
-  val JesOperationIdKey = "__jes_operation_id"
-}
-
-case class JesJobExecutionActor(jesParams: JesSyncExecutionActorParams)
-  extends StandardSyncExecutionActor(jesParams) {
-
-  override def createAsyncRefName(): String = "JesAsyncBackendJobExecutionActor"
-
-  override def createAsyncProps(): Props = jabjeaProps
-
-  private[jes] def jabjeaProps = {
-    Props(
-      new JesAsyncBackendJobExecutionActor(
-        JesAsyncExecutionActorParams(
-          jesParams.jobDescriptor,
-          jesParams.jesConfiguration,
-          jesParams.jesBackendInitializationData,
-          jesParams.serviceRegistryActor,
-          jesParams.jesBackendSingletonActorOption,
-          completionPromise)
-      )
-    )
-  }
-}
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesPipelineInfo.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesPipelineInfo.scala
index 063a374b8..990ee7c98 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesPipelineInfo.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesPipelineInfo.scala
@@ -31,13 +31,7 @@ trait JesPipelineInfoBuilder {
 
 object NonPreemptibleJesPipelineInfoBuilder extends JesPipelineInfoBuilder {
   def build(commandLine: String, runtimeAttributes: JesRuntimeAttributes): JesPipelineInfo = {
-    /*
-     It should be impossible for docker to be None here. Enforcing that w/ ADTs seemed more trouble than
-      it was worth. If you're ever debugging a NoSuchElementException which leads you here, that means
-      the more trouble than worth calculation was incorrect and we should have separate RuntimeAttributes for
-      docker and no docker cases
-    */
-    val dockerImage = runtimeAttributes.dockerImage.get
+    val dockerImage = runtimeAttributes.dockerImage
     val resources = buildResources(runtimeAttributes).setPreemptible(false)
     new NonPreemptibleJesPipelineInfoBuilder(resources, buildDockerExecutor(commandLine, dockerImage))
   }
@@ -45,8 +39,7 @@ object NonPreemptibleJesPipelineInfoBuilder extends JesPipelineInfoBuilder {
 
 object PreemptibleJesPipelineInfoBuilder extends JesPipelineInfoBuilder {
   def build(commandLine: String, runtimeAttributes: JesRuntimeAttributes): JesPipelineInfo = {
-    // See comment above
-    val dockerImage = runtimeAttributes.dockerImage.get
+    val dockerImage = runtimeAttributes.dockerImage
     val resources = buildResources(runtimeAttributes).setPreemptible(true)
     new PreemptibleJesPipelineInfoBuilder(resources, buildDockerExecutor(commandLine, dockerImage))
   }
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesRuntimeAttributes.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesRuntimeAttributes.scala
index 374ce05b9..a878974a8 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesRuntimeAttributes.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesRuntimeAttributes.scala
@@ -5,11 +5,8 @@ import cats.syntax.cartesian._
 import cats.syntax.validated._
 import cromwell.backend.MemorySize
 import cromwell.backend.impl.jes.io.{JesAttachedDisk, JesWorkingDisk}
-import cromwell.backend.validation.RuntimeAttributesDefault._
-import cromwell.backend.validation.RuntimeAttributesKeys._
-import cromwell.backend.validation.RuntimeAttributesValidation._
-import cromwell.backend.validation._
-import lenthall.exception.MessageAggregation
+import cromwell.backend.standard.StandardValidatedRuntimeAttributesBuilder
+import cromwell.backend.validation.{BooleanRuntimeAttributesValidation, _}
 import lenthall.validation.ErrorOr._
 import org.slf4j.Logger
 import wdl4s.types._
@@ -21,28 +18,12 @@ case class JesRuntimeAttributes(cpu: Int,
                                 bootDiskSize: Int,
                                 memory: MemorySize,
                                 disks: Seq[JesAttachedDisk],
-                                dockerImage: Option[String],
+                                dockerImage: String,
                                 failOnStderr: Boolean,
                                 continueOnReturnCode: ContinueOnReturnCode,
-                                noAddress: Boolean) {
-  import JesRuntimeAttributes._
-
-  lazy val asMap = Map[String, Any](
-    CpuKey -> cpu.toString,
-    ZonesKey -> zones.mkString(","),
-    PreemptibleKey -> preemptible.toString,
-    BootDiskSizeKey -> bootDiskSize.toString,
-    MemoryKey -> memory.toString,
-    DisksKey -> disks.mkString(","),
-    DockerKey -> dockerImage.get,
-    FailOnStderrKey -> failOnStderr.toString,
-    ContinueOnReturnCodeKey -> continueOnReturnCode
-  )
-}
+                                noAddress: Boolean)
 
 object JesRuntimeAttributes {
-  private val CpuDefaultValue = 1
-  private val ContinueOnReturnCodeDefaultValue = 0
   private val MemoryDefaultValue = "2 GB"
 
   val ZonesKey = "zones"
@@ -60,109 +41,133 @@ object JesRuntimeAttributes {
   val DisksKey = "disks"
   private val DisksDefaultValue = s"${JesWorkingDisk.Name} 10 SSD"
 
-  val staticDefaults = Map(
-    CpuKey -> WdlInteger(CpuDefaultValue),
-    DisksKey -> WdlString(DisksDefaultValue),
-    ZonesKey -> WdlString(ZoneDefaultValue),
-    ContinueOnReturnCodeKey -> WdlInteger(ContinueOnReturnCodeDefaultValue),
-    FailOnStderrKey -> WdlBoolean.False,
-    PreemptibleKey -> WdlInteger(PreemptibleDefaultValue),
-    MemoryKey -> WdlString(MemoryDefaultValue),
-    BootDiskSizeKey -> WdlInteger(BootDiskSizeDefaultValue),
-    NoAddressKey -> WdlBoolean(NoAddressDefaultValue)
-  )
-
-  private[jes] val coercionMap: Map[String, Set[WdlType]] = Map(
-    CpuKey -> Set(WdlIntegerType),
-    DisksKey -> Set(WdlStringType, WdlArrayType(WdlStringType)),
-    ZonesKey -> Set(WdlStringType, WdlArrayType(WdlStringType)),
-    ContinueOnReturnCodeKey -> ContinueOnReturnCode.validWdlTypes,
-    FailOnStderrKey -> Set(WdlBooleanType),
-    PreemptibleKey -> Set(WdlIntegerType),
-    MemoryKey -> Set(WdlStringType),
-    BootDiskSizeKey -> Set(WdlIntegerType),
-    NoAddressKey -> Set(WdlBooleanType),
-    DockerKey -> Set(WdlStringType)
-  )
-
-  def apply(attrs: Map[String, WdlValue], logger: Logger): JesRuntimeAttributes = {
-    warnUnrecognized(attrs.keySet, coercionMap.keySet, logger)
-
-    val cpu = validateCpu(attrs.get(CpuKey), noValueFoundFor(CpuKey))
-    val memory = validateMemory(attrs.get(MemoryKey), noValueFoundFor(MemoryKey))
-    val docker = validateDocker(attrs.get(DockerKey), noValueFoundFor(DockerKey))
-    val failOnStderr = validateFailOnStderr(attrs.get(FailOnStderrKey), noValueFoundFor(FailOnStderrKey))
-    val continueOnReturnCode = validateContinueOnReturnCode(attrs.get(ContinueOnReturnCodeKey), noValueFoundFor(ContinueOnReturnCodeKey))
-
-    val zones = validateZone(attrs(ZonesKey))
-    val preemptible = validatePreemptible(attrs(PreemptibleKey))
-    val noAddress = validateNoAddress(attrs(NoAddressKey))
-    val bootDiskSize = validateBootDisk(attrs(BootDiskSizeKey))
-    val disks = validateLocalDisks(attrs(DisksKey))
-    (cpu |@| zones |@| preemptible |@| bootDiskSize |@| memory |@| disks |@| docker |@| failOnStderr |@| continueOnReturnCode |@| noAddress) map {
-      new JesRuntimeAttributes(_, _, _, _, _, _, _, _, _, _)
-    } match {
-      case Valid(x) => x
-      case Invalid(nel) => throw new RuntimeException with MessageAggregation {
-        override def exceptionContext: String = "Runtime attribute validation failed"
-        override def errorMessages: Traversable[String] = nel.toList
-      }
-    }
+  private val cpuValidation: RuntimeAttributesValidation[Int] = CpuValidation.default
+
+  private val disksValidation: RuntimeAttributesValidation[Seq[JesAttachedDisk]] =
+    DisksValidation.withDefault(WdlString(DisksDefaultValue))
+
+  private val zonesValidation: RuntimeAttributesValidation[Vector[String]] =
+    ZonesValidation.withDefault(WdlString(ZoneDefaultValue))
+
+  private val preemptibleValidation: RuntimeAttributesValidation[Int] =
+    new IntRuntimeAttributesValidation(JesRuntimeAttributes.PreemptibleKey)
+      .withDefault(WdlInteger(PreemptibleDefaultValue))
+
+  private val memoryValidation: RuntimeAttributesValidation[MemorySize] =
+    MemoryValidation.withDefaultMemory(MemorySize.parse(MemoryDefaultValue).get)
+
+  private val bootDiskSizeValidation: RuntimeAttributesValidation[Int] =
+    new IntRuntimeAttributesValidation(JesRuntimeAttributes.BootDiskSizeKey)
+      .withDefault(WdlInteger(BootDiskSizeDefaultValue))
+
+  private val noAddressValidation: RuntimeAttributesValidation[Boolean] =
+    new BooleanRuntimeAttributesValidation(JesRuntimeAttributes.NoAddressKey)
+      .withDefault(WdlBoolean(NoAddressDefaultValue))
+
+  private val dockerValidation: RuntimeAttributesValidation[String] = DockerValidation.instance
+
+  val runtimeAttributesBuilder: StandardValidatedRuntimeAttributesBuilder =
+    StandardValidatedRuntimeAttributesBuilder.default.withValidation(
+      cpuValidation,
+      disksValidation,
+      zonesValidation,
+      preemptibleValidation,
+      memoryValidation,
+      bootDiskSizeValidation,
+      noAddressValidation,
+      dockerValidation
+    )
+
+  def apply(validatedRuntimeAttributes: ValidatedRuntimeAttributes): JesRuntimeAttributes = {
+    val cpu: Int = RuntimeAttributesValidation.extract(cpuValidation, validatedRuntimeAttributes)
+    val zones: Vector[String] = RuntimeAttributesValidation.extract(zonesValidation, validatedRuntimeAttributes)
+    val preemptible: Int = RuntimeAttributesValidation.extract(preemptibleValidation, validatedRuntimeAttributes)
+    val bootDiskSize: Int = RuntimeAttributesValidation.extract(bootDiskSizeValidation, validatedRuntimeAttributes)
+    val memory: MemorySize = RuntimeAttributesValidation.extract(memoryValidation, validatedRuntimeAttributes)
+    val disks: Seq[JesAttachedDisk] = RuntimeAttributesValidation.extract(disksValidation, validatedRuntimeAttributes)
+    val docker: String = RuntimeAttributesValidation.extract(dockerValidation, validatedRuntimeAttributes)
+    val failOnStderr = RuntimeAttributesValidation.extract(FailOnStderrValidation.default, validatedRuntimeAttributes)
+    val continueOnReturnCode =
+      RuntimeAttributesValidation.extract(ContinueOnReturnCodeValidation.default, validatedRuntimeAttributes)
+    val noAddress: Boolean = RuntimeAttributesValidation.extract(noAddressValidation, validatedRuntimeAttributes)
+
+    new JesRuntimeAttributes(
+      cpu,
+      zones,
+      preemptible,
+      bootDiskSize,
+      memory,
+      disks,
+      docker,
+      failOnStderr,
+      continueOnReturnCode,
+      noAddress
+    )
   }
 
-  private def validateZone(zoneValue: WdlValue): ErrorOr[Vector[String]] = {
-    zoneValue match {
-      case WdlString(s) => s.split("\\s+").toVector.validNel
-      case WdlArray(wdlType, value) if wdlType.memberType == WdlStringType =>
-        value.map(_.valueString).toVector.validNel
-      case _ => s"Expecting $ZonesKey runtime attribute to be either a whitespace separated String or an Array[String]".invalidNel
-    }
+  // NOTE: Currently only used by test specs
+  private[jes] def apply(attrs: Map[String, WdlValue], logger: Logger): JesRuntimeAttributes = {
+    val runtimeAttributesBuilder = JesRuntimeAttributes.runtimeAttributesBuilder
+    val validatedRuntimeAttributes = runtimeAttributesBuilder.build(attrs, logger)
+    apply(validatedRuntimeAttributes)
   }
+}
 
-  private def contextualizeFailure[T](validation: ErrorOr[T], key: String): ErrorOr[T] = {
-    validation.leftMap[String](errors => s"Failed to validate $key runtime attribute: " + errors.toList.mkString(",")).toValidatedNel
-  }
+object ZonesValidation extends RuntimeAttributesValidation[Vector[String]] {
+  override def key: String = JesRuntimeAttributes.ZonesKey
+
+  override def coercion: Traversable[WdlType] = Set(WdlStringType, WdlArrayType(WdlStringType))
 
-  private def validatePreemptible(preemptible: WdlValue): ErrorOr[Int] = {
-    contextualizeFailure(validateInt(preemptible), PreemptibleKey)
+  override protected def validateValue: PartialFunction[WdlValue, ErrorOr[Vector[String]]] = {
+    case WdlString(s) => s.split("\\s+").toVector.validNel
+    case WdlArray(wdlType, value) if wdlType.memberType == WdlStringType =>
+      value.map(_.valueString).toVector.validNel
   }
 
-  private def validateNoAddress(noAddress: WdlValue): ErrorOr[Boolean] = {
-    contextualizeFailure(validateBoolean(noAddress), NoAddressKey)
+  override protected def missingValueMessage: String =
+    s"Expecting $key runtime attribute to be either a whitespace separated String or an Array[String]"
+}
+
+object DisksValidation extends RuntimeAttributesValidation[Seq[JesAttachedDisk]] {
+  override def key: String = JesRuntimeAttributes.DisksKey
+
+  override def coercion: Traversable[WdlType] = Set(WdlStringType, WdlArrayType(WdlStringType))
+
+  override protected def validateValue: PartialFunction[WdlValue, ErrorOr[Seq[JesAttachedDisk]]] = {
+    case WdlString(value) => validateLocalDisks(value.split(",\\s*").toSeq)
+    case WdlArray(wdlType, values) if wdlType.memberType == WdlStringType =>
+      validateLocalDisks(values.map(_.valueString))
   }
 
-  private def validateBootDisk(diskSize: WdlValue): ErrorOr[Int] = diskSize match {
-    case x if WdlIntegerType.isCoerceableFrom(x.wdlType) =>
-      WdlIntegerType.coerceRawValue(x) match {
-        case scala.util.Success(x: WdlInteger) => x.value.intValue.validNel
-        case scala.util.Success(unhandled) => s"Coercion was expected to create an Integer but instead got $unhandled".invalidNel
-        case scala.util.Failure(t) => s"Expecting $BootDiskSizeKey runtime attribute to be an Integer".invalidNel
-      }
+  private def validateLocalDisks(disks: Seq[String]): ErrorOr[Seq[JesAttachedDisk]] = {
+    val diskNels: Seq[ErrorOr[JesAttachedDisk]] = disks map validateLocalDisk
+    val sequenced: ErrorOr[Seq[JesAttachedDisk]] = sequenceNels(diskNels)
+    val defaulted: ErrorOr[Seq[JesAttachedDisk]] = addDefault(sequenced)
+    defaulted
   }
 
-  private def validateLocalDisks(value: WdlValue): ErrorOr[Seq[JesAttachedDisk]] = {
-    val nels: Seq[ErrorOr[JesAttachedDisk]] = value match {
-      case WdlString(s) => s.split(",\\s*").toSeq.map(validateLocalDisk)
-      case WdlArray(wdlType, seq) if wdlType.memberType == WdlStringType =>
-        seq.map(_.valueString).map(validateLocalDisk)
-      case _ =>
-        Seq(s"Expecting $DisksKey runtime attribute to be a comma separated String or Array[String]".invalidNel)
+  private def validateLocalDisk(disk: String): ErrorOr[JesAttachedDisk] = {
+    JesAttachedDisk.parse(disk) match {
+      case scala.util.Success(attachedDisk) => attachedDisk.validNel
+      case scala.util.Failure(ex) => ex.getMessage.invalidNel
     }
+  }
 
+  private def sequenceNels(nels: Seq[ErrorOr[JesAttachedDisk]]): ErrorOr[Seq[JesAttachedDisk]] = {
     val emptyDiskNel = Vector.empty[JesAttachedDisk].validNel[String]
-    val disksNel = nels.foldLeft(emptyDiskNel)((acc, v) => (acc |@| v) map { (a, v) => a :+ v })
-
-    disksNel map {
-      case disks if disks.exists(_.name == JesWorkingDisk.Name) => disks
-      case disks => disks :+ JesAttachedDisk.parse(DisksDefaultValue).get
+    val disksNel: ErrorOr[Vector[JesAttachedDisk]] = nels.foldLeft(emptyDiskNel) {
+      (acc, v) => (acc |@| v) map { (a, v) => a :+ v }
     }
+    disksNel
   }
 
-  private def validateLocalDisk(disk: String): ErrorOr[JesAttachedDisk] = {
-    JesAttachedDisk.parse(disk) match {
-      case scala.util.Success(localDisk) => localDisk.validNel
-      case scala.util.Failure(ex) => ex.getMessage.invalidNel
+  private def addDefault(disksNel: ErrorOr[Seq[JesAttachedDisk]]): ErrorOr[Seq[JesAttachedDisk]] = {
+    disksNel map {
+      case disks if disks.exists(_.name == JesWorkingDisk.Name) => disks
+      case disks => disks :+ JesWorkingDisk.Default
     }
   }
 
+  override protected def missingValueMessage: String =
+    s"Expecting $key runtime attribute to be a comma separated String or Array[String]"
 }
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesWorkflowPaths.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesWorkflowPaths.scala
index 7feade96d..615525e2d 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesWorkflowPaths.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesWorkflowPaths.scala
@@ -13,13 +13,14 @@ import cromwell.core.path.PathImplicits._
 import cromwell.filesystems.gcs.{GcsPathBuilderFactory, RetryableGcsPathBuilder}
 
 import scala.language.postfixOps
+import scala.util.Try
 
 object JesWorkflowPaths {
   private val GcsRootOptionKey = "jes_gcs_root"
   private val AuthFilePathOptionKey = "auth_bucket"
 
   def apply(workflowDescriptor: BackendWorkflowDescriptor,
-            jesConfiguration: JesConfiguration)(implicit actorSystem: ActorSystem) = {
+            jesConfiguration: JesConfiguration)(implicit actorSystem: ActorSystem): JesWorkflowPaths = {
     new JesWorkflowPaths(workflowDescriptor, jesConfiguration)
   }
 }
@@ -27,11 +28,12 @@ object JesWorkflowPaths {
 class JesWorkflowPaths(val workflowDescriptor: BackendWorkflowDescriptor,
                        jesConfiguration: JesConfiguration)(implicit actorSystem: ActorSystem) extends WorkflowPaths {
 
-  override lazy val executionRootString = workflowDescriptor.workflowOptions.getOrElse(JesWorkflowPaths.GcsRootOptionKey, jesConfiguration.root)
+  override lazy val executionRootString: String =
+    workflowDescriptor.workflowOptions.getOrElse(JesWorkflowPaths.GcsRootOptionKey, jesConfiguration.root)
   private val workflowOptions: WorkflowOptions = workflowDescriptor.workflowOptions
   val gcsPathBuilder: RetryableGcsPathBuilder = jesConfiguration.gcsPathBuilderFactory.withOptions(workflowOptions)
 
-  def getHash(gcsUrl: Path) = gcsPathBuilder.getHash(gcsUrl)
+  def getHash(gcsUrl: Path): Try[String] = gcsPathBuilder.getHash(gcsUrl)
 
   val gcsAuthFilePath: Path = {
     /*
@@ -39,10 +41,10 @@ class JesWorkflowPaths(val workflowDescriptor: BackendWorkflowDescriptor,
      * unlike everywhere else where the filesystem used is built from gcsFileSystemAuth
      */
     val genomicsCredentials = jesConfiguration.jesAuths.genomics
-    
+
     // The default auth file bucket is always at the root of the root workflow
     val defaultBucket = executionRoot.resolve(workflowDescriptor.rootWorkflow.unqualifiedName).resolve(workflowDescriptor.rootWorkflowId.toString)
-    
+
     val bucket = workflowDescriptor.workflowOptions.get(JesWorkflowPaths.AuthFilePathOptionKey) getOrElse defaultBucket.toRealString
     val authBucket = GcsPathBuilderFactory(genomicsCredentials).withOptions(workflowOptions).build(bucket) recover {
       case ex => throw new Exception(s"Invalid gcs auth_bucket path $bucket", ex)
@@ -50,14 +52,16 @@ class JesWorkflowPaths(val workflowDescriptor: BackendWorkflowDescriptor,
 
     authBucket.resolve(s"${workflowDescriptor.rootWorkflowId}_auth.json")
   }
-  
-  
-  val monitoringPath = workflowOptions.get(WorkflowOptionKeys.MonitoringScript).toOption map { path =>
+
+  val monitoringPath: Option[Path] = workflowOptions.get(WorkflowOptionKeys.MonitoringScript).toOption map { path =>
     // Fail here if the path exists but can't be built
     getPath(path).get
   }
 
-  override def toJobPaths(jobKey: BackendJobDescriptorKey) = JesJobPaths(jobKey, workflowDescriptor, jesConfiguration)
+  override def toJobPaths(jobKey: BackendJobDescriptorKey,
+                          jobWorkflowDescriptor: BackendWorkflowDescriptor): JesJobPaths = {
+    JesJobPaths(jobKey, jobWorkflowDescriptor, jesConfiguration)
+  }
   override def config: Config = jesConfiguration.configurationDescriptor.backendConfig
   override def pathBuilders: List[PathBuilder] = List(gcsPathBuilder)
 }
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/JesAttachedDisk.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/JesAttachedDisk.scala
index 4066c1f41..38e702dc2 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/JesAttachedDisk.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/JesAttachedDisk.scala
@@ -11,14 +11,15 @@ import lenthall.exception.MessageAggregation
 import wdl4s.values._
 
 import scala.util.Try
+import scala.util.matching.Regex
 
 
 object JesAttachedDisk {
   val Identifier = "[a-zA-Z0-9-_]+"
   val Directory = """/[^\s]+"""
   val Integer = "[1-9][0-9]*"
-  val WorkingDiskPattern = s"""${JesWorkingDisk.Name}\\s+($Integer)\\s+($Identifier)""".r
-  val MountedDiskPattern = s"""($Directory)\\s+($Integer)\\s+($Identifier)""".r
+  val WorkingDiskPattern: Regex = s"""${JesWorkingDisk.Name}\\s+($Integer)\\s+($Identifier)""".r
+  val MountedDiskPattern: Regex = s"""($Directory)\\s+($Integer)\\s+($Identifier)""".r
 
   def parse(s: String): Try[JesAttachedDisk] = {
     val validation: ErrorOr[JesAttachedDisk] = s match {
@@ -38,7 +39,7 @@ object JesAttachedDisk {
       case Invalid(nels) =>
         throw new UnsupportedOperationException with MessageAggregation {
           val exceptionContext = ""
-          val errorMessages = nels.toList
+          val errorMessages: List[String] = nels.toList
         }
     })
   }
@@ -81,8 +82,9 @@ case class JesEmptyMountedDisk(diskType: DiskType, sizeGb: Int, mountPoint: Path
 }
 
 object JesWorkingDisk {
-  val MountPoint = Paths.get("/cromwell_root")
+  val MountPoint: Path = Paths.get("/cromwell_root")
   val Name = "local-disk"
+  val Default = JesWorkingDisk(DiskType.SSD, 10)
 }
 
 case class JesWorkingDisk(diskType: DiskType, sizeGb: Int) extends JesAttachedDisk {
diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala
index bbd713343..40fc7f44c 100644
--- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala
+++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala
@@ -14,21 +14,21 @@ import cromwell.backend.impl.jes.JesAsyncBackendJobExecutionActor.JesPendingExec
 import cromwell.backend.impl.jes.RunStatus.Failed
 import cromwell.backend.impl.jes.io.{DiskType, JesWorkingDisk}
 import cromwell.backend.impl.jes.statuspolling.JesApiQueryManager.DoPoll
-import cromwell.backend.standard.StandardAsyncJob
+import cromwell.backend.standard.{DefaultStandardAsyncExecutionActorParams, StandardAsyncExecutionActorParams, StandardAsyncJob}
+import cromwell.core._
 import cromwell.core.logging.JobLogger
 import cromwell.core.path.PathImplicits._
-import cromwell.core.{WorkflowId, WorkflowOptions, _}
-import cromwell.filesystems.gcs.{GcsPathBuilder, GcsPathBuilderFactory}
 import cromwell.filesystems.gcs.auth.GoogleAuthMode.NoAuthMode
+import cromwell.filesystems.gcs.{GcsPathBuilder, GcsPathBuilderFactory}
 import cromwell.util.SampleWdl
 import org.scalatest._
 import org.scalatest.prop.Tables.Table
 import org.slf4j.Logger
 import org.specs2.mock.Mockito
 import spray.json.{JsObject, JsValue}
+import wdl4s._
 import wdl4s.types.{WdlArrayType, WdlFileType, WdlMapType, WdlStringType}
 import wdl4s.values.{WdlArray, WdlFile, WdlMap, WdlString, WdlValue}
-import wdl4s.{LocallyQualifiedName, FullyQualifiedName => _, _}
 
 import scala.concurrent.duration._
 import scala.concurrent.{Await, ExecutionContext, Future, Promise}
@@ -76,11 +76,11 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
 
   private def buildInitializationData(jobDescriptor: BackendJobDescriptor, configuration: JesConfiguration) = {
     val workflowPaths = JesWorkflowPaths(jobDescriptor.workflowDescriptor, configuration)(system)
-    JesBackendInitializationData(workflowPaths, null)
+    JesBackendInitializationData(workflowPaths, configuration, null)
   }
 
-  class TestableJesJobExecutionActor(jesParams: JesAsyncExecutionActorParams, functions: JesExpressionFunctions)
-    extends JesAsyncBackendJobExecutionActor(jesParams) {
+  class TestableJesJobExecutionActor(params: StandardAsyncExecutionActorParams, functions: JesExpressionFunctions)
+    extends JesAsyncBackendJobExecutionActor(params) {
 
     def this(jobDescriptor: BackendJobDescriptor,
              promise: Promise[BackendJobExecutionResponse],
@@ -88,11 +88,12 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
              functions: JesExpressionFunctions = TestableJesExpressionFunctions,
              jesSingletonActor: ActorRef = emptyActor) = {
       this(
-        JesAsyncExecutionActorParams(
-          jobDescriptor,
-          jesConfiguration,
-          buildInitializationData(jobDescriptor, jesConfiguration),
+        DefaultStandardAsyncExecutionActorParams(
+          JesAsyncBackendJobExecutionActor.JesOperationIdKey,
           emptyActor,
+          jobDescriptor,
+          jesConfiguration.configurationDescriptor,
+          Option(buildInitializationData(jobDescriptor, jesConfiguration)),
           Option(jesSingletonActor),
           promise
         ),
diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala
index 73b8cb6de..44688edd8 100644
--- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala
+++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala
@@ -22,11 +22,11 @@ import scala.concurrent.duration._
 
 class JesInitializationActorSpec extends TestKitSuite("JesInitializationActorSpec") with FlatSpecLike with Matchers
   with ImplicitSender with Mockito {
-  val Timeout = 5.second.dilated
+  val Timeout: FiniteDuration = 5.second.dilated
 
   import BackendSpec._
 
-  val HelloWorld =
+  val HelloWorld: String =
     s"""
       |task hello {
       |  String addressee = "you"
@@ -45,7 +45,7 @@ class JesInitializationActorSpec extends TestKitSuite("JesInitializationActorSpe
       |}
     """.stripMargin
 
-  val globalConfig = ConfigFactory.parseString(
+  val globalConfig: Config = ConfigFactory.parseString(
     """
       |google {
       |
@@ -66,7 +66,7 @@ class JesInitializationActorSpec extends TestKitSuite("JesInitializationActorSpe
       |}
       | """.stripMargin)
 
-  val backendConfigTemplate =
+  val backendConfigTemplate: String =
     """
       |  // Google project
       |  project = "my-cromwell-workflows"
@@ -96,7 +96,7 @@ class JesInitializationActorSpec extends TestKitSuite("JesInitializationActorSpe
       |[DOCKERHUBCONFIG]
       |""".stripMargin
 
-  val refreshTokenConfigTemplate =
+  val refreshTokenConfigTemplate: String =
     """
       |  // Google project
       |  project = "my-cromwell-workflows"
@@ -124,9 +124,9 @@ class JesInitializationActorSpec extends TestKitSuite("JesInitializationActorSpe
       |  }
       |""".stripMargin
 
-  val backendConfig = ConfigFactory.parseString(backendConfigTemplate.replace("[DOCKERHUBCONFIG]", ""))
+  val backendConfig: Config = ConfigFactory.parseString(backendConfigTemplate.replace("[DOCKERHUBCONFIG]", ""))
 
-  val dockerBackendConfig = ConfigFactory.parseString(backendConfigTemplate.replace("[DOCKERHUBCONFIG]",
+  val dockerBackendConfig: Config = ConfigFactory.parseString(backendConfigTemplate.replace("[DOCKERHUBCONFIG]",
     """
       |dockerhub {
       |  account = "my@docker.account"
@@ -136,7 +136,7 @@ class JesInitializationActorSpec extends TestKitSuite("JesInitializationActorSpe
 
   val defaultBackendConfig = BackendConfigurationDescriptor(backendConfig, globalConfig)
 
-  val refreshTokenConfig = ConfigFactory.parseString(refreshTokenConfigTemplate)
+  val refreshTokenConfig: Config = ConfigFactory.parseString(refreshTokenConfigTemplate)
 
   private def getJesBackend(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[TaskCall], conf: BackendConfigurationDescriptor) = {
     system.actorOf(JesInitializationActor.props(workflowDescriptor, calls, new JesConfiguration(conf), emptyActor))
@@ -151,7 +151,7 @@ class JesInitializationActorSpec extends TestKitSuite("JesInitializationActorSpe
       val backend = getJesBackend(workflowDescriptor, workflowDescriptor.workflow.taskCalls,
         defaultBackendConfig)
       val eventPattern =
-        "Key/s [test] is/are not supported by JesBackend. Unsupported attributes will not be part of jobs executions."
+        "Key/s [test] is/are not supported by backend. Unsupported attributes will not be part of job executions."
       EventFilter.warning(pattern = escapePattern(eventPattern), occurrences = 1) intercept {
         backend ! Initialize
       }
diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesJobExecutionActorSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesJobExecutionActorSpec.scala
index b12703c23..5b72db08b 100644
--- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesJobExecutionActorSpec.scala
+++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesJobExecutionActorSpec.scala
@@ -1,17 +1,16 @@
 package cromwell.backend.impl.jes
 
 import akka.actor.{Actor, ActorRef, Props}
-import akka.testkit.{TestActorRef, TestProbe}
+import akka.testkit._
 import cromwell.backend.BackendJobDescriptor
+import cromwell.backend.BackendJobExecutionActor.{ExecuteJobCommand, JobFailedNonRetryableResponse}
+import cromwell.backend.impl.jes.ControllableFailingJabjea.JabjeaExplode
+import cromwell.backend.standard.{DefaultStandardSyncExecutionActorParams, StandardSyncExecutionActor, StandardSyncExecutionActorParams}
 import cromwell.core.TestKitSuite
 import org.scalatest.{FlatSpecLike, Matchers}
 import org.specs2.mock.Mockito
 
 import scala.concurrent.duration._
-import akka.testkit._
-import cromwell.backend.BackendJobExecutionActor.{ExecuteJobCommand, JobFailedNonRetryableResponse}
-import cromwell.backend.impl.jes.ControllableFailingJabjea.JabjeaExplode
-
 import scala.concurrent.{ExecutionContext, Promise}
 
 class JesJobExecutionActorSpec extends TestKitSuite("JesJobExecutionActorSpec") with FlatSpecLike with Matchers with Mockito {
@@ -29,10 +28,13 @@ class JesJobExecutionActorSpec extends TestKitSuite("JesJobExecutionActorSpec")
     val serviceRegistryActor = system.actorOf(Props.empty)
     val jesBackendSingletonActor = Option(system.actorOf(Props.empty))
 
+    initializationData.jesConfiguration returns jesWorkflowInfo
+
     val parent = TestProbe()
     val deathwatch = TestProbe()
-    val params = JesSyncExecutionActorParams(jobDescriptor, jesWorkflowInfo, initializationData, serviceRegistryActor,
-      jesBackendSingletonActor)
+    val params = DefaultStandardSyncExecutionActorParams(JesAsyncBackendJobExecutionActor.JesOperationIdKey, serviceRegistryActor,
+      jobDescriptor, null, Option(initializationData), jesBackendSingletonActor,
+      classOf[JesAsyncBackendJobExecutionActor])
     val testJJEA = TestActorRef[TestJesJobExecutionActor](
       props = Props(new TestJesJobExecutionActor(params, Props(new ConstructorFailingJABJEA))),
       supervisor = parent.ref)
@@ -57,11 +59,14 @@ class JesJobExecutionActorSpec extends TestKitSuite("JesJobExecutionActorSpec")
     val serviceRegistryActor = system.actorOf(Props.empty)
     val jesBackendSingletonActor = Option(system.actorOf(Props.empty))
 
+    initializationData.jesConfiguration returns jesWorkflowInfo
+
     val parent = TestProbe()
     val deathwatch = TestProbe()
     val jabjeaConstructionPromise = Promise[ActorRef]()
-    val params = JesSyncExecutionActorParams(jobDescriptor, jesWorkflowInfo, initializationData, serviceRegistryActor,
-      jesBackendSingletonActor)
+    val params = DefaultStandardSyncExecutionActorParams(JesAsyncBackendJobExecutionActor.JesOperationIdKey, serviceRegistryActor,
+      jobDescriptor, null, Option(initializationData), jesBackendSingletonActor,
+      classOf[JesAsyncBackendJobExecutionActor])
     val testJJEA = TestActorRef[TestJesJobExecutionActor](
       props = Props(new TestJesJobExecutionActor(params, Props(new ControllableFailingJabjea(jabjeaConstructionPromise)))),
       supervisor = parent.ref)
@@ -85,9 +90,9 @@ class JesJobExecutionActorSpec extends TestKitSuite("JesJobExecutionActorSpec")
   }
 }
 
-class TestJesJobExecutionActor(jesParams: JesSyncExecutionActorParams,
-                               fakeJabjeaProps: Props) extends JesJobExecutionActor(jesParams) {
-  override def jabjeaProps: Props = fakeJabjeaProps
+class TestJesJobExecutionActor(params: StandardSyncExecutionActorParams,
+                               fakeJabjeaProps: Props) extends StandardSyncExecutionActor(params) {
+  override def createAsyncProps(): Props = fakeJabjeaProps
 }
 
 class ConstructorFailingJABJEA extends ControllableFailingJabjea(Promise[ActorRef]()) {
diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesRuntimeAttributesSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesRuntimeAttributesSpec.scala
index 204907110..43b62287c 100644
--- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesRuntimeAttributesSpec.scala
+++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesRuntimeAttributesSpec.scala
@@ -14,14 +14,14 @@ import wdl4s.values.{WdlArray, WdlBoolean, WdlInteger, WdlString, WdlValue}
 
 class JesRuntimeAttributesSpec extends WordSpecLike with Matchers with Mockito {
 
-  def workflowOptionsWithDefaultRA(defaults: Map[String, JsValue]) = {
+  def workflowOptionsWithDefaultRA(defaults: Map[String, JsValue]): WorkflowOptions = {
     WorkflowOptions(JsObject(Map(
       "default_runtime_attributes" -> JsObject(defaults)
     )))
   }
 
-  val expectedDefaults = new JesRuntimeAttributes(1, Vector("us-central1-b"), 0, 10, MemorySize(2, MemoryUnit.GB), Seq(JesWorkingDisk(DiskType.SSD, 10)), None, false, ContinueOnReturnCodeSet(Set(0)), false)
-  val expectedDefaultsPlusUbuntuDocker = expectedDefaults.copy(dockerImage = Some("ubuntu:latest"))
+  val expectedDefaults = new JesRuntimeAttributes(1, Vector("us-central1-b"), 0, 10, MemorySize(2, MemoryUnit.GB),
+    Seq(JesWorkingDisk(DiskType.SSD, 10)), "ubuntu:latest", false, ContinueOnReturnCodeSet(Set(0)), false)
 
   "JesRuntimeAttributes" should {
 
@@ -32,7 +32,7 @@ class JesRuntimeAttributesSpec extends WordSpecLike with Matchers with Mockito {
 
     "validate a valid Docker entry" in {
       val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest"))
-      val expectedRuntimeAttributes = expectedDefaults.copy(dockerImage = Option("ubuntu:latest"))
+      val expectedRuntimeAttributes = expectedDefaults
       assertJesRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes)
     }
 
@@ -43,7 +43,7 @@ class JesRuntimeAttributesSpec extends WordSpecLike with Matchers with Mockito {
 
     "validate a valid failOnStderr entry" in {
       val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest"), "failOnStderr" -> WdlBoolean(true))
-      val expectedRuntimeAttributes = expectedDefaultsPlusUbuntuDocker.copy(failOnStderr = true)
+      val expectedRuntimeAttributes = expectedDefaults.copy(failOnStderr = true)
       assertJesRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes)
     }
 
@@ -54,19 +54,19 @@ class JesRuntimeAttributesSpec extends WordSpecLike with Matchers with Mockito {
 
     "validate a valid continueOnReturnCode entry" in {
       val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest"), "continueOnReturnCode" -> WdlInteger(1))
-      val expectedRuntimeAttributes = expectedDefaultsPlusUbuntuDocker.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1)))
+      val expectedRuntimeAttributes = expectedDefaults.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1)))
       assertJesRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes)
     }
 
     "validate a valid continueOnReturnCode array entry" in {
       val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest"), "continueOnReturnCode" -> WdlArray(WdlArrayType(WdlIntegerType), Array(WdlInteger(1), WdlInteger(2))))
-      val expectedRuntimeAttributes = expectedDefaultsPlusUbuntuDocker.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1, 2)))
+      val expectedRuntimeAttributes = expectedDefaults.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1, 2)))
       assertJesRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes)
     }
 
     "coerce then validate a valid continueOnReturnCode array entry" in {
       val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest"), "continueOnReturnCode" -> WdlArray(WdlArrayType(WdlStringType), Array(WdlString("1"), WdlString("2"))))
-      val expectedRuntimeAttributes = expectedDefaultsPlusUbuntuDocker.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1, 2)))
+      val expectedRuntimeAttributes = expectedDefaults.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1, 2)))
       assertJesRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes)
     }
 
@@ -77,13 +77,13 @@ class JesRuntimeAttributesSpec extends WordSpecLike with Matchers with Mockito {
 
     "validate a valid cpu entry" in {
       val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest"), "cpu" -> WdlInteger(2))
-      val expectedRuntimeAttributes = expectedDefaultsPlusUbuntuDocker.copy(cpu = 2)
+      val expectedRuntimeAttributes = expectedDefaults.copy(cpu = 2)
       assertJesRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes)
     }
 
     "validate a valid cpu string entry" in {
       val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest"), "cpu" -> WdlString("2"))
-      val expectedRuntimeAttributes = expectedDefaultsPlusUbuntuDocker.copy(cpu = 2)
+      val expectedRuntimeAttributes = expectedDefaults.copy(cpu = 2)
       assertJesRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes)
     }
 
@@ -94,7 +94,7 @@ class JesRuntimeAttributesSpec extends WordSpecLike with Matchers with Mockito {
 
     "validate a valid zones entry" in {
       val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest"), "zones" -> WdlString("us-central-z"))
-      val expectedRuntimeAttributes = expectedDefaultsPlusUbuntuDocker.copy(zones = Vector("us-central-z"))
+      val expectedRuntimeAttributes = expectedDefaults.copy(zones = Vector("us-central-z"))
       assertJesRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes)
     }
 
@@ -105,7 +105,7 @@ class JesRuntimeAttributesSpec extends WordSpecLike with Matchers with Mockito {
 
     "validate a valid array zones entry" in {
       val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest"), "zones" -> WdlArray(WdlArrayType(WdlStringType), Array(WdlString("us-central1-y"), WdlString("us-central1-z"))))
-      val expectedRuntimeAttributes = expectedDefaultsPlusUbuntuDocker.copy(zones = Vector("us-central1-y", "us-central1-z"))
+      val expectedRuntimeAttributes = expectedDefaults.copy(zones = Vector("us-central1-y", "us-central1-z"))
       assertJesRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes)
     }
 
@@ -116,18 +116,19 @@ class JesRuntimeAttributesSpec extends WordSpecLike with Matchers with Mockito {
 
     "validate a valid preemptible entry" in {
       val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest"), "preemptible" -> WdlInteger(3))
-      val expectedRuntimeAttributes = expectedDefaultsPlusUbuntuDocker.copy(preemptible = 3)
+      val expectedRuntimeAttributes = expectedDefaults.copy(preemptible = 3)
       assertJesRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes)
     }
 
     "fail to validate an invalid preemptible entry" in {
       val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest"), "preemptible" -> WdlString("value"))
-      assertJesRuntimeAttributesFailedCreation(runtimeAttributes, "Failed to validate preemptible runtime attribute: Could not coerce value into an integer")
+      assertJesRuntimeAttributesFailedCreation(runtimeAttributes,
+        "Expecting preemptible runtime attribute to be an Integer")
     }
 
     "validate a valid bootDiskSizeGb entry" in {
       val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest"), "bootDiskSizeGb" -> WdlInteger(4))
-      val expectedRuntimeAttributes = expectedDefaultsPlusUbuntuDocker.copy(bootDiskSize = 4)
+      val expectedRuntimeAttributes = expectedDefaults.copy(bootDiskSize = 4)
       assertJesRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes)
     }
 
@@ -138,7 +139,7 @@ class JesRuntimeAttributesSpec extends WordSpecLike with Matchers with Mockito {
 
     "validate a valid disks entry" in {
       val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest"), "disks" -> WdlString("local-disk 20 SSD"))
-      val expectedRuntimeAttributes = expectedDefaultsPlusUbuntuDocker.copy(disks = Seq(JesAttachedDisk.parse("local-disk 20 SSD").get))
+      val expectedRuntimeAttributes = expectedDefaults.copy(disks = Seq(JesAttachedDisk.parse("local-disk 20 SSD").get))
       assertJesRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes)
     }
 
@@ -149,7 +150,7 @@ class JesRuntimeAttributesSpec extends WordSpecLike with Matchers with Mockito {
 
     "validate a valid disks array entry" in {
       val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest"), "disks" -> WdlArray(WdlArrayType(WdlStringType), Array(WdlString("local-disk 20 SSD"), WdlString("local-disk 30 SSD"))))
-      val expectedRuntimeAttributes = expectedDefaultsPlusUbuntuDocker.copy(disks = Seq(JesAttachedDisk.parse("local-disk 20 SSD").get, JesAttachedDisk.parse("local-disk 30 SSD").get))
+      val expectedRuntimeAttributes = expectedDefaults.copy(disks = Seq(JesAttachedDisk.parse("local-disk 20 SSD").get, JesAttachedDisk.parse("local-disk 30 SSD").get))
       assertJesRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes)
     }
 
@@ -160,7 +161,7 @@ class JesRuntimeAttributesSpec extends WordSpecLike with Matchers with Mockito {
 
     "validate a valid memory entry" in {
       val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest"), "memory" -> WdlString("1 GB"))
-      val expectedRuntimeAttributes = expectedDefaultsPlusUbuntuDocker.copy(memory = MemorySize.parse("1 GB").get)
+      val expectedRuntimeAttributes = expectedDefaults.copy(memory = MemorySize.parse("1 GB").get)
       assertJesRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes)
     }
 
@@ -171,18 +172,19 @@ class JesRuntimeAttributesSpec extends WordSpecLike with Matchers with Mockito {
 
     "validate a valid noAddress entry" in {
       val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest"), "noAddress" -> WdlBoolean(true))
-      val expectedRuntimeAttributes = expectedDefaultsPlusUbuntuDocker.copy(noAddress = true)
+      val expectedRuntimeAttributes = expectedDefaults.copy(noAddress = true)
       assertJesRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes)
     }
 
     "fail to validate an invalid noAddress entry" in {
       val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest"), "noAddress" -> WdlInteger(1))
-      assertJesRuntimeAttributesFailedCreation(runtimeAttributes, "Failed to validate noAddress runtime attribute: Could not coerce 1 into a boolean")
+      assertJesRuntimeAttributesFailedCreation(runtimeAttributes,
+        "Expecting noAddress runtime attribute to be a Boolean")
     }
 
     "use reasonable default values" in {
       val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest"))
-      val expectedRuntimeAttributes = expectedDefaultsPlusUbuntuDocker
+      val expectedRuntimeAttributes = expectedDefaults
       assertJesRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes)
     }
   }
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigAsyncJobExecutionActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigAsyncJobExecutionActor.scala
index 12ca089eb..da3ee2e87 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigAsyncJobExecutionActor.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigAsyncJobExecutionActor.scala
@@ -83,7 +83,7 @@ $command
   private lazy val dockerInputs: WorkflowCoercedInputs = {
     if (isDockerRun) {
       Map(
-        DockerCwdInput -> WdlString(jobPaths.callDockerRoot.toString)
+        DockerCwdInput -> WdlString(jobPathsWithDocker.callDockerRoot.toString)
       )
     } else {
       Map.empty
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendLifecycleActorFactory.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendLifecycleActorFactory.scala
index 0debe3aee..5296020c6 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendLifecycleActorFactory.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendLifecycleActorFactory.scala
@@ -4,10 +4,11 @@ import com.typesafe.config.Config
 import cromwell.backend.callcaching.FileHashingActor.FileHashingFunction
 import cromwell.backend.impl.sfs.config.ConfigConstants._
 import cromwell.backend.sfs._
+import cromwell.backend.standard.StandardInitializationData
 import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, RuntimeAttributeDefinition}
 import cromwell.core.JobExecutionToken.JobExecutionTokenType
 import net.ceedubs.ficus.Ficus._
-import org.slf4j.LoggerFactory
+import org.slf4j.{Logger, LoggerFactory}
 
 /**
   * Builds a backend by reading the job control from the config.
@@ -17,14 +18,14 @@ import org.slf4j.LoggerFactory
 class ConfigBackendLifecycleActorFactory(name: String, val configurationDescriptor: BackendConfigurationDescriptor)
   extends SharedFileSystemBackendLifecycleActorFactory {
 
-  lazy val logger = LoggerFactory.getLogger(getClass)
-  lazy val hashingStrategy = {
+  lazy val logger: Logger = LoggerFactory.getLogger(getClass)
+  lazy val hashingStrategy: ConfigHashingStrategy = {
     configurationDescriptor.backendConfig.as[Option[Config]]("filesystems.local.caching") map ConfigHashingStrategy.apply getOrElse ConfigHashingStrategy.defaultStrategy
   }
 
-  override def initializationActorClass = classOf[ConfigInitializationActor]
+  override def initializationActorClass: Class[ConfigInitializationActor] = classOf[ConfigInitializationActor]
 
-  override def asyncJobExecutionActorClass: Class[_ <: ConfigAsyncJobExecutionActor] = {
+  override def asyncExecutionActorClass: Class[_ <: ConfigAsyncJobExecutionActor] = {
     val runInBackground = configurationDescriptor.backendConfig.as[Option[Boolean]](RunInBackgroundConfig).getOrElse(false)
     if (runInBackground)
       classOf[BackgroundConfigAsyncJobExecutionActor]
@@ -35,7 +36,7 @@ class ConfigBackendLifecycleActorFactory(name: String, val configurationDescript
   override def runtimeAttributeDefinitions(initializationDataOption: Option[BackendInitializationData]):
   Set[RuntimeAttributeDefinition] = {
     val initializationData = BackendInitializationData.
-      as[SharedFileSystemBackendInitializationData](initializationDataOption)
+      as[StandardInitializationData](initializationDataOption)
 
     initializationData.runtimeAttributesBuilder.definitions.toSet
   }
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategy.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategy.scala
index f453cf315..3f528022f 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategy.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategy.scala
@@ -4,18 +4,18 @@ import akka.event.LoggingAdapter
 import better.files.File
 import com.typesafe.config.Config
 import cromwell.backend.callcaching.FileHashingActor.SingleFileHashRequest
-import cromwell.backend.sfs.SharedFileSystemBackendInitializationData
+import cromwell.backend.standard.StandardInitializationData
 import cromwell.core.path.PathFactory
-import cromwell.util.TryWithResource._
 import cromwell.util.FileUtil._
+import cromwell.util.TryWithResource._
 import net.ceedubs.ficus.Ficus._
 import org.apache.commons.codec.digest.DigestUtils
-import org.slf4j.LoggerFactory
+import org.slf4j.{Logger, LoggerFactory}
 
 import scala.util.{Failure, Try}
 
 object ConfigHashingStrategy {
-  val logger = LoggerFactory.getLogger(getClass)
+  val logger: Logger = LoggerFactory.getLogger(getClass)
   val defaultStrategy = HashFileStrategy(false)
 
   def apply(hashingConfig: Config): ConfigHashingStrategy = {
@@ -36,10 +36,11 @@ abstract class ConfigHashingStrategy {
   protected def hash(file: File): Try[String]
   protected def description: String
 
-  protected lazy val checkSiblingMessage = if (checkSiblingMd5) "Check first for sibling md5 and if not found " else ""
+  protected lazy val checkSiblingMessage: String =
+    if (checkSiblingMd5) "Check first for sibling md5 and if not found " else ""
 
   def getHash(request: SingleFileHashRequest, log: LoggingAdapter): Try[String] = {
-    def usingSFSInitData(initData: SharedFileSystemBackendInitializationData) = {
+    def usingStandardInitData(initData: StandardInitializationData) = {
       val pathBuilders = initData.workflowPaths.pathBuilders
       val file = PathFactory.buildFile(request.file.valueString, pathBuilders).followSymlinks
 
@@ -52,7 +53,7 @@ abstract class ConfigHashingStrategy {
     }
 
     request.initializationData match {
-      case Some(initData: SharedFileSystemBackendInitializationData) => usingSFSInitData(initData)
+      case Some(initData: StandardInitializationData) => usingStandardInitData(initData)
       case _ => Failure(new IllegalArgumentException("Need SharedFileSystemBackendInitializationData to calculate hash."))
     }
   }
@@ -62,7 +63,7 @@ abstract class ConfigHashingStrategy {
     if (md5.exists) Option(md5) else None
   }
 
-  override def toString = {
+  override def toString: String = {
     s"Call caching hashing strategy: $checkSiblingMessage$description."
   }
 }
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigInitializationActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigInitializationActor.scala
index f9f5a882f..c78013582 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigInitializationActor.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigInitializationActor.scala
@@ -2,11 +2,12 @@ package cromwell.backend.impl.sfs.config
 
 import cromwell.backend.io.WorkflowPaths
 import cromwell.backend.sfs._
+import cromwell.backend.standard.{DefaultInitializationActorParams, StandardInitializationData, StandardValidatedRuntimeAttributesBuilder}
 import wdl4s.WdlNamespace
 
 /**
   * Extension of the SharedFileSystemBackendInitializationData with declarations of extra runtime attributes, and a
-  * wdl namespace containing various tasks for submiting, killing, etc.
+  * wdl namespace containing various tasks for submitting, killing, etc.
   *
   * @param workflowPaths            The paths for the workflow.
   * @param runtimeAttributesBuilder The customized runtime attributes builder with extra validations for the
@@ -17,10 +18,10 @@ import wdl4s.WdlNamespace
 class ConfigInitializationData
 (
   workflowPaths: WorkflowPaths,
-  runtimeAttributesBuilder: SharedFileSystemValidatedRuntimeAttributesBuilder,
+  runtimeAttributesBuilder: StandardValidatedRuntimeAttributesBuilder,
   val declarationValidations: Seq[DeclarationValidation],
   val wdlNamespace: WdlNamespace)
-  extends SharedFileSystemBackendInitializationData(workflowPaths, runtimeAttributesBuilder)
+  extends StandardInitializationData(workflowPaths, runtimeAttributesBuilder)
 
 /**
   * Extends the SharedFileSystemInitializationActor to create an instance of the ConfigInitializationData.
@@ -29,7 +30,7 @@ class ConfigInitializationData
   *
   * @param params Parameters to create an initialization actor.
   */
-class ConfigInitializationActor(params: SharedFileSystemInitializationActorParams)
+class ConfigInitializationActor(params: DefaultInitializationActorParams)
   extends SharedFileSystemInitializationActor(params) {
 
   lazy val configWdlNamespace = new ConfigWdlNamespace(params.configurationDescriptor.backendConfig)
@@ -38,12 +39,12 @@ class ConfigInitializationActor(params: SharedFileSystemInitializationActorParam
     DeclarationValidation.fromDeclarations(configWdlNamespace.runtimeDeclarations)
   }
 
-  override lazy val initializationData = {
+  override lazy val initializationData: ConfigInitializationData = {
     val wdlNamespace = configWdlNamespace.wdlNamespace
     new ConfigInitializationData(workflowPaths, runtimeAttributesBuilder, declarationValidations, wdlNamespace)
   }
 
-  override lazy val runtimeAttributesBuilder = {
+  override lazy val runtimeAttributesBuilder: StandardValidatedRuntimeAttributesBuilder = {
     val declared = declarationValidations.map(_.makeValidation())
     super.runtimeAttributesBuilder.withValidation(declared: _*)
   }
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/DeclarationValidation.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/DeclarationValidation.scala
index 707f1a93f..904705180 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/DeclarationValidation.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/DeclarationValidation.scala
@@ -26,8 +26,9 @@ object DeclarationValidation {
   def fromDeclaration(declaration: Declaration): DeclarationValidation = {
     declaration.unqualifiedName match {
       // Docker and CPU are special keys understood by cromwell.
-      case DockerValidation.key => new DeclarationValidation(declaration, DockerValidation.instance)
-      case CpuValidation.key => new DeclarationValidation(declaration, CpuValidation.default)
+      case name if name == DockerValidation.instance.key =>
+        new DeclarationValidation(declaration, DockerValidation.instance)
+      case name if name == CpuValidation.instance.key => new DeclarationValidation(declaration, CpuValidation.default)
       // See MemoryDeclarationValidation for more info
       case name if MemoryDeclarationValidation.isMemoryDeclaration(name) =>
         new MemoryDeclarationValidation(declaration)
@@ -38,13 +39,15 @@ object DeclarationValidation {
     }
   }
 
-  private def validator(wdlType: WdlType, unqualifiedName: String): PrimitiveRuntimeAttributesValidation[_] = wdlType match {
-    case WdlBooleanType => new BooleanRuntimeAttributesValidation(unqualifiedName)
-    case WdlFloatType => new FloatRuntimeAttributesValidation(unqualifiedName)
-    case WdlIntegerType => new IntRuntimeAttributesValidation(unqualifiedName)
-    case WdlStringType => new StringRuntimeAttributesValidation(unqualifiedName)
-    case WdlOptionalType(x) => validator(x, unqualifiedName)
-    case other => throw new RuntimeException(s"Unsupported config runtime attribute $other $unqualifiedName")
+  private def validator(wdlType: WdlType, unqualifiedName: String): PrimitiveRuntimeAttributesValidation[_, _] = {
+    wdlType match {
+      case WdlBooleanType => new BooleanRuntimeAttributesValidation(unqualifiedName)
+      case WdlFloatType => new FloatRuntimeAttributesValidation(unqualifiedName)
+      case WdlIntegerType => new IntRuntimeAttributesValidation(unqualifiedName)
+      case WdlStringType => new StringRuntimeAttributesValidation(unqualifiedName)
+      case WdlOptionalType(x) => validator(x, unqualifiedName)
+      case other => throw new RuntimeException(s"Unsupported config runtime attribute $other $unqualifiedName")
+    }
   }
 }
 
@@ -55,7 +58,7 @@ object DeclarationValidation {
   * @param instanceValidation A basic instance validation for the declaration.
   */
 class DeclarationValidation(declaration: Declaration, instanceValidation: RuntimeAttributesValidation[_]) {
-  val key = declaration.unqualifiedName
+  val key: String = declaration.unqualifiedName
 
   /**
     * Creates a validation, by adding on defaults if they're specified in the declaration, and then making the
@@ -142,7 +145,8 @@ class MemoryDeclarationValidation(declaration: Declaration)
     * @param wdlExpression The declaration expression to retrieve the default.
     * @return The new validation.
     */
-  override protected def default(validation: RuntimeAttributesValidation[_], wdlExpression: WdlExpression) = {
+  override protected def default(validation: RuntimeAttributesValidation[_],
+                                 wdlExpression: WdlExpression): RuntimeAttributesValidation[_] = {
     val wdlValue = declaration.expression.get.evaluate(NoLookup, NoFunctions).get
     val amount: Double = wdlValue match {
       case WdlInteger(value) => value.toDouble
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala
index f0a0814c3..2c850aaf8 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala
@@ -2,30 +2,28 @@ package cromwell.backend.sfs
 
 import java.nio.file.{FileAlreadyExistsException, Path}
 
-import akka.actor.{Actor, ActorLogging}
 import better.files._
 import cromwell.backend._
-import cromwell.backend.async.{AsyncBackendJobExecutionActor, ExecutionHandle, FailedNonRetryableExecutionHandle, PendingExecutionHandle, SuccessfulExecutionHandle}
-import cromwell.backend.io.WorkflowPathsBackendInitializationData
-import cromwell.backend.standard.{StandardAsyncExecutionActor, StandardAsyncJob}
+import cromwell.backend.async.{ExecutionHandle, FailedNonRetryableExecutionHandle, PendingExecutionHandle, SuccessfulExecutionHandle}
+import cromwell.backend.io.JobPathsWithDocker
+import cromwell.backend.standard.{StandardAsyncExecutionActor, StandardAsyncJob, StandardInitializationData}
 import cromwell.backend.validation._
 import cromwell.backend.wdl.OutputEvaluator
-import cromwell.core.WorkflowId
 import cromwell.core.path.PathFactory._
 import cromwell.core.path.{DefaultPathBuilder, PathBuilder}
 import cromwell.core.retry.SimpleExponentialBackoff
+import wdl4s.EvaluatedTaskInputs
 import wdl4s.values.{WdlArray, WdlFile, WdlGlobFile, WdlMap, WdlValue}
-import wdl4s.{EvaluatedTaskInputs, TaskCall}
 
 import scala.concurrent.duration._
 import scala.util.{Failure, Success, Try}
 
-object SharedFileSystemJob {
+case class SharedFileSystemRunStatus(returnCodeFileExists: Boolean)
+
+object SharedFileSystemAsyncJobExecutionActor {
   val JobIdKey = "sfs_job_id"
 }
 
-case class SharedFileSystemRunStatus(returnCodeFileExists: Boolean)
-
 /**
   * Runs a job on a shared backend, with the ability to (abstractly) submit asynchronously, then poll, kill, etc.
   *
@@ -51,8 +49,7 @@ case class SharedFileSystemRunStatus(returnCodeFileExists: Boolean)
   * messages.
   */
 trait SharedFileSystemAsyncJobExecutionActor
-  extends Actor with ActorLogging with BackendJobLifecycleActor with AsyncBackendJobExecutionActor
-    with StandardAsyncExecutionActor with SharedFileSystemJobCachingActorHelper {
+  extends BackendJobLifecycleActor with StandardAsyncExecutionActor with SharedFileSystemJobCachingActorHelper {
 
   override type StandardAsyncRunInfo = Any
 
@@ -98,11 +95,13 @@ trait SharedFileSystemAsyncJobExecutionActor
     */
   def killArgs(job: StandardAsyncJob): SharedFileSystemCommand
 
+  lazy val jobPathsWithDocker: JobPathsWithDocker = jobPaths.asInstanceOf[JobPathsWithDocker]
+
   def toUnixPath(docker: Boolean)(path: WdlValue): WdlValue = {
     path match {
       case _: WdlFile =>
         val cleanPath = DefaultPathBuilder.build(path.valueString).get
-        WdlFile(if (docker) jobPaths.toDockerPath(cleanPath).toString else cleanPath.toString)
+        WdlFile(if (docker) jobPathsWithDocker.toDockerPath(cleanPath).toString else cleanPath.toString)
       case array: WdlArray => WdlArray(array.wdlType, array.value map toUnixPath(docker))
       case map: WdlMap => WdlMap(map.wdlType, map.value mapValues toUnixPath(docker))
       case wdlValue => wdlValue
@@ -111,12 +110,8 @@ trait SharedFileSystemAsyncJobExecutionActor
 
   def jobName: String = s"cromwell_${jobDescriptor.workflowDescriptor.id.shortString}_${jobDescriptor.call.unqualifiedName}"
 
-  lazy val workflowDescriptor: BackendWorkflowDescriptor = jobDescriptor.workflowDescriptor
-  lazy val call: TaskCall = jobDescriptor.key.call
-  lazy val pathBuilders: List[PathBuilder] = WorkflowPathsBackendInitializationData.pathBuilders(backendInitializationDataOption)
-  private[sfs] lazy val backendEngineFunctions = SharedFileSystemExpressionFunctions(jobPaths, pathBuilders)
-  override lazy val workflowId: WorkflowId = jobDescriptor.workflowDescriptor.id
-  override lazy val jobTag: String = jobDescriptor.key.tag
+  lazy val pathBuilders: List[PathBuilder] = StandardInitializationData.pathBuilders(backendInitializationDataOption)
+  lazy val backendEngineFunctions = SharedFileSystemExpressionFunctions(jobPaths, pathBuilders)
 
   lazy val isDockerRun: Boolean = RuntimeAttributesValidation.extractOption(
     DockerValidation.instance, validatedRuntimeAttributes).isDefined
@@ -124,19 +119,15 @@ trait SharedFileSystemAsyncJobExecutionActor
   override lazy val commandLineFunctions: SharedFileSystemExpressionFunctions = backendEngineFunctions
 
   override lazy val commandLinePreProcessor: (EvaluatedTaskInputs) => Try[EvaluatedTaskInputs] =
-    sharedFileSystem.localizeInputs(jobPaths.callInputsRoot, isDockerRun)
+    sharedFileSystem.localizeInputs(jobPathsWithDocker.callInputsRoot, isDockerRun)
 
   override lazy val commandLineValueMapper: (WdlValue) => WdlValue = toUnixPath(isDockerRun)
 
-  override lazy val startMetadataKeyValues: Map[String, Any] = {
-    super[SharedFileSystemJobCachingActorHelper].startMetadataKeyValues
-  }
-
   override def execute(): ExecutionHandle = {
     val script = instantiatedCommand
     jobLogger.info(s"`$script`")
     File(jobPaths.callExecutionRoot).createDirectories()
-    val cwd = if (isDockerRun) jobPaths.callExecutionDockerRoot else jobPaths.callExecutionRoot
+    val cwd = if (isDockerRun) jobPathsWithDocker.callExecutionDockerRoot else jobPaths.callExecutionRoot
     writeScript(script, cwd, backendEngineFunctions.findGlobOutputs(call, jobDescriptor))
     jobLogger.info(s"command: $processArgs")
     val runner = makeProcessRunner()
@@ -168,7 +159,7 @@ trait SharedFileSystemAsyncJobExecutionActor
     * as some extra shell code for monitoring jobs
     */
   private def writeScript(instantiatedCommand: String, cwd: Path, globFiles: Set[WdlGlobFile]) = {
-    val rcPath = if (isDockerRun) jobPaths.toDockerPath(jobPaths.returnCode) else jobPaths.returnCode
+    val rcPath = if (isDockerRun) jobPathsWithDocker.toDockerPath(jobPaths.returnCode) else jobPaths.returnCode
     val rcTmpPath = pathPlusSuffix(rcPath, "tmp").path
 
     def globManipulation(globFile: WdlGlobFile) = {
@@ -243,16 +234,6 @@ trait SharedFileSystemAsyncJobExecutionActor
     ()
   }
 
-  override def remoteStdErrPath: Path = jobPaths.stderr
-
-  override def remoteReturnCodePath: Path = jobPaths.returnCode
-
-  override def continueOnReturnCode: ContinueOnReturnCode = RuntimeAttributesValidation.extract(
-    ContinueOnReturnCodeValidation.instance, validatedRuntimeAttributes)
-
-  override def failOnStdErr: Boolean = RuntimeAttributesValidation.extract(
-    FailOnStderrValidation.instance, validatedRuntimeAttributes)
-
   override def pollStatus(handle: StandardAsyncPendingExecutionHandle): SharedFileSystemRunStatus = {
     SharedFileSystemRunStatus(File(jobPaths.returnCode).exists)
   }
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemBackendLifecycleActorFactory.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemBackendLifecycleActorFactory.scala
index 43d77dd5d..150f8b199 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemBackendLifecycleActorFactory.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemBackendLifecycleActorFactory.scala
@@ -1,16 +1,7 @@
 package cromwell.backend.sfs
 
-import akka.actor.{ActorRef, Props}
-import cats.data.Validated.{Invalid, Valid}
 import cromwell.backend._
-import cromwell.backend.standard.StandardLifecycleActorFactory
-import cromwell.core.Dispatcher
-import cromwell.core.Dispatcher._
-import cromwell.core.path.{DefaultPathBuilderFactory, PathBuilderFactory}
-import cromwell.filesystems.gcs.{GcsPathBuilderFactory, GoogleConfiguration}
-import lenthall.exception.MessageAggregation
-import net.ceedubs.ficus.Ficus._
-import wdl4s.TaskCall
+import cromwell.backend.standard._
 import wdl4s.expression.WdlStandardLibraryFunctions
 
 /**
@@ -20,50 +11,10 @@ import wdl4s.expression.WdlStandardLibraryFunctions
   */
 trait SharedFileSystemBackendLifecycleActorFactory extends StandardLifecycleActorFactory {
 
-  override def jobIdKey: String = SharedFileSystemJob.JobIdKey
+  override def jobIdKey: String = SharedFileSystemAsyncJobExecutionActor.JobIdKey
 
-  /**
-    * If the backend sets a gcs authentication mode, try to create a PathBuilderFactory with it.
-    */
-  lazy val gcsPathBuilderFactory: Option[GcsPathBuilderFactory] = {
-    configurationDescriptor.backendConfig.as[Option[String]]("filesystems.gcs.auth") map { configAuth =>
-      GoogleConfiguration(configurationDescriptor.globalConfig).auth(configAuth) match {
-        case Valid(auth) => GcsPathBuilderFactory(auth)
-        case Invalid(error) => throw new MessageAggregation {
-          override def exceptionContext: String = "Failed to parse gcs auth configuration"
-
-          override def errorMessages: Traversable[String] = error.toList
-        }
-      }
-    }
-  }
-
-  lazy val pathBuilderFactories: List[PathBuilderFactory] = List(gcsPathBuilderFactory, Option(DefaultPathBuilderFactory)).flatten
-
-  /**
-    * Returns the initialization class, or by default uses the `SharedFileSystemInitializationActor`.
-    *
-    * @return the initialization class.
-    */
-  def initializationActorClass: Class[_ <: SharedFileSystemInitializationActor] =
-    classOf[SharedFileSystemInitializationActor]
-
-  override def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[TaskCall],
-                                                serviceRegistryActor: ActorRef): Option[Props] = {
-    val params = SharedFileSystemInitializationActorParams(serviceRegistryActor, workflowDescriptor,
-      configurationDescriptor, calls, pathBuilderFactories)
-    Option(Props(initializationActorClass, params).withDispatcher(Dispatcher.BackendDispatcher))
-  }
-
-  override def cacheHitCopyingActorProps = Option(cacheHitCopyingActorInner _)
-
-  def cacheHitCopyingActorInner(jobDescriptor: BackendJobDescriptor,
-                                initializationDataOption: Option[BackendInitializationData],
-                                serviceRegistryActor: ActorRef): Props = {
-    Props(
-      new SharedFileSystemCacheHitCopyingActor(
-        jobDescriptor, configurationDescriptor, initializationDataOption, serviceRegistryActor)
-    ).withDispatcher(BackendDispatcher)
+  override lazy val standardCacheHitCopyingActorOption: Option[Class[_ <: StandardCacheHitCopyingActor]] = {
+    Option(classOf[SharedFileSystemCacheHitCopyingActor])
   }
 
   override def expressionLanguageFunctions(workflowDescriptor: BackendWorkflowDescriptor,
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemCacheHitCopyingActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemCacheHitCopyingActor.scala
index 6ba44914d..f6a8ad65c 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemCacheHitCopyingActor.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemCacheHitCopyingActor.scala
@@ -2,27 +2,11 @@ package cromwell.backend.sfs
 
 import java.nio.file.Path
 
-import akka.actor.ActorRef
-import cromwell.backend.callcaching.CacheHitDuplicating
-import cromwell.backend.{BackendCacheHitCopyingActor, BackendConfigurationDescriptor, BackendInitializationData, BackendJobDescriptor}
-import cromwell.core.path.PathFactory
+import cromwell.backend.standard.{StandardCacheHitCopyingActor, StandardCacheHitCopyingActorParams}
 
-import scala.util.Try
-
-class SharedFileSystemCacheHitCopyingActor(override val jobDescriptor: BackendJobDescriptor,
-                                           override val configurationDescriptor: BackendConfigurationDescriptor,
-                                           override val backendInitializationDataOption:
-                                           Option[BackendInitializationData],
-                                           override val serviceRegistryActor: ActorRef)
-  extends SharedFileSystemJobCachingActorHelper with BackendCacheHitCopyingActor with CacheHitDuplicating {
-
-  override lazy val destinationCallRootPath = jobPaths.callRoot
-
-  override lazy val destinationJobDetritusPaths = jobPaths.detritusPaths
-
-  override protected def getPath(file: String) = Try(PathFactory.buildPath(file, jobPaths.pathBuilders))
-
-  override protected def duplicate(source: Path, destination: Path) = {
+class SharedFileSystemCacheHitCopyingActor(override val standardParams: StandardCacheHitCopyingActorParams)
+  extends StandardCacheHitCopyingActor with SharedFileSystemJobCachingActorHelper {
+  override protected def duplicate(source: Path, destination: Path): Unit = {
     // -Ywarn-value-discard
     sharedFileSystem.cacheCopy(source, destination)
     ()
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemExpressionFunctions.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemExpressionFunctions.scala
index 859b98830..0250c7a10 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemExpressionFunctions.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemExpressionFunctions.scala
@@ -3,6 +3,7 @@ package cromwell.backend.sfs
 import java.nio.file.Path
 
 import cromwell.backend.io._
+import cromwell.backend.standard.StandardInitializationData
 import cromwell.backend.wdl._
 import cromwell.backend._
 import cromwell.core.CallContext
@@ -15,7 +16,8 @@ import scala.util.{Success, Try}
 object SharedFileSystemExpressionFunctions {
   private val LocalFileSystemScheme = "file"
 
-  def isLocalPath(path: Path) = path.toUri.getScheme == SharedFileSystemExpressionFunctions.LocalFileSystemScheme
+  def isLocalPath(path: Path): Boolean =
+    path.toUri.getScheme == SharedFileSystemExpressionFunctions.LocalFileSystemScheme
 
   def apply(workflowDescriptor: BackendWorkflowDescriptor,
             jobKey: BackendJobDescriptorKey,
@@ -42,7 +44,7 @@ object SharedFileSystemExpressionFunctions {
   def apply(workflowDescriptor: BackendWorkflowDescriptor,
             configurationDescriptor: BackendConfigurationDescriptor,
             jobKey: BackendJobDescriptorKey,
-            initializationData: Option[BackendInitializationData]) = {
+            initializationData: Option[BackendInitializationData]): SharedFileSystemExpressionFunctions = {
     val jobPaths = new JobPathsWithDocker(jobKey, workflowDescriptor, configurationDescriptor.backendConfig)
     val callContext = CallContext(
       jobPaths.callExecutionRoot,
@@ -50,7 +52,7 @@ object SharedFileSystemExpressionFunctions {
       jobPaths.stderr.toString
     )
 
-    new SharedFileSystemExpressionFunctions(WorkflowPathsBackendInitializationData.pathBuilders(initializationData), callContext)
+    new SharedFileSystemExpressionFunctions(StandardInitializationData.pathBuilders(initializationData), callContext)
   }
 }
 
@@ -63,10 +65,11 @@ class SharedFileSystemExpressionFunctions(override val pathBuilders: List[PathBu
 
   override def writeTempFile(path: String, prefix: String, suffix: String, content: String): String = super[WriteFunctions].writeTempFile(path, prefix, suffix, content)
 
-  override val writeDirectory = context.root
+  override val writeDirectory: Path = context.root
 
   override def stdout(params: Seq[Try[WdlValue]]) = Success(WdlFile(context.stdout))
   override def stderr(params: Seq[Try[WdlValue]]) = Success(WdlFile(context.stderr))
 
-  override def postMapping(path: Path) = if (!path.isAbsolute && isLocalPath(path)) context.root.resolve(path) else path
+  override def postMapping(path: Path): Path =
+    if (!path.isAbsolute && isLocalPath(path)) context.root.resolve(path) else path
 }
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemInitializationActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemInitializationActor.scala
index 52c8fbeaf..6509ee3f6 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemInitializationActor.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemInitializationActor.scala
@@ -1,59 +1,57 @@
 package cromwell.backend.sfs
 
-import akka.actor.ActorRef
 import better.files._
-import cromwell.backend.io.{WorkflowPaths, WorkflowPathsBackendInitializationData}
-import cromwell.backend.validation.RuntimeAttributesDefault
+import cats.data.Validated.{Invalid, Valid}
+import cromwell.backend.BackendInitializationData
+import cromwell.backend.io.WorkflowPaths
+import cromwell.backend.standard.{DefaultInitializationActorParams, StandardInitializationActor, StandardInitializationActorParams, StandardInitializationData}
 import cromwell.backend.wfs.WorkflowPathBuilder
-import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendWorkflowDescriptor, BackendWorkflowInitializationActor}
-import cromwell.core.WorkflowOptions
-import cromwell.core.path.PathBuilderFactory
-import wdl4s.TaskCall
-import wdl4s.values.WdlValue
+import cromwell.core.path.{DefaultPathBuilderFactory, PathBuilder, PathBuilderFactory}
+import cromwell.filesystems.gcs.{GcsPathBuilderFactory, GoogleConfiguration}
+import lenthall.exception.MessageAggregation
+import net.ceedubs.ficus.Ficus._
 
 import scala.concurrent.Future
 import scala.util.Try
 
-case class SharedFileSystemInitializationActorParams
-(
-  serviceRegistryActor: ActorRef,
-  workflowDescriptor: BackendWorkflowDescriptor,
-  configurationDescriptor: BackendConfigurationDescriptor,
-  calls: Set[TaskCall],
-  pathBuilderFactories: List[PathBuilderFactory]
-)
-
-class SharedFileSystemBackendInitializationData
-(
-  val workflowPaths: WorkflowPaths,
-  val runtimeAttributesBuilder: SharedFileSystemValidatedRuntimeAttributesBuilder)
-  extends WorkflowPathsBackendInitializationData
-
 /**
   * Initializes a shared file system actor factory and creates initialization data to pass to the execution actors.
   *
   * @param params Initialization parameters.
   */
-class SharedFileSystemInitializationActor(params: SharedFileSystemInitializationActorParams)
-  extends BackendWorkflowInitializationActor {
-
-  override lazy val workflowDescriptor: BackendWorkflowDescriptor = params.workflowDescriptor
-  override lazy val configurationDescriptor: BackendConfigurationDescriptor = params.configurationDescriptor
-  override lazy val calls: Set[TaskCall] = params.calls
-  override lazy val serviceRegistryActor: ActorRef = params.serviceRegistryActor
+class SharedFileSystemInitializationActor(params: DefaultInitializationActorParams)
+  extends StandardInitializationActor {
 
-  def runtimeAttributesBuilder: SharedFileSystemValidatedRuntimeAttributesBuilder =
-    SharedFileSystemValidatedRuntimeAttributesBuilder.default
+  override val standardParams: StandardInitializationActorParams = params
 
-  override protected def runtimeAttributeValidators: Map[String, (Option[WdlValue]) => Boolean] = {
-    runtimeAttributesBuilder.validations.map(validation =>
-      validation.key -> validation.validateOptionalExpression _
-    ).toMap
+  /**
+    * If the backend sets a gcs authentication mode, try to create a PathBuilderFactory with it.
+    */
+  lazy val gcsPathBuilderFactory: Option[GcsPathBuilderFactory] = {
+    configurationDescriptor.backendConfig.as[Option[String]]("filesystems.gcs.auth") map { configAuth =>
+      GoogleConfiguration(configurationDescriptor.globalConfig).auth(configAuth) match {
+        case Valid(auth) => GcsPathBuilderFactory(auth)
+        case Invalid(error) => throw new MessageAggregation {
+          override def exceptionContext: String = "Failed to parse gcs auth configuration"
+
+          override def errorMessages: Traversable[String] = error.toList
+        }
+      }
+    }
   }
 
-  val pathBuilders = params.pathBuilderFactories map { _.withOptions(workflowDescriptor.workflowOptions)(context.system) }
+  lazy val pathBuilderFactories: List[PathBuilderFactory] =
+    List(gcsPathBuilderFactory, Option(DefaultPathBuilderFactory)).flatten
+
+  lazy val pathBuilders: List[PathBuilder] =
+    pathBuilderFactories map { _.withOptions(workflowDescriptor.workflowOptions)(context.system) }
+
+  val workflowPaths: WorkflowPaths =
+    WorkflowPathBuilder.workflowPaths(configurationDescriptor, workflowDescriptor, pathBuilders)
 
-  val workflowPaths = WorkflowPathBuilder.workflowPaths(configurationDescriptor, workflowDescriptor, pathBuilders)
+  def initializationData: StandardInitializationData = {
+    new StandardInitializationData(workflowPaths, runtimeAttributesBuilder)
+  }
 
   override def beforeAll(): Future[Option[BackendInitializationData]] = {
     Future.fromTry(Try {
@@ -62,31 +60,4 @@ class SharedFileSystemInitializationActor(params: SharedFileSystemInitialization
       Option(initializationData)
     })
   }
-
-  def initializationData: SharedFileSystemBackendInitializationData = {
-    new SharedFileSystemBackendInitializationData(workflowPaths, runtimeAttributesBuilder)
-  }
-
-  /**
-    * Log a warning if there are non-supported runtime attributes defined for the call.
-    */
-  override def validate(): Future[Unit] = {
-    Future.fromTry(Try {
-      calls foreach { call =>
-        val runtimeAttributeKeys = call.task.runtimeAttributes.attrs.keys.toList
-        val notSupportedAttributes = runtimeAttributesBuilder.unsupportedKeys(runtimeAttributeKeys).toList
-
-        if (notSupportedAttributes.nonEmpty) {
-          val notSupportedAttrString = notSupportedAttributes mkString ", "
-          workflowLogger.warn(
-            s"Key/s [$notSupportedAttrString] is/are not supported by backend. " +
-              s"Unsupported attributes will not be part of jobs executions.")
-        }
-      }
-    })
-  }
-
-  override protected def coerceDefaultRuntimeAttributes(options: WorkflowOptions): Try[Map[String, WdlValue]] = {
-    RuntimeAttributesDefault.workflowOptionsDefault(options, runtimeAttributesBuilder.validations.map(v => v.key -> v.coercion).toMap)
-  }
 }
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemJobCachingActorHelper.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemJobCachingActorHelper.scala
index 2c2954cf6..ff78494d3 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemJobCachingActorHelper.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemJobCachingActorHelper.scala
@@ -1,45 +1,18 @@
 package cromwell.backend.sfs
 
-import akka.actor.{Actor, ActorRef}
+import akka.actor.Actor
 import com.typesafe.config.{Config, ConfigFactory}
-import cromwell.backend.BackendInitializationData
-import cromwell.backend.callcaching.JobCachingActorHelper
-import cromwell.backend.io.{JobPathsWithDocker, WorkflowPathsBackendInitializationData}
-import cromwell.backend.validation.{RuntimeAttributesValidation, ValidatedRuntimeAttributes}
+import cromwell.backend.standard.{StandardCachingActorHelper, StandardInitializationData}
 import cromwell.core.logging.JobLogging
 import cromwell.core.path.PathBuilder
 import net.ceedubs.ficus.Ficus._
 
-trait SharedFileSystemJobCachingActorHelper extends JobCachingActorHelper {
+trait SharedFileSystemJobCachingActorHelper extends StandardCachingActorHelper {
   this: Actor with JobLogging =>
 
-  def backendInitializationDataOption: Option[BackendInitializationData]
-
-  def serviceRegistryActor: ActorRef
-
-  lazy val jobPaths =
-    new JobPathsWithDocker(jobDescriptor.key, jobDescriptor.workflowDescriptor, configurationDescriptor.backendConfig)
-
-  lazy val initializationData: SharedFileSystemBackendInitializationData = BackendInitializationData.
-    as[SharedFileSystemBackendInitializationData](backendInitializationDataOption)
-
-  lazy val validatedRuntimeAttributes: ValidatedRuntimeAttributes = {
-    val builder = initializationData.runtimeAttributesBuilder
-    builder.build(jobDescriptor.runtimeAttributes, jobLogger)
-  }
-
-  def startMetadataKeyValues: Map[String, Any] = {
-    val runtimeAttributesMetadata = RuntimeAttributesValidation.extract(validatedRuntimeAttributes) map {
-      case (key, value) => (s"runtimeAttributes:$key", value)
-    }
-    val fileMetadata = jobPaths.metadataPaths
-    val otherMetadata = Map("cache:allowResultReuse" -> true)
-    runtimeAttributesMetadata ++ fileMetadata ++ otherMetadata
-  }
-
   lazy val sharedFileSystem = new SharedFileSystem {
     override val pathBuilders: List[PathBuilder] = {
-      WorkflowPathsBackendInitializationData.pathBuilders(backendInitializationDataOption)
+      StandardInitializationData.pathBuilders(backendInitializationDataOption)
     }
     override lazy val sharedFileSystemConfig: Config = {
       configurationDescriptor.backendConfig.as[Option[Config]]("filesystems.local").getOrElse(ConfigFactory.empty())
diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategySpec.scala b/supportedBackends/sfs/src/test/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategySpec.scala
index bd25dfc51..d9a3d765d 100644
--- a/supportedBackends/sfs/src/test/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategySpec.scala
+++ b/supportedBackends/sfs/src/test/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategySpec.scala
@@ -7,7 +7,7 @@ import better.files._
 import com.typesafe.config.{ConfigFactory, ConfigValueFactory}
 import cromwell.backend.callcaching.FileHashingActor.SingleFileHashRequest
 import cromwell.backend.io.WorkflowPaths
-import cromwell.backend.sfs.SharedFileSystemBackendInitializationData
+import cromwell.backend.standard.StandardInitializationData
 import cromwell.core.path.DefaultPathBuilder
 import org.apache.commons.codec.digest.DigestUtils
 import org.scalatest.prop.TableDrivenPropertyChecks
@@ -49,7 +49,7 @@ class ConfigHashingStrategySpec extends FlatSpec with Matchers with TableDrivenP
     val workflowPaths = mock[WorkflowPaths]
     workflowPaths.pathBuilders returns List(DefaultPathBuilder)
 
-    val initData = mock[SharedFileSystemBackendInitializationData]
+    val initData = mock[StandardInitializationData]
     initData.workflowPaths returns workflowPaths
 
     request.file returns WdlFile(requestFile.pathAsString)
diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemInitializationActorSpec.scala b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemInitializationActorSpec.scala
index 1cbfd2cfe..085ad173d 100644
--- a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemInitializationActorSpec.scala
+++ b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemInitializationActorSpec.scala
@@ -4,6 +4,7 @@ import akka.actor.Props
 import akka.testkit.{EventFilter, ImplicitSender, TestDuration}
 import cromwell.backend.BackendSpec._
 import cromwell.backend.BackendWorkflowInitializationActor.Initialize
+import cromwell.backend.standard.DefaultInitializationActorParams
 import cromwell.backend.{BackendConfigurationDescriptor, BackendWorkflowDescriptor}
 import cromwell.core.TestKitSuite
 import cromwell.core.logging.LoggingTest._
@@ -14,9 +15,9 @@ import scala.concurrent.duration._
 
 class SharedFileSystemInitializationActorSpec extends TestKitSuite("SharedFileSystemInitializationActorSpec")
   with WordSpecLike with Matchers with ImplicitSender {
-  val Timeout = 5.second.dilated
+  val Timeout: FiniteDuration = 5.second.dilated
 
-  val HelloWorld =
+  val HelloWorld: String =
     s"""
       |task hello {
       |  String addressee = "you"
@@ -37,7 +38,7 @@ class SharedFileSystemInitializationActorSpec extends TestKitSuite("SharedFileSy
 
   private def getActorRef(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[TaskCall],
                           conf: BackendConfigurationDescriptor) = {
-    val params = SharedFileSystemInitializationActorParams(emptyActor, workflowDescriptor, conf, calls, List.empty)
+    val params = DefaultInitializationActorParams(workflowDescriptor, calls, emptyActor, conf)
     val props = Props(new SharedFileSystemInitializationActor(params))
     system.actorOf(props, "SharedFileSystemInitializationActor")
   }
@@ -49,7 +50,7 @@ class SharedFileSystemInitializationActorSpec extends TestKitSuite("SharedFileSy
         val conf = emptyBackendConfig
         val backend = getActorRef(workflowDescriptor, workflowDescriptor.workflow.taskCalls, conf)
         val pattern = "Key/s [unsupported] is/are not supported by backend. " +
-          "Unsupported attributes will not be part of jobs executions."
+          "Unsupported attributes will not be part of job executions."
         EventFilter.warning(pattern = escapePattern(pattern), occurrences = 1) intercept {
           backend ! Initialize
         }
diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala
index c5053e416..f5ada20d9 100644
--- a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala
+++ b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala
@@ -10,6 +10,7 @@ import cromwell.backend.BackendLifecycleActor.AbortJobCommand
 import cromwell.backend.io.TestWorkflows._
 import cromwell.backend.io.{JobPathsWithDocker, TestWorkflows}
 import cromwell.backend.sfs.TestLocalAsyncJobExecutionActor._
+import cromwell.backend.standard.StandardValidatedRuntimeAttributesBuilder
 import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptor, BackendJobDescriptorKey, BackendSpec, RuntimeAttributeDefinition}
 import cromwell.core.Tags._
 import cromwell.core._
@@ -29,7 +30,7 @@ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSyst
   behavior of "SharedFileSystemJobExecutionActor"
 
   lazy val runtimeAttributeDefinitions: Set[RuntimeAttributeDefinition] =
-    SharedFileSystemValidatedRuntimeAttributesBuilder.default.definitions.toSet
+    StandardValidatedRuntimeAttributesBuilder.default.definitions.toSet
 
   def executeSpec(docker: Boolean): Any = {
     val expectedOutputs: CallOutputs = Map(
@@ -183,7 +184,7 @@ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSyst
 
     val kvJobKey =
       KvJobKey(jobDescriptor.key.call.fullyQualifiedName, jobDescriptor.key.index, jobDescriptor.key.attempt)
-    val scopedKey = ScopedKey(workflowDescriptor.id, kvJobKey, SharedFileSystemJob.JobIdKey)
+    val scopedKey = ScopedKey(workflowDescriptor.id, kvJobKey, SharedFileSystemAsyncJobExecutionActor.JobIdKey)
     val kvPair = KvPair(scopedKey, Option(pid))
 
     backendRef ! kvPair
diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/TestLocalAsyncJobExecutionActor.scala b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/TestLocalAsyncJobExecutionActor.scala
index 39fc21d6b..2c2b9ab20 100644
--- a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/TestLocalAsyncJobExecutionActor.scala
+++ b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/TestLocalAsyncJobExecutionActor.scala
@@ -2,7 +2,7 @@ package cromwell.backend.sfs
 
 import akka.actor.{ActorSystem, Props}
 import akka.testkit.TestActorRef
-import cromwell.backend.standard.{StandardAsyncExecutionActorParams, StandardSyncExecutionActor, DefaultStandardSyncExecutionActorParams}
+import cromwell.backend.standard._
 import cromwell.backend.io.WorkflowPathsWithDocker
 import cromwell.backend.validation.{DockerValidation, RuntimeAttributesValidation}
 import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptor}
@@ -14,7 +14,7 @@ class TestLocalAsyncJobExecutionActor(override val standardParams: StandardAsync
     if (isDockerRun) {
       val docker = RuntimeAttributesValidation.extract(DockerValidation.instance, validatedRuntimeAttributes)
       val cwd = jobPaths.callRoot.toString
-      val dockerCwd = jobPaths.callDockerRoot.toString
+      val dockerCwd = jobPathsWithDocker.callDockerRoot.toString
       SharedFileSystemCommand("/bin/bash", "-c",
         s"docker run --rm -v $cwd:$dockerCwd -i $docker /bin/bash < $script")
     } else {
@@ -33,12 +33,12 @@ object TestLocalAsyncJobExecutionActor {
                       (implicit system: ActorSystem): TestActorRef[StandardSyncExecutionActor] = {
     val emptyActor = system.actorOf(Props.empty)
     val workflowPaths = new WorkflowPathsWithDocker(jobDescriptor.workflowDescriptor, configurationDescriptor.backendConfig)
-    val initializationData = new SharedFileSystemBackendInitializationData(workflowPaths,
-      SharedFileSystemValidatedRuntimeAttributesBuilder.default.withValidation(DockerValidation.optional))
+    val initializationData = new StandardInitializationData(workflowPaths,
+      StandardValidatedRuntimeAttributesBuilder.default.withValidation(DockerValidation.optional))
     val asyncClass = classOf[TestLocalAsyncJobExecutionActor]
 
-    val params = DefaultStandardSyncExecutionActorParams(SharedFileSystemJob.JobIdKey, emptyActor, jobDescriptor,
-      configurationDescriptor, Option(initializationData), asyncClass)
+    val params = DefaultStandardSyncExecutionActorParams(SharedFileSystemAsyncJobExecutionActor.JobIdKey, emptyActor,
+      jobDescriptor, configurationDescriptor, Option(initializationData), None, asyncClass)
 
     TestActorRef(new StandardSyncExecutionActor(params))
   }

From dc595a4ff7c04f546356f759bea876737df00b62 Mon Sep 17 00:00:00 2001
From: Khalid Shakir <kshakir@broadinstitute.org>
Date: Sun, 8 Jan 2017 03:14:46 -0500
Subject: [PATCH 05/84] Various patches. MetadataValue.apply was throwing an
 NPE exception when passed null, even though it had a call to .getOrElse("").
 Consolidated standard backend runtimeAttributeDefinitions implementation.
 Added a GoogleAuthModeSpec.assumeHasApplicationDefaultCredentials in tests
 that use application default credentials. Refactored away
 JesBackendLifecycleActorFactory's toJes, only used in one place where a
 similar standard method now exists. Refactored away
 JesBackendLifecycleActorFactory's staticRuntimeAttributeDefinitions, only
 used in specs. CromwellServer no longer hard codes the binding timeout.

---
 .../standard/StandardLifecycleActorFactory.scala   |  7 +++
 core/src/main/resources/reference.conf             |  1 +
 .../scala/cromwell/server/CromwellServer.scala     | 30 ++++++------
 .../filesystems/gcs/GcsPathBuilderSpec.scala       |  6 +--
 .../filesystems/gcs/auth/GoogleAuthModeSpec.scala  | 18 ++++++++
 .../cromwell/services/metadata/MetadataQuery.scala | 14 +++---
 .../metadata/impl/MetadataServiceActorSpec.scala   | 54 +++++++++++++++-------
 .../impl/jes/JesBackendLifecycleActorFactory.scala | 23 ++-------
 .../backend/impl/jes/JesRuntimeAttributes.scala    |  5 +-
 .../jes/JesAsyncBackendJobExecutionActorSpec.scala | 13 ++++--
 .../backend/impl/jes/JesCallPathsSpec.scala        |  7 +++
 .../impl/jes/JesInitializationActorSpec.scala      |  4 +-
 .../impl/jes/JesRuntimeAttributesSpec.scala        |  8 +++-
 .../backend/impl/jes/JesWorkflowPathsSpec.scala    |  3 ++
 .../ConfigBackendLifecycleActorFactory.scala       | 11 +----
 15 files changed, 126 insertions(+), 78 deletions(-)
 create mode 100644 filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/auth/GoogleAuthModeSpec.scala

diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardLifecycleActorFactory.scala b/backend/src/main/scala/cromwell/backend/standard/StandardLifecycleActorFactory.scala
index aa7e57356..01cbe6eef 100644
--- a/backend/src/main/scala/cromwell/backend/standard/StandardLifecycleActorFactory.scala
+++ b/backend/src/main/scala/cromwell/backend/standard/StandardLifecycleActorFactory.scala
@@ -144,4 +144,11 @@ trait StandardLifecycleActorFactory extends BackendLifecycleActorFactory {
     initializationData.get.asInstanceOf[StandardInitializationData].workflowPaths.workflowRoot
   }
 
+  override def runtimeAttributeDefinitions(initializationDataOption: Option[BackendInitializationData]):
+  Set[RuntimeAttributeDefinition] = {
+    val initializationData = BackendInitializationData.
+      as[StandardInitializationData](initializationDataOption)
+
+    initializationData.runtimeAttributesBuilder.definitions.toSet
+  }
 }
diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf
index 0e57508b7..60e3903de 100644
--- a/core/src/main/resources/reference.conf
+++ b/core/src/main/resources/reference.conf
@@ -8,6 +8,7 @@
 webservice {
   port = 8000
   interface = 0.0.0.0
+  timeout = 5s
   instance.name = "reference"
 }
 
diff --git a/engine/src/main/scala/cromwell/server/CromwellServer.scala b/engine/src/main/scala/cromwell/server/CromwellServer.scala
index 882150776..ac47e1ef6 100644
--- a/engine/src/main/scala/cromwell/server/CromwellServer.scala
+++ b/engine/src/main/scala/cromwell/server/CromwellServer.scala
@@ -2,8 +2,7 @@ package cromwell.server
 
 import java.util.concurrent.TimeoutException
 
-import akka.actor.Props
-import akka.util.Timeout
+import akka.actor.{ActorContext, ActorSystem, Props}
 import com.typesafe.config.Config
 import cromwell.core.Dispatcher.EngineDispatcher
 import cromwell.webservice.WorkflowJsonSupport._
@@ -11,28 +10,29 @@ import cromwell.webservice.{APIResponse, CromwellApiService, SwaggerService}
 import lenthall.spray.SprayCanHttpService._
 import lenthall.spray.WrappedRoute._
 import net.ceedubs.ficus.Ficus._
-import spray.http.{ContentType, MediaTypes, _}
+import spray.http._
 import spray.json._
+import spray.routing.Route
 
 import scala.concurrent.duration._
-import scala.concurrent.{Await, Future}
+import scala.concurrent.{Await, ExecutionContextExecutor, Future}
 import scala.util.{Failure, Success}
 
 // Note that as per the language specification, this is instantiated lazily and only used when necessary (i.e. server mode)
 object CromwellServer {
-  implicit val timeout = Timeout(5.seconds)
-  import scala.concurrent.ExecutionContext.Implicits.global
-
 
   def run(cromwellSystem: CromwellSystem): Future[Any] = {
-    implicit val actorSystem = cromwellSystem.actorSystem
+    implicit val executionContext = scala.concurrent.ExecutionContext.Implicits.global
+
+    val actorSystem: ActorSystem = cromwellSystem.actorSystem
 
     val service = actorSystem.actorOf(CromwellServerActor.props(cromwellSystem.conf), "cromwell-service")
     val webserviceConf = cromwellSystem.conf.getConfig("webservice")
 
     val interface = webserviceConf.getString("interface")
     val port = webserviceConf.getInt("port")
-    val futureBind = service.bind(interface = interface, port = port)
+    val timeout = webserviceConf.as[FiniteDuration]("timeout")
+    val futureBind = service.bind(interface, port)(implicitly, timeout, actorSystem, implicitly)
     futureBind andThen {
       case Success(_) =>
         actorSystem.log.info("Cromwell service started...")
@@ -52,16 +52,18 @@ object CromwellServer {
 }
 
 class CromwellServerActor(config: Config) extends CromwellRootActor with CromwellApiService with SwaggerService {
-  implicit def executionContext = actorRefFactory.dispatcher
+  implicit def executionContext: ExecutionContextExecutor = actorRefFactory.dispatcher
 
   override val serverMode = true
   override val abortJobsOnTerminate = false
 
-  override def actorRefFactory = context
-  override def receive = handleTimeouts orElse runRoute(possibleRoutes)
+  override def actorRefFactory: ActorContext = context
+  override def receive: PartialFunction[Any, Unit] = handleTimeouts orElse runRoute(possibleRoutes)
 
-  val possibleRoutes = workflowRoutes.wrapped("api", config.as[Option[Boolean]]("api.routeUnwrapped").getOrElse(false)) ~ swaggerUiResourceRoute
-  val timeoutError = APIResponse.error(new TimeoutException("The server was not able to produce a timely response to your request.")).toJson.prettyPrint
+  val routeUnwrapped: Boolean = config.as[Option[Boolean]]("api.routeUnwrapped").getOrElse(false)
+  val possibleRoutes: Route = workflowRoutes.wrapped("api", routeUnwrapped) ~ swaggerUiResourceRoute
+  val timeoutError: String = APIResponse.error(new TimeoutException(
+    "The server was not able to produce a timely response to your request.")).toJson.prettyPrint
 
   def handleTimeouts: Receive = {
     case Timedout(_: HttpRequest) =>
diff --git a/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GcsPathBuilderSpec.scala b/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GcsPathBuilderSpec.scala
index 598cb5461..3ce135e2f 100644
--- a/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GcsPathBuilderSpec.scala
+++ b/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GcsPathBuilderSpec.scala
@@ -5,16 +5,16 @@ import com.google.cloud.storage.contrib.nio.CloudStorageConfiguration
 import cromwell.core.path.CustomRetryParams
 import cromwell.core.path.proxy.RetryableFileSystemProviderProxy
 import cromwell.core.{TestKitSuite, WorkflowOptions}
-import cromwell.filesystems.gcs.auth.GoogleAuthMode
+import cromwell.filesystems.gcs.auth.{GoogleAuthMode, GoogleAuthModeSpec}
 import org.scalatest.{FlatSpecLike, Matchers}
 
 class GcsPathBuilderSpec extends TestKitSuite with FlatSpecLike with Matchers {
 
-  implicit val as = system
-
   behavior of "GcsPathBuilderSpec"
 
   it should "create a path with a retryable provider" in {
+    GoogleAuthModeSpec.assumeHasApplicationDefaultCredentials()
+
     val retryablePathBuilder = new RetryableGcsPathBuilder(
       GoogleAuthMode.NoAuthMode,
       RetryParams.defaultInstance(),
diff --git a/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/auth/GoogleAuthModeSpec.scala b/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/auth/GoogleAuthModeSpec.scala
new file mode 100644
index 000000000..327f98fe7
--- /dev/null
+++ b/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/auth/GoogleAuthModeSpec.scala
@@ -0,0 +1,18 @@
+package cromwell.filesystems.gcs.auth
+
+import cromwell.core.WorkflowOptions
+import org.scalatest.Assertions._
+
+object GoogleAuthModeSpec {
+  def assumeHasApplicationDefaultCredentials(): Unit = {
+    try {
+      val authMode = ApplicationDefaultMode("application-default")
+      val workflowOptions = WorkflowOptions.empty
+      authMode.authCredentials(workflowOptions)
+      authMode.credential(workflowOptions)
+      ()
+    } catch {
+      case exception: Exception => cancel(exception.getMessage)
+    }
+  }
+}
diff --git a/services/src/main/scala/cromwell/services/metadata/MetadataQuery.scala b/services/src/main/scala/cromwell/services/metadata/MetadataQuery.scala
index f17118aa7..9015d59f0 100644
--- a/services/src/main/scala/cromwell/services/metadata/MetadataQuery.scala
+++ b/services/src/main/scala/cromwell/services/metadata/MetadataQuery.scala
@@ -6,7 +6,7 @@ import java.time.OffsetDateTime
 import cats.data.NonEmptyList
 import cromwell.core.WorkflowId
 import cromwell.core.path.PathImplicits._
-import org.slf4j.LoggerFactory
+import org.slf4j.{Logger, LoggerFactory}
 import wdl4s.values.{WdlBoolean, WdlFloat, WdlInteger, WdlOptionalValue, WdlValue}
 
 case class MetadataJobKey(callFqn: String, index: Option[Int], attempt: Int)
@@ -21,7 +21,7 @@ object MetadataKey {
     new MetadataKey(workflowId, jobKey, compositeKey(keys:_*))
   }
 
-  def compositeKey(keys: String*) = keys.toList.mkString(KeySeparator.toString)
+  def compositeKey(keys: String*): String = keys.toList.mkString(KeySeparator.toString)
 }
 
 object MetadataEvent {
@@ -48,15 +48,15 @@ object MetadataValue {
       case _: Double | Float => new MetadataValue(value.toString, MetadataNumber)
       case _: Boolean => new MetadataValue(value.toString, MetadataBoolean)
       case path: Path => new MetadataValue(path.toRealString, MetadataString)
-      case _ => new MetadataValue(value.toString, MetadataString)
+      case other => new MetadataValue(other.toString, MetadataString)
     }
   }
 }
 
 object MetadataType {
-  val log = LoggerFactory.getLogger("Metadata Type")
+  val log: Logger = LoggerFactory.getLogger("Metadata Type")
 
-  def fromString(s: String) = s match {
+  def fromString(s: String): MetadataType = s match {
     case MetadataString.typeName => MetadataString
     case MetadataInt.typeName => MetadataInt
     case MetadataNumber.typeName => MetadataNumber
@@ -85,11 +85,11 @@ case class MetadataQuery(workflowId: WorkflowId, jobKey: Option[MetadataQueryJob
 object MetadataQuery {
   def forWorkflow(workflowId: WorkflowId) = MetadataQuery(workflowId, None, None, None, None, expandSubWorkflows = false)
 
-  def forJob(workflowId: WorkflowId, jobKey: MetadataJobKey) = {
+  def forJob(workflowId: WorkflowId, jobKey: MetadataJobKey): MetadataQuery = {
     MetadataQuery(workflowId, Option(MetadataQueryJobKey.forMetadataJobKey(jobKey)), None, None, None, expandSubWorkflows = false)
   }
 
-  def forKey(key: MetadataKey) = {
+  def forKey(key: MetadataKey): MetadataQuery = {
     MetadataQuery(key.workflowId, key.jobKey map MetadataQueryJobKey.forMetadataJobKey, Option(key.key), None, None, expandSubWorkflows = false)
   }
 }
diff --git a/services/src/test/scala/cromwell/services/metadata/impl/MetadataServiceActorSpec.scala b/services/src/test/scala/cromwell/services/metadata/impl/MetadataServiceActorSpec.scala
index 72e6b370a..12d201269 100644
--- a/services/src/test/scala/cromwell/services/metadata/impl/MetadataServiceActorSpec.scala
+++ b/services/src/test/scala/cromwell/services/metadata/impl/MetadataServiceActorSpec.scala
@@ -10,29 +10,29 @@ import cromwell.services.metadata.MetadataService._
 import cromwell.services.metadata._
 
 class MetadataServiceActorSpec extends ServicesSpec("Metadata") {
+  "MetadataServiceActor" should {
 
-  val config = ConfigFactory.empty()
-  val actor = system.actorOf(MetadataServiceActor.props(config, config))
+    val config = ConfigFactory.empty()
+    val actor = system.actorOf(MetadataServiceActor.props(config, config))
 
-  val workflowId = WorkflowId.randomId()
+    val workflowId = WorkflowId.randomId()
 
-  /*
-  Simple store / retrieve
-   */
+    /*
+    Simple store / retrieve
+     */
 
-  val key1 = MetadataKey(workflowId, None, "key1")
-  val key2 = MetadataKey(workflowId, None, "key2")
-  val supJob = MetadataJobKey("sup.sup", None, 1)
-  val key3 = MetadataKey(workflowId, Option(supJob), "dog")
-  val moment = OffsetDateTime.now
+    val key1 = MetadataKey(workflowId, None, "key1")
+    val key2 = MetadataKey(workflowId, None, "key2")
+    val supJob = MetadataJobKey("sup.sup", None, 1)
+    val key3 = MetadataKey(workflowId, Option(supJob), "dog")
+    val moment = OffsetDateTime.now
 
-  val event1_1 = MetadataEvent(key1, Option(MetadataValue("value1")), moment)
-  val event1_2 = MetadataEvent(key1, Option(MetadataValue("value2")), moment)
-  val event2_1 = MetadataEvent(key2, Option(MetadataValue("value1")), moment)
-  val event3_1 = MetadataEvent(key3, Option(MetadataValue("value3")), moment)
-  val event3_2 = MetadataEvent(key3, None, moment)
+    val event1_1 = MetadataEvent(key1, Option(MetadataValue("value1")), moment)
+    val event1_2 = MetadataEvent(key1, Option(MetadataValue("value2")), moment)
+    val event2_1 = MetadataEvent(key2, Option(MetadataValue("value1")), moment)
+    val event3_1 = MetadataEvent(key3, Option(MetadataValue("value3")), moment)
+    val event3_2 = MetadataEvent(key3, None, moment)
 
-  "MetadataServiceActor" should {
     "Store values for different keys" in {
       val putAction1 = PutMetadataAction(event1_1)
       val putAction2 = PutMetadataAction(event1_2)
@@ -72,5 +72,25 @@ class MetadataServiceActorSpec extends ServicesSpec("Metadata") {
 
       } yield ()).futureValue
     }
+
+    "store and retrieve null values" in {
+      val metadataWorkflowId = WorkflowId.randomId()
+      val metadataKey = MetadataKey(metadataWorkflowId, None, "nullTestKey")
+      val metadataValue = MetadataValue(null)
+      val metadataEvent = MetadataEvent(metadataKey, Option(metadataValue), moment)
+
+      val putMetadataAction = PutMetadataAction(metadataEvent)
+
+      val metadataQuery = MetadataQuery.forKey(metadataKey)
+
+      (for {
+        putResponse <- (actor ? putMetadataAction).mapTo[MetadataServiceResponse]
+        _ = putResponse shouldBe MetadataPutAcknowledgement(putMetadataAction)
+
+        queryResponse <- (actor ? GetMetadataQueryAction(metadataQuery)).mapTo[MetadataServiceResponse]
+        _ = queryResponse shouldBe MetadataLookupResponse(metadataQuery, Seq(metadataEvent))
+      } yield ()).futureValue
+    }
+
   }
 }
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala
index 00b5bfc96..7026c8cc2 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala
@@ -11,7 +11,6 @@ import wdl4s.expression.WdlStandardLibraryFunctions
 
 case class JesBackendLifecycleActorFactory(name: String, configurationDescriptor: BackendConfigurationDescriptor)
   extends StandardLifecycleActorFactory {
-  import JesBackendLifecycleActorFactory._
 
   override def initializationActorClass: Class[_ <: StandardInitializationActor] = classOf[JesInitializationActor]
 
@@ -45,14 +44,14 @@ case class JesBackendLifecycleActorFactory(name: String, configurationDescriptor
       initializationDataOption)
   }
 
-  override def runtimeAttributeDefinitions(initializationDataOption: Option[BackendInitializationData]):
-  Set[RuntimeAttributeDefinition] = staticRuntimeAttributeDefinitions
-
   override def expressionLanguageFunctions(workflowDescriptor: BackendWorkflowDescriptor,
                                            jobKey: BackendJobDescriptorKey,
-                                           initializationData: Option[BackendInitializationData]): WdlStandardLibraryFunctions = {
+                                           initializationDataOption: Option[BackendInitializationData]
+                                          ): WdlStandardLibraryFunctions = {
+    val initializationData = BackendInitializationData.
+      as[JesBackendInitializationData](initializationDataOption)
 
-    val jesCallPaths = initializationData.toJes.get.workflowPaths.toJobPaths(jobKey, workflowDescriptor)
+    val jesCallPaths = initializationData.workflowPaths.toJobPaths(jobKey, workflowDescriptor)
     new JesExpressionFunctions(List(jesCallPaths.gcsPathBuilder), jesCallPaths.callContext)
   }
 
@@ -60,15 +59,3 @@ case class JesBackendLifecycleActorFactory(name: String, configurationDescriptor
 
   override lazy val fileHashingFunction: Option[FileHashingFunction] = Option(FileHashingFunction(JesBackendFileHashing.getCrc32c))
 }
-
-object JesBackendLifecycleActorFactory {
-  implicit class Jessify(val genericInitializationData: Option[BackendInitializationData]) {
-    // This leaves the result in an `Option` as finalization will be called even if initialization has failed, and if
-    // initialization fails there won't be any initialization data.  The various `.get`s that occur below are in instances
-    // where the workflow has successfully gotten past initialization and the JES initialization data is defined.
-    def toJes: Option[JesBackendInitializationData] = genericInitializationData collectFirst { case d: JesBackendInitializationData => d }
-  }
-
-  val staticRuntimeAttributeDefinitions: Set[RuntimeAttributeDefinition] =
-    JesRuntimeAttributes.runtimeAttributesBuilder.definitions.toSet
-}
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesRuntimeAttributes.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesRuntimeAttributes.scala
index a878974a8..dbd3bd392 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesRuntimeAttributes.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesRuntimeAttributes.scala
@@ -86,8 +86,9 @@ object JesRuntimeAttributes {
     val memory: MemorySize = RuntimeAttributesValidation.extract(memoryValidation, validatedRuntimeAttributes)
     val disks: Seq[JesAttachedDisk] = RuntimeAttributesValidation.extract(disksValidation, validatedRuntimeAttributes)
     val docker: String = RuntimeAttributesValidation.extract(dockerValidation, validatedRuntimeAttributes)
-    val failOnStderr = RuntimeAttributesValidation.extract(FailOnStderrValidation.default, validatedRuntimeAttributes)
-    val continueOnReturnCode =
+    val failOnStderr: Boolean =
+      RuntimeAttributesValidation.extract(FailOnStderrValidation.default, validatedRuntimeAttributes)
+    val continueOnReturnCode: ContinueOnReturnCode =
       RuntimeAttributesValidation.extract(ContinueOnReturnCodeValidation.default, validatedRuntimeAttributes)
     val noAddress: Boolean = RuntimeAttributesValidation.extract(noAddressValidation, validatedRuntimeAttributes)
 
diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala
index 40fc7f44c..f331f73a8 100644
--- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala
+++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala
@@ -19,6 +19,7 @@ import cromwell.core._
 import cromwell.core.logging.JobLogger
 import cromwell.core.path.PathImplicits._
 import cromwell.filesystems.gcs.auth.GoogleAuthMode.NoAuthMode
+import cromwell.filesystems.gcs.auth.GoogleAuthModeSpec
 import cromwell.filesystems.gcs.{GcsPathBuilder, GcsPathBuilderFactory}
 import cromwell.util.SampleWdl
 import org.scalatest._
@@ -37,7 +38,10 @@ import scala.util.{Success, Try}
 class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackendJobExecutionActorSpec")
   with FlatSpecLike with Matchers with ImplicitSender with Mockito with BackendSpec {
 
-  val mockPathBuilder: GcsPathBuilder = GcsPathBuilderFactory(NoAuthMode).withOptions(mock[WorkflowOptions])
+  lazy val mockPathBuilder: GcsPathBuilder = {
+    GoogleAuthModeSpec.assumeHasApplicationDefaultCredentials()
+    GcsPathBuilderFactory(NoAuthMode).withOptions(mock[WorkflowOptions])
+  }
 
   import JesTestConfig._
 
@@ -68,9 +72,9 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
 
   val NoOptions = WorkflowOptions(JsObject(Map.empty[String, JsValue]))
 
-  val TestableCallContext = CallContext(mockPathBuilder.build("gs://root").get, "out", "err")
+  lazy val TestableCallContext = CallContext(mockPathBuilder.build("gs://root").get, "out", "err")
 
-  val TestableJesExpressionFunctions: JesExpressionFunctions = {
+  lazy val TestableJesExpressionFunctions: JesExpressionFunctions = {
     new JesExpressionFunctions(List(mockPathBuilder), TestableCallContext)
   }
 
@@ -676,6 +680,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
 
   private def makeRuntimeAttributes(job: TaskCall) = {
     val evaluatedAttributes = RuntimeAttributeDefinition.evaluateRuntimeAttributes(job.task.runtimeAttributes, TestableJesExpressionFunctions, Map.empty)
-    RuntimeAttributeDefinition.addDefaultsToAttributes(JesBackendLifecycleActorFactory.staticRuntimeAttributeDefinitions, NoOptions)(evaluatedAttributes.get) // Fine to throw the exception if this "get" fails. This is a test after all!
+    RuntimeAttributeDefinition.addDefaultsToAttributes(
+      JesRuntimeAttributes.runtimeAttributesBuilder.definitions.toSet, NoOptions)(evaluatedAttributes.get)
   }
 }
diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesCallPathsSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesCallPathsSpec.scala
index 7682ffb7b..1abf3f22f 100644
--- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesCallPathsSpec.scala
+++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesCallPathsSpec.scala
@@ -3,6 +3,7 @@ package cromwell.backend.impl.jes
 import cromwell.backend.BackendSpec
 import cromwell.core.TestKitSuite
 import cromwell.core.path.PathImplicits._
+import cromwell.filesystems.gcs.auth.GoogleAuthModeSpec
 import cromwell.util.SampleWdl
 import org.scalatest.{FlatSpecLike, Matchers}
 import org.specs2.mock.Mockito
@@ -15,6 +16,8 @@ class JesCallPathsSpec extends TestKitSuite with FlatSpecLike with Matchers with
   behavior of "JesCallPaths"
 
   it should "map the correct filenames" in {
+    GoogleAuthModeSpec.assumeHasApplicationDefaultCredentials()
+
     val workflowDescriptor = buildWorkflowDescriptor(SampleWdl.HelloWorld.wdlSource())
     val jobDescriptorKey = firstJobDescriptorKey(workflowDescriptor)
     val jesConfiguration = new JesConfiguration(JesBackendConfigurationDescriptor)
@@ -28,6 +31,8 @@ class JesCallPathsSpec extends TestKitSuite with FlatSpecLike with Matchers with
   }
 
   it should "map the correct paths" in {
+    GoogleAuthModeSpec.assumeHasApplicationDefaultCredentials()
+
     val workflowDescriptor = buildWorkflowDescriptor(SampleWdl.HelloWorld.wdlSource())
     val jobDescriptorKey = firstJobDescriptorKey(workflowDescriptor)
     val jesConfiguration = new JesConfiguration(JesBackendConfigurationDescriptor)
@@ -44,6 +49,8 @@ class JesCallPathsSpec extends TestKitSuite with FlatSpecLike with Matchers with
   }
 
   it should "map the correct call context" in {
+    GoogleAuthModeSpec.assumeHasApplicationDefaultCredentials()
+
     val workflowDescriptor = buildWorkflowDescriptor(SampleWdl.HelloWorld.wdlSource())
     val jobDescriptorKey = firstJobDescriptorKey(workflowDescriptor)
     val jesConfiguration = new JesConfiguration(JesBackendConfigurationDescriptor)
diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala
index 44688edd8..844fe73d8 100644
--- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala
+++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala
@@ -11,7 +11,7 @@ import cromwell.core.Tags.IntegrationTest
 import cromwell.core.logging.LoggingTest._
 import cromwell.core.{TestKitSuite, WorkflowOptions}
 import cromwell.filesystems.gcs.GoogleConfiguration
-import cromwell.filesystems.gcs.auth.{RefreshTokenMode, SimpleClientSecrets}
+import cromwell.filesystems.gcs.auth.{GoogleAuthModeSpec, RefreshTokenMode, SimpleClientSecrets}
 import cromwell.util.{EncryptionSpec, SampleWdl}
 import org.scalatest.{FlatSpecLike, Matchers}
 import org.specs2.mock.Mockito
@@ -145,6 +145,8 @@ class JesInitializationActorSpec extends TestKitSuite("JesInitializationActorSpe
   behavior of "JesInitializationActor"
 
   it should "log a warning message when there are unsupported runtime attributes" taggedAs IntegrationTest in {
+    GoogleAuthModeSpec.assumeHasApplicationDefaultCredentials()
+
     within(Timeout) {
       val workflowDescriptor = buildWorkflowDescriptor(HelloWorld,
         runtime = """runtime { docker: "ubuntu/latest" test: true }""")
diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesRuntimeAttributesSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesRuntimeAttributesSpec.scala
index 43b62287c..d8ddd68e2 100644
--- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesRuntimeAttributesSpec.scala
+++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesRuntimeAttributesSpec.scala
@@ -190,7 +190,8 @@ class JesRuntimeAttributesSpec extends WordSpecLike with Matchers with Mockito {
   }
 
   private def assertJesRuntimeAttributesSuccessfulCreation(runtimeAttributes: Map[String, WdlValue], expectedRuntimeAttributes: JesRuntimeAttributes, workflowOptions: WorkflowOptions = emptyWorkflowOptions): Unit = {
-    val withDefaults = RuntimeAttributeDefinition.addDefaultsToAttributes(JesBackendLifecycleActorFactory.staticRuntimeAttributeDefinitions, workflowOptions) _
+    val withDefaults = RuntimeAttributeDefinition.addDefaultsToAttributes(
+      staticRuntimeAttributeDefinitions, workflowOptions) _
     try {
       assert(JesRuntimeAttributes(withDefaults(runtimeAttributes), NOPLogger.NOP_LOGGER) == expectedRuntimeAttributes)
     } catch {
@@ -200,7 +201,8 @@ class JesRuntimeAttributesSpec extends WordSpecLike with Matchers with Mockito {
   }
 
   private def assertJesRuntimeAttributesFailedCreation(runtimeAttributes: Map[String, WdlValue], exMsg: String, workflowOptions: WorkflowOptions = emptyWorkflowOptions): Unit = {
-    val withDefaults = RuntimeAttributeDefinition.addDefaultsToAttributes(JesBackendLifecycleActorFactory.staticRuntimeAttributeDefinitions, workflowOptions) _
+    val withDefaults = RuntimeAttributeDefinition.addDefaultsToAttributes(
+      staticRuntimeAttributeDefinitions, workflowOptions) _
     try {
       JesRuntimeAttributes(withDefaults(runtimeAttributes), NOPLogger.NOP_LOGGER)
       fail("A RuntimeException was expected.")
@@ -211,4 +213,6 @@ class JesRuntimeAttributesSpec extends WordSpecLike with Matchers with Mockito {
   }
 
   private val emptyWorkflowOptions = WorkflowOptions.fromMap(Map.empty).get
+  private val staticRuntimeAttributeDefinitions: Set[RuntimeAttributeDefinition] =
+    JesRuntimeAttributes.runtimeAttributesBuilder.definitions.toSet
 }
diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesWorkflowPathsSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesWorkflowPathsSpec.scala
index 9826b4ef0..2b3794356 100644
--- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesWorkflowPathsSpec.scala
+++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesWorkflowPathsSpec.scala
@@ -3,6 +3,7 @@ package cromwell.backend.impl.jes
 import cromwell.backend.BackendSpec
 import cromwell.core.TestKitSuite
 import cromwell.core.path.PathImplicits._
+import cromwell.filesystems.gcs.auth.GoogleAuthModeSpec
 import cromwell.util.SampleWdl
 import org.scalatest.{FlatSpecLike, Matchers}
 import org.specs2.mock.Mockito
@@ -14,6 +15,8 @@ class JesWorkflowPathsSpec extends TestKitSuite with FlatSpecLike with Matchers
   behavior of "JesWorkflowPaths"
 
   it should "map the correct paths" in {
+    GoogleAuthModeSpec.assumeHasApplicationDefaultCredentials()
+
     val workflowDescriptor = buildWorkflowDescriptor(SampleWdl.HelloWorld.wdlSource())
     val jesConfiguration = new JesConfiguration(JesBackendConfigurationDescriptor)
 
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendLifecycleActorFactory.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendLifecycleActorFactory.scala
index 5296020c6..78060285d 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendLifecycleActorFactory.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendLifecycleActorFactory.scala
@@ -1,11 +1,10 @@
 package cromwell.backend.impl.sfs.config
 
 import com.typesafe.config.Config
+import cromwell.backend.BackendConfigurationDescriptor
 import cromwell.backend.callcaching.FileHashingActor.FileHashingFunction
 import cromwell.backend.impl.sfs.config.ConfigConstants._
 import cromwell.backend.sfs._
-import cromwell.backend.standard.StandardInitializationData
-import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, RuntimeAttributeDefinition}
 import cromwell.core.JobExecutionToken.JobExecutionTokenType
 import net.ceedubs.ficus.Ficus._
 import org.slf4j.{Logger, LoggerFactory}
@@ -33,14 +32,6 @@ class ConfigBackendLifecycleActorFactory(name: String, val configurationDescript
       classOf[DispatchedConfigAsyncJobExecutionActor]
   }
 
-  override def runtimeAttributeDefinitions(initializationDataOption: Option[BackendInitializationData]):
-  Set[RuntimeAttributeDefinition] = {
-    val initializationData = BackendInitializationData.
-      as[StandardInitializationData](initializationDataOption)
-
-    initializationData.runtimeAttributesBuilder.definitions.toSet
-  }
-
   override lazy val fileHashingFunction: Option[FileHashingFunction] = {
     logger.debug(hashingStrategy.toString)
     Option(FileHashingFunction(hashingStrategy.getHash))

From 24688163c61e5a56794c85841c7beaa37d3e379d Mon Sep 17 00:00:00 2001
From: Chris Llanwarne <cjllanwarne@gmail.com>
Date: Wed, 4 Jan 2017 17:53:39 -0500
Subject: [PATCH 06/84] Cromwell workflow ID as a JES label

---
 CHANGELOG.md                                       |  2 +-
 README.md                                          | 24 ++++++++++
 project/Dependencies.scala                         |  4 +-
 .../main/scala/cromwell/backend/impl/jes/Run.scala | 23 +++++++++
 .../cromwell/backend/impl/jes/labels/Label.scala   | 54 ++++++++++++++++++++++
 .../cromwell/backend/impl/jes/labels/Labels.scala  | 17 +++++++
 .../backend/impl/jes/labels/LabelSpec.scala        | 45 ++++++++++++++++++
 7 files changed, 166 insertions(+), 3 deletions(-)
 create mode 100644 supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/labels/Label.scala
 create mode 100644 supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/labels/Labels.scala
 create mode 100644 supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/labels/LabelSpec.scala

diff --git a/CHANGELOG.md b/CHANGELOG.md
index a12ac4c8c..acfd1f07a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,7 +2,7 @@
 
 ## 25
 
-
+* Cromwell now applies default labels automatically to JES pipeline runs.
 
 ## 24
 
diff --git a/README.md b/README.md
index 49c09df42..1d8d30a05 100644
--- a/README.md
+++ b/README.md
@@ -53,6 +53,8 @@ A [Workflow Management System](https://en.wikipedia.org/wiki/Workflow_management
       * [Refresh Token](#refresh-token)
     * [Docker](#docker)
     * [Monitoring](#monitoring)
+    * [Labelling Runs](#labelling-runs)
+      * [Label Escaping and Padding](#label-escaping-and-padding)
 * [Runtime Attributes](#runtime-attributes)
   * [Specifying Default Values](#specifying-default-values)
   * [continueOnReturnCode](#continueonreturncode)
@@ -1383,6 +1385,28 @@ In order to monitor metrics (CPU, Memory, Disk usage...) about the VM during Cal
 
 The output of this script will be written to a `monitoring.log` file that will be available in the call gcs bucket when the call completes.  This feature is meant to run a script in the background during long-running processes.  It's possible that if the task is very short that the log file does not flush before de-localization happens and you will end up with a zero byte file.
 
+### Labelling Runs
+
+Every call to JES from a Cromwell instance will be automatically labelled by Cromwell so that it can be queried about later. The current label set automatically applied is:
+
+| Key | Value | Example | Notes |
+|-----|-------|---------|-------|
+| cromwell-workflow-id | The Cromwell ID given to the root workflow (i.e. the ID returned by Cromwell on submission) | cromwell-d4b412c5-bf3d-4169-91b0-1b635ce47a26 | To fit the required [format](#label-escaping-and-padding), we prefix with 'cromwell-' |
+| cromwell-workflow-name | The name of the root workflow | my-root-workflow | See [format](#label-escaping-and-padding). |
+| cromwell-sub-workflow-name | The name of this job's sub-workflow | my-sub-workflow | Only if the task is called in a subworkflow, otherwise 'n-a'. See also [format](#label-escaping-and-padding). |
+| wdl-task-name | The name of the WDL task | my-task | See [format](#label-escaping-and-padding). |
+| wdl-call-name | The name of the WDL call of this job | my-call | Different from 'wdl-task-name' if it was called with an alias. See also [format](#label-escaping-and-padding). |
+
+#### Label Escaping and Padding
+
+To fit in with the Google schema for labels, label key and value strings must match the regex `[a-z]([-a-z0-9]*[a-z0-9])?` and be between 1 and 63 characters in length. For this reason, Cromwell will modify workflow/task/call names and custom labels as follows:
+
+- Any capital letters are lowercased.
+- Any character which is not one of `[a-z]`, `[0-9]` or `-` will be replaced with `-`.
+- If the start character does not match `[a-z]` then prefix with `x--`
+- If the final character does not match `[a-z0-9]` then suffix with `--x`
+- If the string is too long, only take the first 30 and last 30 characters and add `---` between them.
+
 # Runtime Attributes
 
 Runtime attributes are used to customize tasks. Within a task one can specify runtime attributes to customize the environment for the call.
diff --git a/project/Dependencies.scala b/project/Dependencies.scala
index b06413d9d..f683b492b 100644
--- a/project/Dependencies.scala
+++ b/project/Dependencies.scala
@@ -15,7 +15,7 @@ object Dependencies {
   lazy val akkaV = "2.4.14"
   lazy val slickV = "3.1.1"
   lazy val googleClientApiV = "1.22.0"
-  lazy val googleGenomicsServicesApiV = "1.20.0"
+  lazy val googleGenomicsServicesApiV = "1.22.0"
   lazy val betterFilesV = "2.16.0"
   lazy val catsV = "0.7.2"
 
@@ -80,7 +80,7 @@ object Dependencies {
   )
 
   private val googleCloudDependencies = List(
-    "com.google.apis" % "google-api-services-genomics" % ("v1alpha2-rev14-" + googleGenomicsServicesApiV),
+    "com.google.apis" % "google-api-services-genomics" % ("v1alpha2-rev64-" + googleGenomicsServicesApiV),
     "com.google.cloud" % "google-cloud-nio" % "0.3.0"
       exclude("com.google.api.grpc", "grpc-google-common-protos")
       exclude("com.google.cloud.datastore", "datastore-v1-protos")
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala
index 05078c7f2..290cb8f4e 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala
@@ -8,6 +8,7 @@ import com.google.api.services.genomics.Genomics
 import com.google.api.services.genomics.model._
 import cromwell.backend.BackendJobDescriptor
 import cromwell.backend.impl.jes.RunStatus.{Failed, Initializing, Running, Success}
+import cromwell.backend.impl.jes.labels.Labels
 import cromwell.core.ExecutionEvent
 import cromwell.core.logging.JobLogger
 import org.slf4j.LoggerFactory
@@ -59,6 +60,26 @@ object Run {
       resources.setDisks(disksWithoutMountPoint.asJava)
     }
 
+    lazy val labels: Labels = {
+
+      val hasSubworkflow = workflow.workflow.equals(workflow.rootWorkflow)
+
+      val subWorkflowName = if (hasSubworkflow) workflow.workflow.unqualifiedName else "n-a"
+      val subWorkflowLabels = Labels("cromwell-sub-workflow-name" -> subWorkflowName)
+
+      val rootWorkflowLabels = Labels(
+        "cromwell-workflow-id" -> s"cromwell-${workflow.rootWorkflowId}",
+        "cromwell-workflow-name" -> workflow.rootWorkflow.unqualifiedName
+      )
+
+      val callLabels = Labels(
+        "wdl-call-name" -> jobDescriptor.call.unqualifiedName,
+        "wdl-task-name" -> jobDescriptor.call.task.name
+      )
+
+      rootWorkflowLabels ++ subWorkflowLabels ++ callLabels
+    }
+
     def runPipeline: String = {
       val svcAccount = new ServiceAccount().setEmail(computeServiceAccount).setScopes(GenomicsScopes)
       val rpargs = new RunPipelineArgs().setProjectId(projectId).setServiceAccount(svcAccount).setResources(runtimePipelineResources)
@@ -69,6 +90,8 @@ object Run {
       rpargs.setOutputs(jesParameters.collect({ case i: JesFileOutput => i.name -> i.toGoogleRunParameter }).toMap.asJava)
       logger.debug(s"Outputs:\n${stringifyMap(rpargs.getOutputs.asScala.toMap)}")
 
+      rpargs.setLabels(labels.asJesLabels)
+
       val rpr = new RunPipelineRequest().setEphemeralPipeline(pipeline).setPipelineArgs(rpargs)
 
       val logging = new LoggingOptions()
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/labels/Label.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/labels/Label.scala
new file mode 100644
index 000000000..0e7e83101
--- /dev/null
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/labels/Label.scala
@@ -0,0 +1,54 @@
+package cromwell.backend.impl.jes.labels
+
+case class Label private[labels](key: String, value: String)
+
+object Label {
+
+  // Yes, 63. Not a typo for 64.
+  // See 'labels' in https://cloud.google.com/genomics/reference/rpc/google.genomics.v1alpha2#google.genomics.v1alpha2.RunPipelineArgs
+  private val MAX_LABEL_LENGTH = 63
+
+  def validate(s: String) = "[a-z]([-a-z0-9]*[a-z0-9])?".r.pattern.matcher(s).matches && s.length <= MAX_LABEL_LENGTH
+
+  /**
+    * Change to meet the constraint:
+    *  - Must match the regex [a-z]([-a-z0-9]*[a-z0-9])?
+    *  - Must be between 1 and MAX_LABEL_LENGTH characters total
+    */
+  def safeName(mainText: String) = {
+
+    if (validate(mainText)) {
+      mainText
+    } else {
+      def appendSafe(current: String, nextChar: Char): String = {
+        nextChar match {
+          case c if c.isLetterOrDigit || c == '-' => current + c.toLower
+          case _ => current + '-'
+        }
+      }
+
+      val foldResult = mainText.toCharArray.foldLeft("")(appendSafe)
+
+      val startsValid = foldResult.headOption.exists(_.isLetter)
+      val endsValid = foldResult.lastOption.exists(_.isLetterOrDigit)
+
+      val validStart = if (startsValid) foldResult else "x--" + foldResult
+      val validStartAndEnd = if (endsValid) validStart else validStart + "--x"
+
+      val length = validStartAndEnd.length
+      val tooLong = length > MAX_LABEL_LENGTH
+
+      if (tooLong) {
+        val middleSeparator = "---"
+        val subSectionLength = (MAX_LABEL_LENGTH - middleSeparator.length) / 2
+        validStartAndEnd.substring(0, subSectionLength) + middleSeparator + validStartAndEnd.substring(length - subSectionLength, length)
+      } else {
+        validStartAndEnd
+      }
+    }
+  }
+
+  def safeLabel(key: String, value: String): Label = {
+    Label(safeName(key), safeName(value))
+  }
+}
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/labels/Labels.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/labels/Labels.scala
new file mode 100644
index 000000000..6932e1fc8
--- /dev/null
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/labels/Labels.scala
@@ -0,0 +1,17 @@
+package cromwell.backend.impl.jes.labels
+
+import scala.collection.JavaConverters._
+
+case class Labels(value: Vector[Label]) {
+
+  def asJesLabels = (value map { label => label.key -> label.value }).toMap.asJava
+
+  def ++(that: Labels) = Labels(value ++ that.value)
+}
+
+object Labels {
+  def apply(values: (String, String)*): Labels = {
+    val kvps: Seq[(String, String)] = values.toSeq
+    Labels((kvps map { case (k, v) => Label.safeLabel(k, v) }).to[Vector])
+  }
+}
diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/labels/LabelSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/labels/LabelSpec.scala
new file mode 100644
index 000000000..41c19140b
--- /dev/null
+++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/labels/LabelSpec.scala
@@ -0,0 +1,45 @@
+package cromwell.backend.impl.jes.labels
+
+import org.scalatest.{FlatSpec, Matchers}
+
+class LabelSpec extends FlatSpec with Matchers {
+
+  behavior of "Labels"
+
+  /**
+    * In the format 'to validate', 'expected result'
+    */
+  val goodLabelStrings = List(
+    "cromwell-root-workflow-id",
+    "cromwell-11f2468c-39d6-4be3-85c8-32735c01e66b",
+    "just-the-right-length-just-the-right-length-just-the-right-leng"
+  )
+
+  val badLabelConversions = List(
+    "11f2468c-39d6-4be3-85c8-32735c01e66b" -> "x--11f2468c-39d6-4be3-85c8-32735c01e66b",
+    "0-cromwell-root-workflow-id" -> "x--0-cromwell-root-workflow-id",
+    "" -> "x----x",
+    "cromwell-root-workflow-id-" -> "cromwell-root-workflow-id---x",
+    "0-cromwell-root-workflow-id-" -> "x--0-cromwell-root-workflow-id---x",
+    "Cromwell-root-workflow-id" -> "cromwell-root-workflow-id",
+    "cromwell_root_workflow_id" -> "cromwell-root-workflow-id",
+    "too-long-too-long-too-long-too-long-too-long-too-long-too-long-t" -> "too-long-too-long-too-long-too---g-too-long-too-long-too-long-t",
+    "0-too-long-and-invalid-too-long-and-invalid-too-long-and-invali+" -> "x--0-too-long-and-invalid-too----nvalid-too-long-and-invali---x"
+  )
+
+  goodLabelStrings foreach { label =>
+    it should s"validate the good label string '$label'" in {
+      Label.validate(label) should be(true)
+    }
+  }
+
+  badLabelConversions foreach { case (label: String, conversion: String) =>
+    it should s"not validate the bad label string '$label'" in {
+      Label.validate(label) should be(false)
+    }
+
+    it should s"convert the bad label string '$label' into the safe label string '$conversion'" in {
+      Label.safeName(label) should be(conversion)
+    }
+  }
+}

From a7231aee8727c6ebd7c963b52af6c59bd9a1320d Mon Sep 17 00:00:00 2001
From: Joel Thibault <thibault@broadinstitute.org>
Date: Mon, 9 Jan 2017 17:33:31 -0500
Subject: [PATCH 07/84] consistent commenting style

---
 core/src/main/resources/reference.conf | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf
index 60e3903de..6b54f966d 100644
--- a/core/src/main/resources/reference.conf
+++ b/core/src/main/resources/reference.conf
@@ -125,18 +125,18 @@ google {
       name = "application-default"
       scheme = "application_default"
     },
-//    {
-//      name = "user-via-refresh"
-//      scheme = "refresh_token"
-//      client-id = "secret_id"
-//      client-secret = "secret_secret"
-//    },
-//    {
-//      name = "service-account"
-//      scheme = "service_account"
-//      service-account-id = "my-service-account"
-//      pem-file = "/path/to/file.pem"
-//    }
+    #{
+    #  name = "user-via-refresh"
+    #  scheme = "refresh_token"
+    #  client-id = "secret_id"
+    #  client-secret = "secret_secret"
+    #},
+    #{
+    #  name = "service-account"
+    #  scheme = "service_account"
+    #  service-account-id = "my-service-account"
+    #  pem-file = "/path/to/file.pem"
+    #}
   ]
 }
 

From 45818562eed5b7c2aa9161ffc58962686e11cf1b Mon Sep 17 00:00:00 2001
From: Jeff Gentry <geoffjentry@gmail.com>
Date: Mon, 9 Jan 2017 20:46:19 -0500
Subject: [PATCH 08/84]  Allow default zones to be set in the config. Closes
 #1795 (#1797)

* Allow default zones to be set in the config. Closes #1795
---
 CHANGELOG.md                                       |  1 +
 README.md                                          |  2 +-
 core/src/main/resources/reference.conf             |  3 +++
 .../cromwell/backend/impl/jes/JesAttributes.scala  | 28 +++++++++++++++-------
 .../impl/jes/JesBackendInitializationData.scala    |  5 ++--
 .../backend/impl/jes/JesConfiguration.scala        |  1 +
 .../backend/impl/jes/JesInitializationActor.scala  |  8 +++----
 .../backend/impl/jes/JesRuntimeAttributes.scala    | 17 ++++++-------
 .../jes/JesAsyncBackendJobExecutionActorSpec.scala |  6 +++--
 .../backend/impl/jes/JesAttributesSpec.scala       |  5 ++++
 .../backend/impl/jes/JesConfigurationSpec.scala    |  7 +++++-
 .../impl/jes/JesRuntimeAttributesSpec.scala        | 25 ++++++++++++++-----
 12 files changed, 76 insertions(+), 32 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index acfd1f07a..f96e88a9a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,6 +3,7 @@
 ## 25
 
 * Cromwell now applies default labels automatically to JES pipeline runs.
+* Added ability to override the default zone(s) used by JES via the config structure by setting `genomics.default-zones` in the JES configuration
 
 ## 24
 
diff --git a/README.md b/README.md
index 1d8d30a05..63f2701fb 100644
--- a/README.md
+++ b/README.md
@@ -1610,7 +1610,7 @@ runtime {
 }
 ```
 
-Defaults to "us-central1-b"
+Defaults to the configuration setting `genomics.default-zones` in the JES configuration block which in turn defaults to using `us-central1-b`
 
 ## docker
 
diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf
index 6b54f966d..765042ea7 100644
--- a/core/src/main/resources/reference.conf
+++ b/core/src/main/resources/reference.conf
@@ -352,6 +352,9 @@ backend {
     #      # Pipelines and manipulate auth JSONs.
     #      auth = "application-default"
     #
+    #      # Specifies the zone(s) to use for JES jobs unless overridden by a task's runtime attributes
+    #      default-zones = ["us-central1-b"]
+    #
     #      // alternative service account to use on the launched compute instance
     #      // NOTE: If combined with service account authorization, both that serivce account and this service account
     #      // must be able to read and write to the 'root' GCS path
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala
index 14ca8040e..dff6a755c 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala
@@ -2,14 +2,16 @@ package cromwell.backend.impl.jes
 
 import java.net.{URI, URL}
 
+import cats.data.NonEmptyList
 import cats.data.Validated._
 import cats.syntax.cartesian._
+import cats.syntax.validated._
 import com.typesafe.config.{Config, ConfigValue}
 import cromwell.backend.impl.jes.authentication.JesAuths
 import cromwell.filesystems.gcs.GoogleConfiguration
-import lenthall.validation.Validation._
 import lenthall.exception.MessageAggregation
 import lenthall.validation.ErrorOr._
+import lenthall.validation.Validation._
 import net.ceedubs.ficus.Ficus._
 import net.ceedubs.ficus.readers.{StringReader, ValueReader}
 import org.slf4j.LoggerFactory
@@ -22,7 +24,8 @@ case class JesAttributes(project: String,
                          executionBucket: String,
                          endpointUrl: URL,
                          maxPollingInterval: Int,
-                         qps: Int)
+                         qps: Int,
+                         defaultZones: NonEmptyList[String])
 
 object JesAttributes {
   lazy val Logger = LoggerFactory.getLogger("JesAttributes") 
@@ -40,7 +43,8 @@ object JesAttributes {
     "genomics.auth",
     "genomics.endpoint-url",
     "filesystems.gcs.auth",
-    "genomics-api-queries-per-100-seconds"
+    "genomics-api-queries-per-100-seconds",
+    "genomics.default-zones"
   )
 
   private val context = "Jes"
@@ -58,14 +62,14 @@ object JesAttributes {
     val computeServiceAccount: String = backendConfig.as[Option[String]]("genomics.compute-service-account").getOrElse("default")
     val genomicsAuthName: ErrorOr[String] = validate { backendConfig.as[String]("genomics.auth") }
     val gcsFilesystemAuthName: ErrorOr[String] = validate { backendConfig.as[String]("filesystems.gcs.auth") }
-
     val qps = backendConfig.as[Option[Int]]("genomics-api-queries-per-100-seconds").getOrElse(GenomicsApiDefaultQps) / 100
+    val defaultZones = defaultZonesFromConfig(backendConfig)
 
-    (project |@| executionBucket |@| endpointUrl |@| genomicsAuthName |@| gcsFilesystemAuthName) map {
-      (_, _, _, _, _)
-    } flatMap { case (p, b, u, genomicsName, gcsName) =>
+    (project |@| executionBucket |@| endpointUrl |@| genomicsAuthName |@| gcsFilesystemAuthName |@| defaultZones) map {
+      (_, _, _, _, _, _)
+    } flatMap { case (p, b, u, genomicsName, gcsName, d) =>
       (googleConfig.auth(genomicsName) |@| googleConfig.auth(gcsName)) map { case (genomicsAuth, gcsAuth) =>
-        JesAttributes(p, computeServiceAccount, JesAuths(genomicsAuth, gcsAuth), b, u, maxPollingInterval, qps)
+        JesAttributes(p, computeServiceAccount, JesAuths(genomicsAuth, gcsAuth), b, u, maxPollingInterval, qps, d)
       }
     } match {
       case Valid(r) => r
@@ -76,4 +80,12 @@ object JesAttributes {
         }
     }
   }
+
+  def defaultZonesFromConfig(config: Config): ErrorOr[NonEmptyList[String]] = {
+    val zones = config.as[Option[List[String]]]("genomics.default-zones").getOrElse(List("us-central1-b"))
+    zones match {
+      case x :: xs => NonEmptyList(x, xs).validNel
+      case _ => "genomics.default-zones was set but no values were provided".invalidNel
+    }
+  }
 }
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendInitializationData.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendInitializationData.scala
index 0d3d07c31..7662b79e6 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendInitializationData.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendInitializationData.scala
@@ -1,11 +1,12 @@
 package cromwell.backend.impl.jes
 
 import com.google.api.services.genomics.Genomics
-import cromwell.backend.standard.StandardInitializationData
+import cromwell.backend.standard.{StandardInitializationData, StandardValidatedRuntimeAttributesBuilder}
 
 case class JesBackendInitializationData
 (
   override val workflowPaths: JesWorkflowPaths,
+  override val runtimeAttributesBuilder: StandardValidatedRuntimeAttributesBuilder,
   jesConfiguration: JesConfiguration,
   genomics: Genomics
-) extends StandardInitializationData(workflowPaths, JesRuntimeAttributes.runtimeAttributesBuilder)
+) extends StandardInitializationData(workflowPaths, runtimeAttributesBuilder)
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesConfiguration.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesConfiguration.scala
index 20ec8130d..2c1e82f92 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesConfiguration.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesConfiguration.scala
@@ -34,4 +34,5 @@ class JesConfiguration(val configurationDescriptor: BackendConfigurationDescript
   val dockerCredentials = DockerConfiguration.build(configurationDescriptor.backendConfig).dockerCredentials map JesDockerCredentials.apply
   val needAuthFileUpload = jesAuths.gcs.requiresAuthFile || dockerCredentials.isDefined
   val qps = jesAttributes.qps
+  val defaultZones = jesAttributes.defaultZones
 }
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesInitializationActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesInitializationActor.scala
index 20d49fa9a..3ebaa570d 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesInitializationActor.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesInitializationActor.scala
@@ -44,11 +44,11 @@ class JesInitializationActor(jesParams: JesInitializationActorParams)
 
   override val standardParams: StandardInitializationActorParams = jesParams
 
-  override lazy val runtimeAttributesBuilder: StandardValidatedRuntimeAttributesBuilder =
-    JesRuntimeAttributes.runtimeAttributesBuilder
-
   private val jesConfiguration = jesParams.jesConfiguration
 
+  override lazy val runtimeAttributesBuilder: StandardValidatedRuntimeAttributesBuilder =
+    JesRuntimeAttributes.runtimeAttributesBuilder(jesConfiguration)
+
   private[jes] lazy val refreshTokenAuth: Option[JesAuthInformation] = {
     for {
       clientSecrets <- List(jesConfiguration.jesAttributes.auths.gcs) collectFirst { case s: ClientSecrets => s }
@@ -70,7 +70,7 @@ class JesInitializationActor(jesParams: JesInitializationActorParams)
       workflowPaths = new JesWorkflowPaths(workflowDescriptor, jesConfiguration)(context.system)
       _ <- if (jesConfiguration.needAuthFileUpload) writeAuthenticationFile(workflowPaths) else Future.successful(())
       _ = publishWorkflowRoot(workflowPaths.workflowRoot.toString)
-    } yield Option(JesBackendInitializationData(workflowPaths, jesConfiguration, genomics))
+    } yield Option(JesBackendInitializationData(workflowPaths, runtimeAttributesBuilder, jesConfiguration, genomics))
   }
 
   private def writeAuthenticationFile(workflowPath: JesWorkflowPaths): Future[Unit] = {
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesRuntimeAttributes.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesRuntimeAttributes.scala
index dbd3bd392..b62aa445b 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesRuntimeAttributes.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesRuntimeAttributes.scala
@@ -1,5 +1,6 @@
 package cromwell.backend.impl.jes
 
+import cats.data.NonEmptyList
 import cats.data.Validated._
 import cats.syntax.cartesian._
 import cats.syntax.validated._
@@ -27,7 +28,6 @@ object JesRuntimeAttributes {
   private val MemoryDefaultValue = "2 GB"
 
   val ZonesKey = "zones"
-  private val ZoneDefaultValue = "us-central1-b"
 
   val PreemptibleKey = "preemptible"
   private val PreemptibleDefaultValue = 0
@@ -46,8 +46,8 @@ object JesRuntimeAttributes {
   private val disksValidation: RuntimeAttributesValidation[Seq[JesAttachedDisk]] =
     DisksValidation.withDefault(WdlString(DisksDefaultValue))
 
-  private val zonesValidation: RuntimeAttributesValidation[Vector[String]] =
-    ZonesValidation.withDefault(WdlString(ZoneDefaultValue))
+  private def zonesValidation(defaultZones: NonEmptyList[String]): RuntimeAttributesValidation[Vector[String]] =
+    ZonesValidation.withDefault(WdlString(defaultZones.toList.mkString(" ")))
 
   private val preemptibleValidation: RuntimeAttributesValidation[Int] =
     new IntRuntimeAttributesValidation(JesRuntimeAttributes.PreemptibleKey)
@@ -66,11 +66,11 @@ object JesRuntimeAttributes {
 
   private val dockerValidation: RuntimeAttributesValidation[String] = DockerValidation.instance
 
-  val runtimeAttributesBuilder: StandardValidatedRuntimeAttributesBuilder =
+  def runtimeAttributesBuilder(jesConfiguration: JesConfiguration): StandardValidatedRuntimeAttributesBuilder =
     StandardValidatedRuntimeAttributesBuilder.default.withValidation(
       cpuValidation,
       disksValidation,
-      zonesValidation,
+      zonesValidation(jesConfiguration.defaultZones),
       preemptibleValidation,
       memoryValidation,
       bootDiskSizeValidation,
@@ -80,7 +80,7 @@ object JesRuntimeAttributes {
 
   def apply(validatedRuntimeAttributes: ValidatedRuntimeAttributes): JesRuntimeAttributes = {
     val cpu: Int = RuntimeAttributesValidation.extract(cpuValidation, validatedRuntimeAttributes)
-    val zones: Vector[String] = RuntimeAttributesValidation.extract(zonesValidation, validatedRuntimeAttributes)
+    val zones: Vector[String] = RuntimeAttributesValidation.extract(ZonesValidation, validatedRuntimeAttributes)
     val preemptible: Int = RuntimeAttributesValidation.extract(preemptibleValidation, validatedRuntimeAttributes)
     val bootDiskSize: Int = RuntimeAttributesValidation.extract(bootDiskSizeValidation, validatedRuntimeAttributes)
     val memory: MemorySize = RuntimeAttributesValidation.extract(memoryValidation, validatedRuntimeAttributes)
@@ -107,8 +107,9 @@ object JesRuntimeAttributes {
   }
 
   // NOTE: Currently only used by test specs
-  private[jes] def apply(attrs: Map[String, WdlValue], logger: Logger): JesRuntimeAttributes = {
-    val runtimeAttributesBuilder = JesRuntimeAttributes.runtimeAttributesBuilder
+  private[jes] def apply(attrs: Map[String, WdlValue], logger: Logger,
+                         jesConfiguration: JesConfiguration): JesRuntimeAttributes = {
+    val runtimeAttributesBuilder = JesRuntimeAttributes.runtimeAttributesBuilder(jesConfiguration)
     val validatedRuntimeAttributes = runtimeAttributesBuilder.build(attrs, logger)
     apply(validatedRuntimeAttributes)
   }
diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala
index f331f73a8..605cc7201 100644
--- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala
+++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala
@@ -80,7 +80,8 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
 
   private def buildInitializationData(jobDescriptor: BackendJobDescriptor, configuration: JesConfiguration) = {
     val workflowPaths = JesWorkflowPaths(jobDescriptor.workflowDescriptor, configuration)(system)
-    JesBackendInitializationData(workflowPaths, configuration, null)
+    val runtimeAttributesBuilder = JesRuntimeAttributes.runtimeAttributesBuilder(configuration)
+    JesBackendInitializationData(workflowPaths, runtimeAttributesBuilder, configuration, null)
   }
 
   class TestableJesJobExecutionActor(params: StandardAsyncExecutionActorParams, functions: JesExpressionFunctions)
@@ -114,6 +115,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
   }
 
   private val jesConfiguration = new JesConfiguration(JesBackendConfigurationDescriptor)
+  private val runtimeAttributesBuilder = JesRuntimeAttributes.runtimeAttributesBuilder(jesConfiguration)
   private val workingDisk = JesWorkingDisk(DiskType.SSD, 200)
 
   val DockerAndDiskRuntime: String =
@@ -681,6 +683,6 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
   private def makeRuntimeAttributes(job: TaskCall) = {
     val evaluatedAttributes = RuntimeAttributeDefinition.evaluateRuntimeAttributes(job.task.runtimeAttributes, TestableJesExpressionFunctions, Map.empty)
     RuntimeAttributeDefinition.addDefaultsToAttributes(
-      JesRuntimeAttributes.runtimeAttributesBuilder.definitions.toSet, NoOptions)(evaluatedAttributes.get)
+      runtimeAttributesBuilder.definitions.toSet, NoOptions)(evaluatedAttributes.get)
   }
 }
diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAttributesSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAttributesSpec.scala
index 00eeb24fd..5d9a5724e 100644
--- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAttributesSpec.scala
+++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAttributesSpec.scala
@@ -24,6 +24,7 @@ class JesAttributesSpec extends FlatSpec with Matchers {
     jesAttributes.executionBucket should be("gs://myBucket")
     jesAttributes.maxPollingInterval should be(600)
     jesAttributes.computeServiceAccount should be("default")
+    jesAttributes.defaultZones.toList should be (List("us-central1-a"))
   }
 
   it should "parse correct preemptible config" taggedAs IntegrationTest in {
@@ -35,6 +36,7 @@ class JesAttributesSpec extends FlatSpec with Matchers {
     jesAttributes.project should be("myProject")
     jesAttributes.executionBucket should be("gs://myBucket")
     jesAttributes.maxPollingInterval should be(600)
+    jesAttributes.defaultZones.toList should be (List("us-central1-a"))
   }
 
   it should "parse compute service account" taggedAs IntegrationTest in {
@@ -52,6 +54,7 @@ class JesAttributesSpec extends FlatSpec with Matchers {
           |{
           |   genomics {
           |     endpoint-url = "myEndpoint"
+          |     default-zones = []
           |   }
           |}
         """.stripMargin)
@@ -67,6 +70,7 @@ class JesAttributesSpec extends FlatSpec with Matchers {
     errorsList should contain("No configuration setting found for key 'genomics.auth'")
     errorsList should contain("No configuration setting found for key 'filesystems'")
     errorsList should contain("URI is not absolute")
+    errorsList should contain("genomics.default-zones was set but no values were provided")
   }
 
   def configString(preemptible: String = "", genomics: String = ""): String =
@@ -82,6 +86,7 @@ class JesAttributesSpec extends FlatSpec with Matchers {
       |     auth = "application-default"
       |    $genomics
       |     endpoint-url = "http://myEndpoint"
+      |     default-zones = ["us-central1-a"]
       |   }
       |
       |   filesystems = {
diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesConfigurationSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesConfigurationSpec.scala
index 46068343c..f94991670 100644
--- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesConfigurationSpec.scala
+++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesConfigurationSpec.scala
@@ -1,7 +1,7 @@
 package cromwell.backend.impl.jes
 
 import better.files.File
-import com.typesafe.config.{ConfigValueFactory, ConfigFactory}
+import com.typesafe.config.{ConfigFactory, ConfigValueFactory}
 import cromwell.backend.BackendConfigurationDescriptor
 import org.scalatest.prop.TableDrivenPropertyChecks
 import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers}
@@ -63,6 +63,7 @@ class JesConfigurationSpec extends FlatSpec with Matchers with TableDrivenProper
       |     auth = "application-default"
       |     // Endpoint for APIs, no reason to change this unless directed by Google.
       |     endpoint-url = "https://genomics.googleapis.com/"
+      |     default-zones = ["us-central1-a", "us-central1-b"]
       |  }
       |
       |  dockerhub {
@@ -103,6 +104,10 @@ class JesConfigurationSpec extends FlatSpec with Matchers with TableDrivenProper
     new JesConfiguration(BackendConfigurationDescriptor(backendConfig, globalConfig)).root shouldBe "gs://my-cromwell-workflows-bucket"
   }
 
+  it should "have the correct default zones" in {
+    new JesConfiguration(BackendConfigurationDescriptor(backendConfig, globalConfig)).defaultZones.toList shouldBe List("us-central1-a", "us-central1-b")
+  }
+
   it should "have correct docker" in {
     val dockerConf = new JesConfiguration(BackendConfigurationDescriptor(backendConfig, globalConfig)).dockerCredentials
     dockerConf shouldBe defined
diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesRuntimeAttributesSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesRuntimeAttributesSpec.scala
index d8ddd68e2..cb18aa907 100644
--- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesRuntimeAttributesSpec.scala
+++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesRuntimeAttributesSpec.scala
@@ -1,5 +1,6 @@
 package cromwell.backend.impl.jes
 
+import cats.data.NonEmptyList
 import cromwell.backend.impl.jes.io.{DiskType, JesAttachedDisk, JesWorkingDisk}
 import cromwell.backend.validation.ContinueOnReturnCodeSet
 import cromwell.backend.{MemorySize, RuntimeAttributeDefinition}
@@ -20,8 +21,9 @@ class JesRuntimeAttributesSpec extends WordSpecLike with Matchers with Mockito {
     )))
   }
 
-  val expectedDefaults = new JesRuntimeAttributes(1, Vector("us-central1-b"), 0, 10, MemorySize(2, MemoryUnit.GB),
-    Seq(JesWorkingDisk(DiskType.SSD, 10)), "ubuntu:latest", false, ContinueOnReturnCodeSet(Set(0)), false)
+  val expectedDefaults = new JesRuntimeAttributes(1, Vector("us-central1-b", "us-central1-a"), 0, 10,
+    MemorySize(2, MemoryUnit.GB), Seq(JesWorkingDisk(DiskType.SSD, 10)), "ubuntu:latest", false,
+    ContinueOnReturnCodeSet(Set(0)), false)
 
   "JesRuntimeAttributes" should {
 
@@ -189,11 +191,16 @@ class JesRuntimeAttributesSpec extends WordSpecLike with Matchers with Mockito {
     }
   }
 
-  private def assertJesRuntimeAttributesSuccessfulCreation(runtimeAttributes: Map[String, WdlValue], expectedRuntimeAttributes: JesRuntimeAttributes, workflowOptions: WorkflowOptions = emptyWorkflowOptions): Unit = {
+  private def assertJesRuntimeAttributesSuccessfulCreation(runtimeAttributes: Map[String, WdlValue],
+                                                           expectedRuntimeAttributes: JesRuntimeAttributes,
+                                                           workflowOptions: WorkflowOptions = emptyWorkflowOptions,
+                                                           defaultZones: NonEmptyList[String] = defaultZones): Unit = {
     val withDefaults = RuntimeAttributeDefinition.addDefaultsToAttributes(
       staticRuntimeAttributeDefinitions, workflowOptions) _
     try {
-      assert(JesRuntimeAttributes(withDefaults(runtimeAttributes), NOPLogger.NOP_LOGGER) == expectedRuntimeAttributes)
+      val actualRuntimeAttributes =
+        JesRuntimeAttributes(withDefaults(runtimeAttributes), NOPLogger.NOP_LOGGER, jesConfiguration)
+      assert(actualRuntimeAttributes == expectedRuntimeAttributes)
     } catch {
       case ex: RuntimeException => fail(s"Exception was not expected but received: ${ex.getMessage}")
     }
@@ -204,7 +211,7 @@ class JesRuntimeAttributesSpec extends WordSpecLike with Matchers with Mockito {
     val withDefaults = RuntimeAttributeDefinition.addDefaultsToAttributes(
       staticRuntimeAttributeDefinitions, workflowOptions) _
     try {
-      JesRuntimeAttributes(withDefaults(runtimeAttributes), NOPLogger.NOP_LOGGER)
+      JesRuntimeAttributes(withDefaults(runtimeAttributes), NOPLogger.NOP_LOGGER, jesConfiguration)
       fail("A RuntimeException was expected.")
     } catch {
       case ex: RuntimeException => assert(ex.getMessage.contains(exMsg))
@@ -213,6 +220,12 @@ class JesRuntimeAttributesSpec extends WordSpecLike with Matchers with Mockito {
   }
 
   private val emptyWorkflowOptions = WorkflowOptions.fromMap(Map.empty).get
+  private val defaultZones = NonEmptyList.of("us-central1-b", "us-central1-a")
+  private val jesConfiguration = {
+    val config = mock[JesConfiguration]
+    config.defaultZones returns defaultZones
+    config
+  }
   private val staticRuntimeAttributeDefinitions: Set[RuntimeAttributeDefinition] =
-    JesRuntimeAttributes.runtimeAttributesBuilder.definitions.toSet
+    JesRuntimeAttributes.runtimeAttributesBuilder(jesConfiguration).definitions.toSet
 }

From 7d8ccdc288e178dd75066cfd88c9b5b0faada98c Mon Sep 17 00:00:00 2001
From: Miguel Covarrubias <mcovarr@broadinstitute.org>
Date: Fri, 6 Jan 2017 11:27:38 -0500
Subject: [PATCH 09/84] Add length().  Closes #1604

---
 CHANGELOG.md                                       |  2 ++
 README.md                                          |  1 +
 .../wdl/PureStandardLibraryFunctionsSpec.scala     | 30 ----------------------
 project/Dependencies.scala                         |  2 +-
 4 files changed, 4 insertions(+), 31 deletions(-)
 delete mode 100644 backend/src/test/scala/cromwell/backend/wdl/PureStandardLibraryFunctionsSpec.scala

diff --git a/CHANGELOG.md b/CHANGELOG.md
index f96e88a9a..53bd0ef36 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,8 @@
 
 * Cromwell now applies default labels automatically to JES pipeline runs.
 * Added ability to override the default zone(s) used by JES via the config structure by setting `genomics.default-zones` in the JES configuration
+* Added support for new WDL functions:
+  * `length: (Array[X]) => Integer` - report the length of the specified array 
 
 ## 24
 
diff --git a/README.md b/README.md
index 63f2701fb..69c6ccc8d 100644
--- a/README.md
+++ b/README.md
@@ -378,6 +378,7 @@ For many examples on how to use WDL see [the WDL site](https://github.com/broadi
   * [Array\[Array\[X\]\] transpose(Array\[Array\[X\]\])](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#arrayarrayx-transposearrayarrayx)
   * [Pair(X,Y) zip(X,Y)](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#pairxy-zipxy)
   * [Pair(X,Y) cross(X,Y)](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#pairxy-crossxy)
+  * [Integer length(Array\[X\])](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#integer-lengtharrayx)
 * [Data Types & Serialization](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#data-types--serialization)
   * [Serialization of Task Inputs](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#serialization-of-task-inputs)
     * [Primitive Types](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#primitive-types)
diff --git a/backend/src/test/scala/cromwell/backend/wdl/PureStandardLibraryFunctionsSpec.scala b/backend/src/test/scala/cromwell/backend/wdl/PureStandardLibraryFunctionsSpec.scala
deleted file mode 100644
index fb6507377..000000000
--- a/backend/src/test/scala/cromwell/backend/wdl/PureStandardLibraryFunctionsSpec.scala
+++ /dev/null
@@ -1,30 +0,0 @@
-package cromwell.backend.wdl
-
-import org.scalatest.{FlatSpec, Matchers}
-import wdl4s.expression.PureStandardLibraryFunctions
-import wdl4s.types.{WdlArrayType, WdlIntegerType}
-import wdl4s.values.{WdlArray, WdlInteger}
-
-import scala.util.Success
-
-
-class PureStandardLibraryFunctionsSpec extends FlatSpec with Matchers {
-
-  behavior of "transpose"
-
-  it should "transpose a 2x3 into a 3x2" in {
-    val inArray = WdlArray(WdlArrayType(WdlArrayType(WdlIntegerType)), List(
-      WdlArray(WdlArrayType(WdlIntegerType), List(WdlInteger(1), WdlInteger(2), WdlInteger(3))),
-      WdlArray(WdlArrayType(WdlIntegerType), List(WdlInteger(4), WdlInteger(5), WdlInteger(6)))
-    ))
-
-    val expectedResult = WdlArray(WdlArrayType(WdlArrayType(WdlIntegerType)), List(
-      WdlArray(WdlArrayType(WdlIntegerType), List(WdlInteger(1), WdlInteger(4))),
-      WdlArray(WdlArrayType(WdlIntegerType), List(WdlInteger(2), WdlInteger(5))),
-      WdlArray(WdlArrayType(WdlIntegerType), List(WdlInteger(3), WdlInteger(6)))
-    ))
-
-    PureStandardLibraryFunctions.transpose(Seq(Success(inArray))) should be(Success(expectedResult))
-  }
-
-}
diff --git a/project/Dependencies.scala b/project/Dependencies.scala
index f683b492b..31909fc5e 100644
--- a/project/Dependencies.scala
+++ b/project/Dependencies.scala
@@ -2,7 +2,7 @@ import sbt._
 
 object Dependencies {
   lazy val lenthallV = "0.20"
-  lazy val wdl4sV = "0.8"
+  lazy val wdl4sV = "0.9-369e288-SNAP"
   lazy val sprayV = "1.3.3"
   /*
   spray-json is an independent project from the "spray suite"

From 1930608fb0bc383ec21b9d059b38eaaed7fc0db0 Mon Sep 17 00:00:00 2001
From: Thib <Horneth@users.noreply.github.com>
Date: Wed, 11 Jan 2017 09:11:46 -0500
Subject: [PATCH 10/84] Don't log stack traces for known failures Closes #1817
 (#1824)

* don't log stack traces for known failures
---
 .../backend/async/KnownJobFailureException.scala   | 16 ++++++++++
 .../standard/StandardAsyncExecutionActor.scala     | 35 +++++++++-------------
 .../engine/workflow/WorkflowManagerActor.scala     |  7 ++++-
 .../scala/cromwell/SimpleWorkflowActorSpec.scala   |  2 +-
 .../SharedFileSystemJobExecutionActorSpec.scala    |  3 +-
 5 files changed, 39 insertions(+), 24 deletions(-)
 create mode 100644 backend/src/main/scala/cromwell/backend/async/KnownJobFailureException.scala

diff --git a/backend/src/main/scala/cromwell/backend/async/KnownJobFailureException.scala b/backend/src/main/scala/cromwell/backend/async/KnownJobFailureException.scala
new file mode 100644
index 000000000..4828678b8
--- /dev/null
+++ b/backend/src/main/scala/cromwell/backend/async/KnownJobFailureException.scala
@@ -0,0 +1,16 @@
+package cromwell.backend.async
+
+import java.nio.file.Path
+
+sealed abstract class KnownJobFailureException extends Exception {
+  def stderrPath: Path
+  def jobTag: String
+}
+
+final case class WrongReturnCode(jobTag: String, returnCode: Int, stderrPath: Path) extends KnownJobFailureException {
+  override def getMessage = s"Job $jobTag exited with return code $returnCode which has not been declared as a valid return code. See 'continueOnReturnCode' runtime attribute for more details."
+}
+
+final case class StderrNonEmpty(jobTag: String, stderrLength: Long, stderrPath: Path) extends KnownJobFailureException {
+  override def getMessage = s"stderr for job $jobTag has length $stderrLength and 'failOnStderr' runtime attribute was true."
+}
diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala b/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala
index a88f07fdb..b6735ffc5 100644
--- a/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala
+++ b/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala
@@ -6,7 +6,7 @@ import better.files.File
 import cromwell.backend.BackendJobExecutionActor.{AbortedResponse, BackendJobExecutionResponse}
 import cromwell.backend.BackendLifecycleActor.AbortJobCommand
 import cromwell.backend.async.AsyncBackendJobExecutionActor.{ExecutionMode, JobId, Recover}
-import cromwell.backend.async.{AbortedExecutionHandle, AsyncBackendJobExecutionActor, ExecutionHandle, FailedNonRetryableExecutionHandle, PendingExecutionHandle, SuccessfulExecutionHandle}
+import cromwell.backend.async.{AbortedExecutionHandle, AsyncBackendJobExecutionActor, ExecutionHandle, FailedNonRetryableExecutionHandle, PendingExecutionHandle, StderrNonEmpty, SuccessfulExecutionHandle, WrongReturnCode}
 import cromwell.backend.validation._
 import cromwell.backend.wdl.Command
 import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendJobDescriptor, BackendJobLifecycleActor}
@@ -375,26 +375,19 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
 
         lazy val stderrLength: Long = File(jobPaths.stderr).size
         lazy val returnCode: Try[Int] = Try(File(jobPaths.returnCode).contentAsString).map(_.trim.toInt)
-        status match {
-          case _ if failOnStdErr && stderrLength.intValue > 0 =>
-            // returnCode will be None if it couldn't be downloaded/parsed, which will yield a null in the DB
-            FailedNonRetryableExecutionHandle(new RuntimeException(
-              s"execution failed: stderr has length $stderrLength"), returnCode.toOption)
-          case _ if returnCode.isFailure =>
-            val exception = returnCode.failed.get
-            jobLogger.warn(s"could not download return code file, retrying", exception)
-            // Return handle to try again.
-            oldHandle
-          case _ if returnCode.isFailure =>
-            FailedNonRetryableExecutionHandle(new RuntimeException(
-              s"execution failed: could not parse return code as integer", returnCode.failed.get))
-          case _ if isAbort(returnCode.get) =>
-            AbortedExecutionHandle
-          case _ if !continueOnReturnCode.continueFor(returnCode.get) =>
-            val message = s"Call ${jobDescriptor.key.tag}: return code was ${returnCode.get}"
-            FailedNonRetryableExecutionHandle(new RuntimeException(message), returnCode.toOption)
-          case _ =>
-            handleExecutionSuccess(status, oldHandle, returnCode.get)
+        if (failOnStdErr && stderrLength.intValue > 0) {
+          // returnCode will be None if it couldn't be downloaded/parsed, which will yield a null in the DB
+          FailedNonRetryableExecutionHandle(StderrNonEmpty(jobDescriptor.key.tag, stderrLength, jobPaths.stderr), returnCode.toOption)
+        } else if (returnCode.isFailure) {
+          val exception = returnCode.failed.get
+          FailedNonRetryableExecutionHandle(new RuntimeException(
+            s"execution failed: could not download or parse return code as integer", exception))
+        } else if (isAbort(returnCode.get)) {
+          AbortedExecutionHandle
+        } else if (!continueOnReturnCode.continueFor(returnCode.get)) {
+          FailedNonRetryableExecutionHandle(WrongReturnCode(jobDescriptor.key.tag, returnCode.get, jobPaths.stderr), returnCode.toOption)
+        } else {
+          handleExecutionSuccess(status, oldHandle, returnCode.get)
         }
 
       } else {
diff --git a/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala b/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala
index a19772cd8..766a54f66 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala
@@ -5,6 +5,7 @@ import akka.actor._
 import akka.event.Logging
 import cats.data.NonEmptyList
 import com.typesafe.config.{Config, ConfigFactory}
+import cromwell.backend.async.KnownJobFailureException
 import cromwell.core.Dispatcher.EngineDispatcher
 import cromwell.core.{WorkflowAborted, WorkflowId}
 import cromwell.engine.backend.BackendSingletonCollection
@@ -301,7 +302,11 @@ class WorkflowManagerActor(params: WorkflowManagerActorParams)
   }
 
   private def expandFailureReasons(reasons: Seq[Throwable]) = {
-    reasons map { reason =>
+    import cromwell.core.path.PathImplicits._
+    reasons map {
+      case reason: KnownJobFailureException =>
+        reason.getMessage + "\n" + s"Check the content of stderr for potential additional information: ${reason.stderrPath.toRealString}"
+      case reason =>
       reason.getMessage + "\n" + ExceptionUtils.getStackTrace(reason)
     } mkString "\n"
   }
diff --git a/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala b/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala
index 893f56bfa..29e382623 100644
--- a/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala
+++ b/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala
@@ -119,7 +119,7 @@ class SimpleWorkflowActorSpec extends CromwellTestKitSpec with BeforeAndAfter {
     }
 
     "fail when a call fails" in {
-      val expectedError = "Call wf_goodbye.goodbye:NA:1: return code was 1"
+      val expectedError = "Job wf_goodbye.goodbye:NA:1 exited with return code 1 which has not been declared as a valid return code. See 'continueOnReturnCode' runtime attribute for more details." 
       val failureMatcher = FailureMatcher(expectedError)
       val TestableWorkflowActorAndMetadataPromise(workflowActor, supervisor, promise) = buildWorkflowActor(SampleWdl.GoodbyeWorld, SampleWdl.GoodbyeWorld.wdlJson, workflowId, failureMatcher)
       val probe = TestProbe()
diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala
index f5ada20d9..82575378b 100644
--- a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala
+++ b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala
@@ -7,6 +7,7 @@ import better.files._
 import com.typesafe.config.ConfigFactory
 import cromwell.backend.BackendJobExecutionActor.{AbortedResponse, JobFailedNonRetryableResponse, JobSucceededResponse}
 import cromwell.backend.BackendLifecycleActor.AbortJobCommand
+import cromwell.backend.async.WrongReturnCode
 import cromwell.backend.io.TestWorkflows._
 import cromwell.backend.io.{JobPathsWithDocker, TestWorkflows}
 import cromwell.backend.sfs.TestLocalAsyncJobExecutionActor._
@@ -54,7 +55,7 @@ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSyst
 
   it should "send back an execution failure if the task fails" in {
     val expectedResponse =
-      JobFailedNonRetryableResponse(mock[BackendJobDescriptorKey], new RuntimeException(""), Option(1))
+      JobFailedNonRetryableResponse(mock[BackendJobDescriptorKey], WrongReturnCode("wf_goodbye.goodbye:NA:1", 1, Paths.get("")), Option(1))
     val workflow = TestWorkflow(buildWorkflowDescriptor(GoodbyeWorld), emptyBackendConfig, expectedResponse)
     val backend = createBackend(jobDescriptorFromSingleCallWorkflow(workflow.workflowDescriptor, Map.empty, WorkflowOptions.empty, runtimeAttributeDefinitions), workflow.config)
     testWorkflow(workflow, backend)

From a751a83f40781b71891efcfc365f5952e9de7e6e Mon Sep 17 00:00:00 2001
From: Miguel Covarrubias <mcovarr@broadinstitute.org>
Date: Wed, 11 Jan 2017 18:27:28 -0500
Subject: [PATCH 11/84] Fix some issues found during FireCloud Cromwell 24
 acceptance testing.

---
 CHANGELOG.md                                                            | 1 +
 .../jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala    | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 53bd0ef36..cc0be7d29 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -26,6 +26,7 @@
   * `transpose: (Array[Array[X]]) => Array[Array[X]]` compute the matrix transpose for a 2D array. Assumes each inner array has the same length.
 * By default, `system.abort-jobs-on-terminate` is false when running `java -jar cromwell.jar server`, and true when running `java -jar cromwell.jar run <wdl> <inputs>`.
 * Enable WDL imports when running in Single Workflow Runner Mode.
+* Both batch and non-batch REST workflow submissions now require a multipart/form-data encoded body.
 * Support for sub workflows (see [Annex A](#annex-a---workflow-outputs))
 * Enable WDL imports when running in Single Workflow Runner Mode as well as Server Mode
 * Support for WDL imports through an additional imports.zip parameter
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala
index dff6a755c..dc9534546 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala
@@ -38,6 +38,8 @@ object JesAttributes {
     "maximum-polling-interval",
     "genomics.compute-service-account",
     "dockerhub",
+    "dockerhub.account",
+    "dockerhub.token",
     "genomics",
     "filesystems",
     "genomics.auth",

From 39eb63d1d5e88e379d911737618a85be22ec31a8 Mon Sep 17 00:00:00 2001
From: Thib <Horneth@users.noreply.github.com>
Date: Thu, 12 Jan 2017 10:58:17 -0500
Subject: [PATCH 12/84] Clean up execution result logic (#1840)

* clean up execution success

* fix

* re fix
---
 .../backend/async/KnownJobFailureException.scala   |  9 +++++
 .../standard/StandardAsyncExecutionActor.scala     | 45 ++++++++++++++--------
 2 files changed, 38 insertions(+), 16 deletions(-)

diff --git a/backend/src/main/scala/cromwell/backend/async/KnownJobFailureException.scala b/backend/src/main/scala/cromwell/backend/async/KnownJobFailureException.scala
index 4828678b8..e43895e22 100644
--- a/backend/src/main/scala/cromwell/backend/async/KnownJobFailureException.scala
+++ b/backend/src/main/scala/cromwell/backend/async/KnownJobFailureException.scala
@@ -11,6 +11,15 @@ final case class WrongReturnCode(jobTag: String, returnCode: Int, stderrPath: Pa
   override def getMessage = s"Job $jobTag exited with return code $returnCode which has not been declared as a valid return code. See 'continueOnReturnCode' runtime attribute for more details."
 }
 
+final case class ReturnCodeIsNotAnInt(jobTag: String, returnCode: String, stderrPath: Path) extends KnownJobFailureException {
+  override def getMessage = {
+    if (returnCode.isEmpty)
+      s"The return code file for job $jobTag was empty."
+    else
+      s"Job $jobTag exited with return code $returnCode which couldn't be converted to an Integer."
+  }
+}
+
 final case class StderrNonEmpty(jobTag: String, stderrLength: Long, stderrPath: Path) extends KnownJobFailureException {
   override def getMessage = s"stderr for job $jobTag has length $stderrLength and 'failOnStderr' runtime attribute was true."
 }
diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala b/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala
index b6735ffc5..0d97cbbb9 100644
--- a/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala
+++ b/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala
@@ -6,7 +6,7 @@ import better.files.File
 import cromwell.backend.BackendJobExecutionActor.{AbortedResponse, BackendJobExecutionResponse}
 import cromwell.backend.BackendLifecycleActor.AbortJobCommand
 import cromwell.backend.async.AsyncBackendJobExecutionActor.{ExecutionMode, JobId, Recover}
-import cromwell.backend.async.{AbortedExecutionHandle, AsyncBackendJobExecutionActor, ExecutionHandle, FailedNonRetryableExecutionHandle, PendingExecutionHandle, StderrNonEmpty, SuccessfulExecutionHandle, WrongReturnCode}
+import cromwell.backend.async.{AbortedExecutionHandle, AsyncBackendJobExecutionActor, ExecutionHandle, FailedNonRetryableExecutionHandle, PendingExecutionHandle, ReturnCodeIsNotAnInt, StderrNonEmpty, SuccessfulExecutionHandle, WrongReturnCode}
 import cromwell.backend.validation._
 import cromwell.backend.wdl.Command
 import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendJobDescriptor, BackendJobLifecycleActor}
@@ -373,21 +373,34 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
     try {
       if (isSuccess(status)) {
 
-        lazy val stderrLength: Long = File(jobPaths.stderr).size
-        lazy val returnCode: Try[Int] = Try(File(jobPaths.returnCode).contentAsString).map(_.trim.toInt)
-        if (failOnStdErr && stderrLength.intValue > 0) {
-          // returnCode will be None if it couldn't be downloaded/parsed, which will yield a null in the DB
-          FailedNonRetryableExecutionHandle(StderrNonEmpty(jobDescriptor.key.tag, stderrLength, jobPaths.stderr), returnCode.toOption)
-        } else if (returnCode.isFailure) {
-          val exception = returnCode.failed.get
-          FailedNonRetryableExecutionHandle(new RuntimeException(
-            s"execution failed: could not download or parse return code as integer", exception))
-        } else if (isAbort(returnCode.get)) {
-          AbortedExecutionHandle
-        } else if (!continueOnReturnCode.continueFor(returnCode.get)) {
-          FailedNonRetryableExecutionHandle(WrongReturnCode(jobDescriptor.key.tag, returnCode.get, jobPaths.stderr), returnCode.toOption)
-        } else {
-          handleExecutionSuccess(status, oldHandle, returnCode.get)
+        lazy val stderrLength: Try[Long] = Try(File(jobPaths.stderr).size)
+        lazy val returnCodeAsString: Try[String] = Try(File(jobPaths.returnCode).contentAsString)
+        lazy val returnCodeAsInt: Try[Int] = returnCodeAsString.map(_.trim.toInt)
+        
+        (stderrLength, returnCodeAsString, returnCodeAsInt) match {
+            // Failed to get stderr size -> Retry
+          case (Failure(exception), _, _) =>
+            jobLogger.warn(s"could not get stderr file size, retrying", exception)
+            oldHandle
+            // Failed to get return code content -> Retry
+          case (_, Failure(exception), _) =>
+            jobLogger.warn(s"could not download return code file, retrying", exception)
+            oldHandle
+            // Failed to convert return code content to Int -> Fail
+          case (_, _, Failure(exception)) =>
+            FailedNonRetryableExecutionHandle(ReturnCodeIsNotAnInt(jobDescriptor.key.tag, returnCodeAsString.get, jobPaths.stderr))
+            // Stderr is not empty and failOnStdErr is true -> Fail
+          case (Success(length), _, _) if failOnStdErr && length.intValue > 0 =>
+            FailedNonRetryableExecutionHandle(StderrNonEmpty(jobDescriptor.key.tag, length, jobPaths.stderr), returnCodeAsInt.toOption)
+            // Return code is abort code -> Abort
+          case (_, _, Success(rc)) if isAbort(rc) =>
+            AbortedExecutionHandle
+            // Return code is not valid -> Fail
+          case (_, _, Success(rc)) if !continueOnReturnCode.continueFor(rc) =>
+            FailedNonRetryableExecutionHandle(WrongReturnCode(jobDescriptor.key.tag, returnCodeAsInt.get, jobPaths.stderr), returnCodeAsInt.toOption)
+            // Otherwise -> Succeed
+          case (_, _, Success(rc)) => 
+            handleExecutionSuccess(status, oldHandle, rc)
         }
 
       } else {

From 3042e289e8ee34743452a169f9ac618ae4a9b8e2 Mon Sep 17 00:00:00 2001
From: Khalid Shakir <kshakir@broadinstitute.org>
Date: Thu, 12 Jan 2017 15:07:34 -0500
Subject: [PATCH 13/84] Added changelog message regarding webservice timeout

---
 CHANGELOG.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index cc0be7d29..e50a45b6b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,8 @@
 * Added ability to override the default zone(s) used by JES via the config structure by setting `genomics.default-zones` in the JES configuration
 * Added support for new WDL functions:
   * `length: (Array[X]) => Integer` - report the length of the specified array 
+* The cromwell server TCP binding timeout is now configurable via the config key `webservice.timeout`, defaulted to the
+  previous value `5s` (five seconds) via the reference.conf.
 
 ## 24
 

From 5054a036a3d6f6435c7ac11886f7d7aa3f8ea7a8 Mon Sep 17 00:00:00 2001
From: Jeff Gentry <geoffjentry@gmail.com>
Date: Thu, 12 Jan 2017 15:47:04 -0500
Subject: [PATCH 14/84] Move a metadata service config option to the right
 location (#1853)

---
 CHANGELOG.md                                                        | 1 +
 core/src/main/resources/reference.conf                              | 6 ++++--
 .../cromwell/services/metadata/impl/MetadataServiceActor.scala      | 2 +-
 3 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index cc0be7d29..4e952e44f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,7 @@
 
 ## 25
 
+* Moved the config value `services.MetadataService.metadata-summary-refresh-interval` to `services.MetadataService.config.metadata-summary-refresh-interval`
 * Cromwell now applies default labels automatically to JES pipeline runs.
 * Added ability to override the default zone(s) used by JES via the config structure by setting `genomics.default-zones` in the JES configuration
 * Added support for new WDL functions:
diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf
index 765042ea7..b60bac4ec 100644
--- a/core/src/main/resources/reference.conf
+++ b/core/src/main/resources/reference.conf
@@ -391,8 +391,10 @@ services {
   }
   MetadataService {
     class = "cromwell.services.metadata.impl.MetadataServiceActor"
-    # Set this value to "Inf" to turn off metadata summary refresh.  The default value is currently "2 seconds".
-    # metadata-summary-refresh-interval = "Inf"
+    config {
+      # Set this value to "Inf" to turn off metadata summary refresh.  The default value is currently "2 seconds".
+      # metadata-summary-refresh-interval = "Inf"
+    }
   }
 }
 
diff --git a/services/src/main/scala/cromwell/services/metadata/impl/MetadataServiceActor.scala b/services/src/main/scala/cromwell/services/metadata/impl/MetadataServiceActor.scala
index 2ce7c5b92..1f66a5b52 100644
--- a/services/src/main/scala/cromwell/services/metadata/impl/MetadataServiceActor.scala
+++ b/services/src/main/scala/cromwell/services/metadata/impl/MetadataServiceActor.scala
@@ -17,7 +17,7 @@ import scala.util.{Failure, Success, Try}
 object MetadataServiceActor {
 
   val MetadataSummaryRefreshInterval: Option[FiniteDuration] = {
-    val duration = Duration(ConfigFactory.load().as[Option[String]]("services.MetadataService.metadata-summary-refresh-interval").getOrElse("2 seconds"))
+    val duration = Duration(ConfigFactory.load().as[Option[String]]("services.MetadataService.config.metadata-summary-refresh-interval").getOrElse("2 seconds"))
     if (duration.isFinite()) Option(duration.asInstanceOf[FiniteDuration]) else None
   }
 

From 9f27c5e925c008a1dc78efa3131ebb8fd0b2eda9 Mon Sep 17 00:00:00 2001
From: Chris Llanwarne <cjllanwarne@gmail.com>
Date: Mon, 9 Jan 2017 11:20:41 -0500
Subject: [PATCH 15/84] Custom labels sent to JES

---
 CHANGELOG.md                                       |  4 ++
 README.md                                          | 72 ++++++++++++---------
 .../src/main/scala/cromwell/backend/backend.scala  |  7 ++-
 .../test/scala/cromwell/backend/BackendSpec.scala  |  4 +-
 .../core/WorkflowSourceFilesCollection.scala       | 30 ++++++---
 .../main/scala/cromwell/core/labels/Label.scala    | 73 ++++++++++++++++++++++
 .../main/scala/cromwell/core}/labels/Labels.scala  |  4 +-
 .../scala/cromwell/core}/labels/LabelSpec.scala    | 10 ++-
 core/src/test/scala/cromwell/util/SampleWdl.scala  |  4 +-
 .../migration/src/main/resources/changelog.xml     |  1 +
 .../changesets/workflow_store_labels_file.xml      | 17 +++++
 .../slick/tables/WorkflowStoreEntryComponent.scala |  4 +-
 .../database/sql/tables/WorkflowStoreEntry.scala   |  1 +
 engine/src/main/resources/swagger/cromwell.yaml    | 10 +++
 .../MaterializeWorkflowDescriptorActor.scala       | 39 +++++++++---
 .../workflow/workflowstore/SqlWorkflowStore.scala  | 16 ++---
 .../cromwell/webservice/CromwellApiHandler.scala   |  2 +-
 .../cromwell/webservice/CromwellApiService.scala   | 27 +++++---
 .../cromwell/webservice/WorkflowJsonSupport.scala  |  4 +-
 .../test/scala/cromwell/CromwellTestKitSpec.scala  |  3 +-
 .../test/scala/cromwell/RestartWorkflowSpec.scala  |  3 +-
 .../scala/cromwell/SimpleWorkflowActorSpec.scala   |  2 +-
 .../MaterializeWorkflowDescriptorActorSpec.scala   | 69 ++++++++++++++++----
 .../lifecycle/execution/ejea/PerTestHelper.scala   |  2 +-
 .../subworkflowstore/SubWorkflowStoreSpec.scala    |  2 +-
 src/main/scala/cromwell/CromwellCommandLine.scala  | 61 +++++++++++-------
 src/main/scala/cromwell/Main.scala                 | 10 +--
 .../main/scala/cromwell/backend/impl/jes/Run.scala | 27 ++++----
 .../cromwell/backend/impl/jes/labels/Label.scala   | 54 ----------------
 .../jes/JesAsyncBackendJobExecutionActorSpec.scala | 34 ++++++----
 .../impl/spark/SparkRuntimeAttributesSpec.scala    |  4 +-
 31 files changed, 403 insertions(+), 197 deletions(-)
 create mode 100644 core/src/main/scala/cromwell/core/labels/Label.scala
 rename {supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes => core/src/main/scala/cromwell/core}/labels/Labels.scala (86%)
 rename {supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes => core/src/test/scala/cromwell/core}/labels/LabelSpec.scala (84%)
 create mode 100644 database/migration/src/main/resources/changesets/workflow_store_labels_file.xml
 delete mode 100644 supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/labels/Label.scala

diff --git a/CHANGELOG.md b/CHANGELOG.md
index bcbb8f2dc..94edff1e4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,10 @@
   * `length: (Array[X]) => Integer` - report the length of the specified array 
 * The cromwell server TCP binding timeout is now configurable via the config key `webservice.timeout`, defaulted to the
   previous value `5s` (five seconds) via the reference.conf.
+  * `length: (Array[X]) => Integer` - report the length of the specified array.
+
+### Database schema changes
+* Added CUSTOM_LABELS as a field of WORKFLOW_STORE_ENTRY, to store workflow store entries.
 
 ## 24
 
diff --git a/README.md b/README.md
index 69c6ccc8d..f73226582 100644
--- a/README.md
+++ b/README.md
@@ -53,8 +53,6 @@ A [Workflow Management System](https://en.wikipedia.org/wiki/Workflow_management
       * [Refresh Token](#refresh-token)
     * [Docker](#docker)
     * [Monitoring](#monitoring)
-    * [Labelling Runs](#labelling-runs)
-      * [Label Escaping and Padding](#label-escaping-and-padding)
 * [Runtime Attributes](#runtime-attributes)
   * [Specifying Default Values](#specifying-default-values)
   * [continueOnReturnCode](#continueonreturncode)
@@ -68,6 +66,9 @@ A [Workflow Management System](https://en.wikipedia.org/wiki/Workflow_management
   * [preemptible](#preemptible)
 * [Logging](#logging)
 * [Workflow Options](#workflow-options)
+* [Labels](#labels)
+  * [Custom Labels File](#custom-labels-file)
+  * [Label Format](#label-format)
 * [Call Caching](#call-caching)
   * [Configuring Call Caching](#configuring-call-caching)
   * [Call Caching Workflow Options](#call-caching-workflow-options)
@@ -144,7 +145,7 @@ java -jar cromwell.jar <action> <parameters>
 
 Actions:
 run <WDL file> [<JSON inputs file>] [<JSON workflow options>]
-  [<OUTPUT workflow metadata>] [<Zip of WDL Files>]
+  [<OUTPUT workflow metadata>] [<Zip of WDL Files>] [<JSON labels file>]
 
   Given a WDL file and JSON file containing the value of the
   workflow inputs, this will run the workflow locally and
@@ -289,6 +290,8 @@ The command to run this WDL, without needing any inputs, workflow options or met
 $ java -jar cromwell.jar run threestep.wdl - - - /path/to/my_WDLs.zip
 ```
 
+The sixth optional parameter is a path to a labels file. See [Labels](#labels) for information and the expected format.
+
 ## server
 
 Start a server on port 8000, the API for the server is described in the [REST API](#rest-api) section.
@@ -1386,28 +1389,6 @@ In order to monitor metrics (CPU, Memory, Disk usage...) about the VM during Cal
 
 The output of this script will be written to a `monitoring.log` file that will be available in the call gcs bucket when the call completes.  This feature is meant to run a script in the background during long-running processes.  It's possible that if the task is very short that the log file does not flush before de-localization happens and you will end up with a zero byte file.
 
-### Labelling Runs
-
-Every call to JES from a Cromwell instance will be automatically labelled by Cromwell so that it can be queried about later. The current label set automatically applied is:
-
-| Key | Value | Example | Notes |
-|-----|-------|---------|-------|
-| cromwell-workflow-id | The Cromwell ID given to the root workflow (i.e. the ID returned by Cromwell on submission) | cromwell-d4b412c5-bf3d-4169-91b0-1b635ce47a26 | To fit the required [format](#label-escaping-and-padding), we prefix with 'cromwell-' |
-| cromwell-workflow-name | The name of the root workflow | my-root-workflow | See [format](#label-escaping-and-padding). |
-| cromwell-sub-workflow-name | The name of this job's sub-workflow | my-sub-workflow | Only if the task is called in a subworkflow, otherwise 'n-a'. See also [format](#label-escaping-and-padding). |
-| wdl-task-name | The name of the WDL task | my-task | See [format](#label-escaping-and-padding). |
-| wdl-call-name | The name of the WDL call of this job | my-call | Different from 'wdl-task-name' if it was called with an alias. See also [format](#label-escaping-and-padding). |
-
-#### Label Escaping and Padding
-
-To fit in with the Google schema for labels, label key and value strings must match the regex `[a-z]([-a-z0-9]*[a-z0-9])?` and be between 1 and 63 characters in length. For this reason, Cromwell will modify workflow/task/call names and custom labels as follows:
-
-- Any capital letters are lowercased.
-- Any character which is not one of `[a-z]`, `[0-9]` or `-` will be replaced with `-`.
-- If the start character does not match `[a-z]` then prefix with `x--`
-- If the final character does not match `[a-z0-9]` then suffix with `--x`
-- If the string is too long, only take the first 30 and last 30 characters and add `---` between them.
-
 # Runtime Attributes
 
 Runtime attributes are used to customize tasks. Within a task one can specify runtime attributes to customize the environment for the call.
@@ -1725,6 +1706,41 @@ Valid keys and their meanings:
     * **auth_bucket** - (JES backend only) defaults to the the value in **jes_gcs_root**.  This should represent a GCS URL that only Cromwell can write to.  The Cromwell account is determined by the `google.authScheme` (and the corresponding `google.userAuth` and `google.serviceAuth`)
     * **monitoring_script** - (JES backend only) Specifies a GCS URL to a script that will be invoked prior to the WDL command being run.  For example, if the value for monitoring_script is "gs://bucket/script.sh", it will be invoked as `./script.sh > monitoring.log &`.  The value `monitoring.log` file will be automatically de-localized.
 
+# Labels
+
+Every call in Cromwell is labelled by Cromwell so that it can be queried about later. The current label set automatically applied is:
+
+| Key | Value | Example | Notes |
+|-----|-------|---------|-------|
+| cromwell-workflow-id | The Cromwell ID given to the root workflow (i.e. the ID returned by Cromwell on submission) | cromwell-d4b412c5-bf3d-4169-91b0-1b635ce47a26 | To fit the required [format](#label-format), we prefix with 'cromwell-' |
+| cromwell-workflow-name | The name of the root workflow | my-root-workflow | |
+| cromwell-sub-workflow-name | The name of this job's sub-workflow | my-sub-workflow | Only present if the task is called in a subworkflow. |
+| wdl-task-name | The name of the WDL task | my-task | |
+| wdl-call-alias | The alias of the WDL call that created this job | my-task-1 | Only present if the task was called with an alias. |
+
+## Custom Labels File
+
+Custom labels can also be applied to every call in a workflow by specifying a custom labels file when the workflow is submitted. This file should be in JSON format and contain a set of fields: `"label-key": "label-value" `. For example:
+```
+{
+  "label-key-1": "label-value-1",
+  "label-key-2": "label-value-2",
+  "label-key-3": "label-value-3"
+}
+```
+
+## Label Format
+
+To fit in with the Google schema for labels, label key and value strings must match the regex `[a-z]([-a-z0-9]*[a-z0-9])?` and be between 1 and 63 characters in length. 
+
+For custom labels, Cromwell will reject any request which is made containing invalid label strings. For automatically applied labels, Cromwell will modify workflow/task/call names to fit the schema, according to the following rules:
+
+- Any capital letters are lowercased.
+- Any character which is not one of `[a-z]`, `[0-9]` or `-` will be replaced with `-`.
+- If the start character does not match `[a-z]` then prefix with `x--`
+- If the final character does not match `[a-z0-9]` then suffix with `--x`
+- If the string is too long, only take the first 30 and last 30 characters and add `---` between them.
+
 # Call Caching
 
 Call Caching allows Cromwell to detect when a job has been run in the past so it doesn't have to re-compute results.  Cromwell searches the cache of previously run jobs for a one that has the exact same command and exact same inputs.  If a previously run job is found in the cache, Cromwell will **copy the results** of the previous job instead of re-running it.
@@ -2277,11 +2293,9 @@ This endpoint accepts a POST request with a `multipart/form-data` encoded body.
 
 * `wdlSource` - *Required* Contains the WDL file to submit for execution.
 * `workflowInputs` - *Optional* JSON file containing the inputs.  A skeleton file can be generated from [wdltool](https://github.com/broadinstitute/wdltool) using the "inputs" subcommand.
-* `workflowInputs_2` - *Optional* JSON file containing the inputs.
-* `workflowInputs_3` - *Optional* JSON file containing the inputs.
-* `workflowInputs_4` - *Optional* JSON file containing the inputs.
-* `workflowInputs_5` - *Optional* JSON file containing the inputs.
+* `workflowInputs_n` - *Optional* Where `n` is an integer. JSON file containing the 'n'th set of auxiliary inputs.
 * `workflowOptions` - *Optional* JSON file containing options for this workflow execution.  See the [run](#run) CLI sub-command for some more information about this.
+* `customLabels` - *Optional* JSON file containing a set of custom labels to apply to this workflow. See [Labels](#labels) for the expected format.
 * `wdlDependencies` - *Optional* ZIP file containing WDL files that are used to resolve import statements.
 
 Regarding the workflowInputs parameter, in case of key conflicts between multiple input JSON files, higher values of x in workflowInputs_x override lower values. For example, an input specified in workflowInputs_3 will override an input with the same name in workflowInputs or workflowInputs_2.
diff --git a/backend/src/main/scala/cromwell/backend/backend.scala b/backend/src/main/scala/cromwell/backend/backend.scala
index 82e863ad0..b59b26b5d 100644
--- a/backend/src/main/scala/cromwell/backend/backend.scala
+++ b/backend/src/main/scala/cromwell/backend/backend.scala
@@ -2,6 +2,7 @@ package cromwell.backend
 
 import com.typesafe.config.Config
 import cromwell.core.WorkflowOptions.WorkflowOption
+import cromwell.core.labels.Labels
 import cromwell.core.{CallKey, WorkflowId, WorkflowOptions}
 import wdl4s._
 import wdl4s.values.WdlValue
@@ -34,8 +35,9 @@ object BackendWorkflowDescriptor {
   def apply(id: WorkflowId,
             workflow: Workflow,
             knownValues: Map[FullyQualifiedName, WdlValue],
-            workflowOptions: WorkflowOptions) = {
-    new BackendWorkflowDescriptor(id, workflow, knownValues, workflowOptions, List.empty)
+            workflowOptions: WorkflowOptions,
+            customLabels: Labels) = {
+    new BackendWorkflowDescriptor(id, workflow, knownValues, workflowOptions, customLabels, List.empty)
   }
 }
 
@@ -46,6 +48,7 @@ case class BackendWorkflowDescriptor(id: WorkflowId,
                                      workflow: Workflow,
                                      knownValues: Map[FullyQualifiedName, WdlValue],
                                      workflowOptions: WorkflowOptions,
+                                     customLabels: Labels,
                                      breadCrumbs: List[BackendJobBreadCrumb]) {
   
   val rootWorkflow = breadCrumbs.headOption.map(_.workflow).getOrElse(workflow)
diff --git a/backend/src/test/scala/cromwell/backend/BackendSpec.scala b/backend/src/test/scala/cromwell/backend/BackendSpec.scala
index 30bcc4a79..aec7c8ade 100644
--- a/backend/src/test/scala/cromwell/backend/BackendSpec.scala
+++ b/backend/src/test/scala/cromwell/backend/BackendSpec.scala
@@ -3,6 +3,7 @@ package cromwell.backend
 import com.typesafe.config.ConfigFactory
 import cromwell.backend.BackendJobExecutionActor.{BackendJobExecutionResponse, JobFailedNonRetryableResponse, JobFailedRetryableResponse, JobSucceededResponse}
 import cromwell.backend.io.TestWorkflows._
+import cromwell.core.labels.Labels
 import cromwell.core.{WorkflowId, WorkflowOptions}
 import lenthall.exception.AggregatedException
 import org.scalatest.Matchers
@@ -30,7 +31,8 @@ trait BackendSpec extends ScalaFutures with Matchers with Mockito {
       WorkflowId.randomId(),
       WdlNamespaceWithWorkflow.load(wdl.replaceAll("RUNTIME", runtime), Seq.empty[ImportResolver]).workflow,
       inputs,
-      options
+      options,
+      Labels.empty
     )
   }
 
diff --git a/core/src/main/scala/cromwell/core/WorkflowSourceFilesCollection.scala b/core/src/main/scala/cromwell/core/WorkflowSourceFilesCollection.scala
index ac7a7d6aa..55fa68b47 100644
--- a/core/src/main/scala/cromwell/core/WorkflowSourceFilesCollection.scala
+++ b/core/src/main/scala/cromwell/core/WorkflowSourceFilesCollection.scala
@@ -10,14 +10,27 @@ sealed trait WorkflowSourceFilesCollection {
   def wdlSource: WdlSource
   def inputsJson: WdlJson
   def workflowOptionsJson: WorkflowOptionsJson
+  def labelsJson: WdlJson
+
+
   def importsZipFileOption: Option[Array[Byte]] = this match {
     case _: WorkflowSourceFilesWithoutImports => None
-    case WorkflowSourceFilesWithDependenciesZip(_, _, _, importsZip) => Option(importsZip) // i.e. Some(importsZip) if our wiring is correct
+    case WorkflowSourceFilesWithDependenciesZip(_, _, _, _, importsZip) => Option(importsZip) // i.e. Some(importsZip) if our wiring is correct
   }
 
   def copyOptions(workflowOptions: WorkflowOptionsJson) = this match {
-    case w: WorkflowSourceFilesWithoutImports => WorkflowSourceFilesWithoutImports(w.wdlSource, w.inputsJson, workflowOptions)
-    case w: WorkflowSourceFilesWithDependenciesZip => WorkflowSourceFilesWithDependenciesZip(w.wdlSource, w.inputsJson, workflowOptions, w.importsZip)
+    case w: WorkflowSourceFilesWithoutImports => WorkflowSourceFilesWithoutImports(
+      wdlSource = w.wdlSource,
+      inputsJson = w.inputsJson,
+      workflowOptionsJson = workflowOptions,
+      labelsJson = w.labelsJson)
+
+    case w: WorkflowSourceFilesWithDependenciesZip => WorkflowSourceFilesWithDependenciesZip(
+      wdlSource = w.wdlSource,
+      inputsJson = w.inputsJson,
+      workflowOptionsJson = workflowOptions,
+      labelsJson = w.labelsJson,
+      importsZip = w.importsZip)
   }
 }
 
@@ -25,19 +38,22 @@ object WorkflowSourceFilesCollection {
   def apply(wdlSource: WdlSource,
             inputsJson: WdlJson,
             workflowOptionsJson: WorkflowOptionsJson,
+            labelsJson: WdlJson,
             importsFile: Option[Array[Byte]]): WorkflowSourceFilesCollection = importsFile match {
-    case Some(imports) => WorkflowSourceFilesWithDependenciesZip(wdlSource, inputsJson, workflowOptionsJson, imports)
-    case None => WorkflowSourceFilesWithoutImports(wdlSource, inputsJson, workflowOptionsJson)
+    case Some(imports) => WorkflowSourceFilesWithDependenciesZip(wdlSource, inputsJson, workflowOptionsJson, labelsJson, imports)
+    case None => WorkflowSourceFilesWithoutImports(wdlSource, inputsJson, workflowOptionsJson, labelsJson)
   }
 }
 
 final case class WorkflowSourceFilesWithoutImports(wdlSource: WdlSource,
                                                    inputsJson: WdlJson,
-                                                   workflowOptionsJson: WorkflowOptionsJson) extends WorkflowSourceFilesCollection
+                                                   workflowOptionsJson: WorkflowOptionsJson,
+                                                   labelsJson: WdlJson) extends WorkflowSourceFilesCollection
 
 final case class WorkflowSourceFilesWithDependenciesZip(wdlSource: WdlSource,
                                                         inputsJson: WdlJson,
                                                         workflowOptionsJson: WorkflowOptionsJson,
+                                                        labelsJson: WdlJson,
                                                         importsZip: Array[Byte]) extends WorkflowSourceFilesCollection {
-  override def toString = s"WorkflowSourceFilesWithDependenciesZip($wdlSource, $inputsJson, $workflowOptionsJson, <<ZIP BINARY CONTENT>>)"
+  override def toString = s"WorkflowSourceFilesWithDependenciesZip($wdlSource, $inputsJson, $workflowOptionsJson, $labelsJson, <<ZIP BINARY CONTENT>>)"
 }
diff --git a/core/src/main/scala/cromwell/core/labels/Label.scala b/core/src/main/scala/cromwell/core/labels/Label.scala
new file mode 100644
index 000000000..4fc1e2959
--- /dev/null
+++ b/core/src/main/scala/cromwell/core/labels/Label.scala
@@ -0,0 +1,73 @@
+package cromwell.core.labels
+
+import lenthall.validation.ErrorOr.ErrorOr
+import cats.data.Validated._
+import cats.syntax.cartesian._
+import cats.syntax.validated._
+
+sealed abstract case class Label(key: String, value: String)
+
+object Label {
+
+  // Yes, 63. Not a typo for 64.
+  // See 'labels' in https://cloud.google.com/genomics/reference/rpc/google.genomics.v1alpha2#google.genomics.v1alpha2.RunPipelineArgs
+  private val MaxLabelLength = 63
+  val LabelRegexPattern = "[a-z]([-a-z0-9]*[a-z0-9])?"
+
+  def validateName(s: String): ErrorOr[String] = {
+    if (LabelRegexPattern.r.pattern.matcher(s).matches) {
+      if (s.length <= MaxLabelLength) s.validNel else s"Invalid label: $s was ${s.length} characters. The maximum is $MaxLabelLength".invalidNel
+    } else {
+      s"Invalid label: $s did not match the regex $LabelRegexPattern".invalidNel
+    }
+  }
+
+  def validateLabel(key: String, value: String): ErrorOr[Label] = {
+    val validatedKey = validateName(key)
+    val validatedValue = validateName(value)
+
+    (validatedKey |@| validatedValue) map { case (k, v) => new Label(k, v) {} }
+  }
+
+  /**
+    * Change to meet the constraint:
+    *  - To match the regex LabelRegexPattern
+    *  - To be between 1 and MaxLabelLength characters total
+    */
+  def safeName(mainText: String): String = {
+
+    validateName(mainText) match {
+      case Valid(labelText) => labelText
+      case _ =>
+        def appendSafe(current: String, nextChar: Char): String = {
+          nextChar match {
+            case c if c.isLetterOrDigit || c == '-' => current + c.toLower
+            case _ => current + '-'
+          }
+        }
+
+        val foldResult = mainText.toCharArray.foldLeft("")(appendSafe)
+
+        val startsValid = foldResult.headOption.exists(_.isLetter)
+        val endsValid = foldResult.lastOption.exists(_.isLetterOrDigit)
+
+        val validStart = if (startsValid) foldResult else "x--" + foldResult
+        val validStartAndEnd = if (endsValid) validStart else validStart + "--x"
+
+        val length = validStartAndEnd.length
+        val tooLong = length > MaxLabelLength
+
+        if (tooLong) {
+          val middleSeparator = "---"
+          val subSectionLength = (MaxLabelLength - middleSeparator.length) / 2
+          validStartAndEnd.substring(0, subSectionLength) + middleSeparator + validStartAndEnd.substring(length - subSectionLength, length)
+        } else {
+          validStartAndEnd
+        }
+    }
+  }
+
+  def safeLabel(key: String, value: String): Label = {
+    new Label(safeName(key), safeName(value)) {}
+  }
+}
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/labels/Labels.scala b/core/src/main/scala/cromwell/core/labels/Labels.scala
similarity index 86%
rename from supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/labels/Labels.scala
rename to core/src/main/scala/cromwell/core/labels/Labels.scala
index 6932e1fc8..8e891fb51 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/labels/Labels.scala
+++ b/core/src/main/scala/cromwell/core/labels/Labels.scala
@@ -1,4 +1,4 @@
-package cromwell.backend.impl.jes.labels
+package cromwell.core.labels
 
 import scala.collection.JavaConverters._
 
@@ -14,4 +14,6 @@ object Labels {
     val kvps: Seq[(String, String)] = values.toSeq
     Labels((kvps map { case (k, v) => Label.safeLabel(k, v) }).to[Vector])
   }
+
+  def empty = Labels(Vector.empty)
 }
diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/labels/LabelSpec.scala b/core/src/test/scala/cromwell/core/labels/LabelSpec.scala
similarity index 84%
rename from supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/labels/LabelSpec.scala
rename to core/src/test/scala/cromwell/core/labels/LabelSpec.scala
index 41c19140b..47060b6d6 100644
--- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/labels/LabelSpec.scala
+++ b/core/src/test/scala/cromwell/core/labels/LabelSpec.scala
@@ -1,5 +1,6 @@
-package cromwell.backend.impl.jes.labels
+package cromwell.core.labels
 
+import cats.data.Validated.{Invalid, Valid}
 import org.scalatest.{FlatSpec, Matchers}
 
 class LabelSpec extends FlatSpec with Matchers {
@@ -29,13 +30,16 @@ class LabelSpec extends FlatSpec with Matchers {
 
   goodLabelStrings foreach { label =>
     it should s"validate the good label string '$label'" in {
-      Label.validate(label) should be(true)
+      Label.validateName(label) should be(Valid(label))
     }
   }
 
   badLabelConversions foreach { case (label: String, conversion: String) =>
     it should s"not validate the bad label string '$label'" in {
-      Label.validate(label) should be(false)
+      Label.validateName(label) match {
+        case Invalid(_) => // Good!
+        case Valid(_) => fail(s"Label validation succeeded but should have failed.")
+      }
     }
 
     it should s"convert the bad label string '$label' into the safe label string '$conversion'" in {
diff --git a/core/src/test/scala/cromwell/util/SampleWdl.scala b/core/src/test/scala/cromwell/util/SampleWdl.scala
index 2e4d77399..b02b6db9d 100644
--- a/core/src/test/scala/cromwell/util/SampleWdl.scala
+++ b/core/src/test/scala/cromwell/util/SampleWdl.scala
@@ -14,8 +14,8 @@ import scala.language.postfixOps
 
 trait SampleWdl extends TestFileUtil {
   def wdlSource(runtime: String = ""): WdlSource
-  def asWorkflowSources(runtime: String = "", workflowOptions: String = "{}") =
-    WorkflowSourceFilesWithoutImports(wdlSource(runtime), wdlJson, workflowOptions)
+  def asWorkflowSources(runtime: String = "", workflowOptions: String = "{}", labels: String = "{}") =
+    WorkflowSourceFilesWithoutImports(wdlSource = wdlSource(runtime), inputsJson = wdlJson, workflowOptionsJson = workflowOptions, labelsJson = labels)
   val rawInputs: WorkflowRawInputs
 
   def name = getClass.getSimpleName.stripSuffix("$")
diff --git a/database/migration/src/main/resources/changelog.xml b/database/migration/src/main/resources/changelog.xml
index c67137ae0..1deb8d50c 100644
--- a/database/migration/src/main/resources/changelog.xml
+++ b/database/migration/src/main/resources/changelog.xml
@@ -54,6 +54,7 @@
     <include file="changesets/encrypt_and_clear_workflow_options.xml" relativeToChangelogFile="true" />
     <include file="changesets/sub_workflow_store.xml" relativeToChangelogFile="true" />
     <include file="changesets/workflow_store_imports_file.xml" relativeToChangelogFile="true" />
+    <include file="changesets/workflow_store_labels_file.xml" relativeToChangelogFile="true" />
 </databaseChangeLog>
 <!--
 
diff --git a/database/migration/src/main/resources/changesets/workflow_store_labels_file.xml b/database/migration/src/main/resources/changesets/workflow_store_labels_file.xml
new file mode 100644
index 000000000..225c13b78
--- /dev/null
+++ b/database/migration/src/main/resources/changesets/workflow_store_labels_file.xml
@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
+
+    <property name="clob.type" value="LONGTEXT" dbms="mysql"/>
+    <property name="clob.type" value="LONGVARCHAR" dbms="hsqldb"/>
+
+    <changeSet author="cjllanwarne" id="workflow-store-labels-file">
+        <addColumn
+                tableName="WORKFLOW_STORE_ENTRY">
+            <column name="CUSTOM_LABELS" type="${clob.type}">
+                <constraints nullable="false"/>
+            </column>
+        </addColumn>
+    </changeSet>
+</databaseChangeLog>
diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowStoreEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowStoreEntryComponent.scala
index dc896ca3d..5f4df4216 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowStoreEntryComponent.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowStoreEntryComponent.scala
@@ -21,6 +21,8 @@ trait WorkflowStoreEntryComponent {
 
     def workflowOptions = column[Clob]("WORKFLOW_OPTIONS")
 
+    def customLabels = column[Clob]("CUSTOM_LABELS")
+
     def workflowState = column[String]("WORKFLOW_STATE")
 
     def submissionTime = column[Timestamp]("SUBMISSION_TIME")
@@ -28,7 +30,7 @@ trait WorkflowStoreEntryComponent {
     def importsZipFile = column[Option[Blob]]("IMPORTS_ZIP")
 
     override def * = (workflowExecutionUuid, workflowDefinition, workflowInputs, workflowOptions, workflowState,
-      submissionTime, importsZipFile, workflowStoreEntryId.?) <> (WorkflowStoreEntry.tupled, WorkflowStoreEntry.unapply)
+      submissionTime, importsZipFile, customLabels, workflowStoreEntryId.?) <> (WorkflowStoreEntry.tupled, WorkflowStoreEntry.unapply)
 
     def ucWorkflowStoreEntryWeu = index("UC_WORKFLOW_STORE_ENTRY_WEU", workflowExecutionUuid, unique = true)
 
diff --git a/database/sql/src/main/scala/cromwell/database/sql/tables/WorkflowStoreEntry.scala b/database/sql/src/main/scala/cromwell/database/sql/tables/WorkflowStoreEntry.scala
index 66cd80085..fe9387582 100644
--- a/database/sql/src/main/scala/cromwell/database/sql/tables/WorkflowStoreEntry.scala
+++ b/database/sql/src/main/scala/cromwell/database/sql/tables/WorkflowStoreEntry.scala
@@ -11,5 +11,6 @@ case class WorkflowStoreEntry
   workflowState: String,
   submissionTime: Timestamp,
   importsZipFile: Option[Blob],
+  customLabels: Clob,
   workflowStoreEntryId: Option[Int] = None
 )
diff --git a/engine/src/main/resources/swagger/cromwell.yaml b/engine/src/main/resources/swagger/cromwell.yaml
index 613ce5e7e..f634f81a0 100644
--- a/engine/src/main/resources/swagger/cromwell.yaml
+++ b/engine/src/main/resources/swagger/cromwell.yaml
@@ -93,6 +93,11 @@ paths:
           required: false
           type: file
           in: formData
+        - name: customLabels
+          description: Custom Labels JSON
+          required: false
+          type: file
+          in: formData
         - name: wdlDependencies
           description: Workflow Options JSON
           required: false
@@ -139,6 +144,11 @@ paths:
           required: false
           type: file
           in: formData
+        - name: customLabels
+          description: Custom Labels JSON
+          required: false
+          type: file
+          in: formData
         - name: wdlDependencies
           description: Workflow Options JSON
           required: false
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala
index d8b2a6a00..ff415892f 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala
@@ -7,10 +7,10 @@ import better.files.File
 import cats.data.NonEmptyList
 import cats.data.Validated._
 import cats.instances.list._
+import cats.instances.vector._
 import cats.syntax.cartesian._
 import cats.syntax.traverse._
 import cats.syntax.validated._
-import cromwell.core._
 import com.typesafe.config.Config
 import com.typesafe.scalalogging.LazyLogging
 import cromwell.backend.BackendWorkflowDescriptor
@@ -18,6 +18,7 @@ import cromwell.core.Dispatcher.EngineDispatcher
 import cromwell.core.WorkflowOptions.{ReadFromCache, WorkflowOption, WriteToCache}
 import cromwell.core._
 import cromwell.core.callcaching._
+import cromwell.core.labels.{Label, Labels}
 import cromwell.core.logging.WorkflowLogging
 import cromwell.core.path.PathBuilder
 import cromwell.engine._
@@ -171,12 +172,13 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef,
                                       conf: Config): ErrorOr[EngineWorkflowDescriptor] = {
     val namespaceValidation = validateNamespace(sourceFiles)
     val workflowOptionsValidation = validateWorkflowOptions(sourceFiles.workflowOptionsJson)
-    (namespaceValidation |@| workflowOptionsValidation) map {
-      (_, _)
-    } flatMap { case (namespace, workflowOptions) =>
+    val labelsValidation = validateLabels(sourceFiles.labelsJson)
+    (namespaceValidation |@| workflowOptionsValidation |@| labelsValidation) map {
+      (_, _, _)
+    } flatMap { case (namespace, workflowOptions, labels) =>
       pushWfNameMetadataService(namespace.workflow.unqualifiedName)
       val pathBuilders = EngineFilesystems(context.system).pathBuildersForWorkflow(workflowOptions)
-      buildWorkflowDescriptor(id, sourceFiles, namespace, workflowOptions, conf, pathBuilders)
+      buildWorkflowDescriptor(id, sourceFiles, namespace, workflowOptions, labels, conf, pathBuilders)
     }
   }
 
@@ -192,18 +194,20 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef,
                                       sourceFiles: WorkflowSourceFilesCollection,
                                       namespace: WdlNamespaceWithWorkflow,
                                       workflowOptions: WorkflowOptions,
+                                      labels: Labels,
                                       conf: Config,
                                       pathBuilders: List[PathBuilder]): ErrorOr[EngineWorkflowDescriptor] = {
     val defaultBackendName = conf.as[Option[String]]("backend.default")
     val rawInputsValidation = validateRawInputs(sourceFiles.inputsJson)
+
     val failureModeValidation = validateWorkflowFailureMode(workflowOptions, conf)
     val backendAssignmentsValidation = validateBackendAssignments(namespace.taskCalls, workflowOptions, defaultBackendName)
     val callCachingModeValidation = validateCallCachingMode(workflowOptions, conf)
 
-    (rawInputsValidation |@| failureModeValidation |@| backendAssignmentsValidation |@| callCachingModeValidation ) map {
+    (rawInputsValidation |@| failureModeValidation |@| backendAssignmentsValidation |@| callCachingModeValidation) map {
       (_, _, _, _)
     } flatMap { case (rawInputs, failureMode, backendAssignments, callCachingMode) =>
-      buildWorkflowDescriptor(id, namespace, rawInputs, backendAssignments, workflowOptions, failureMode, pathBuilders, callCachingMode)
+      buildWorkflowDescriptor(id, namespace, rawInputs, backendAssignments, workflowOptions, labels, failureMode, pathBuilders, callCachingMode)
     }
   }
 
@@ -212,6 +216,7 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef,
                                       rawInputs: Map[String, JsValue],
                                       backendAssignments: Map[TaskCall, String],
                                       workflowOptions: WorkflowOptions,
+                                      labels: Labels,
                                       failureMode: WorkflowFailureMode,
                                       pathBuilders: List[PathBuilder],
                                       callCachingMode: CallCachingMode): ErrorOr[EngineWorkflowDescriptor] = {
@@ -235,7 +240,7 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef,
       _ = pushWfInputsToMetadataService(coercedInputs)
       evaluatedWorkflowsDeclarations <- validateDeclarations(namespace, workflowOptions, coercedInputs, pathBuilders)
       declarationsAndInputs <- checkTypes(evaluatedWorkflowsDeclarations ++ coercedInputs)
-      backendDescriptor = BackendWorkflowDescriptor(id, namespace.workflow, declarationsAndInputs, workflowOptions)
+      backendDescriptor = BackendWorkflowDescriptor(id, namespace.workflow, declarationsAndInputs, workflowOptions, labels)
     } yield EngineWorkflowDescriptor(namespace, backendDescriptor, backendAssignments, failureMode, pathBuilders, callCachingMode)
   }
 
@@ -391,6 +396,24 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef,
     }
   }
 
+  private def validateLabels(json: WdlJson): ErrorOr[Labels] = {
+
+    def toLabels(inputs: Map[String, JsValue]): ErrorOr[Labels] = {
+      val vectorOfValidatedLabel: Vector[ErrorOr[Label]] = inputs.toVector map {
+        case (key, JsString(s)) => Label.validateLabel(key, s)
+        case (key, other) => s"Invalid label '$key: $other': Labels must be strings, and must match the regex ${Label.LabelRegexPattern}".invalidNel
+      }
+
+      vectorOfValidatedLabel.sequence[ErrorOr, Label] map { validatedVectorofLabel => Labels(validatedVectorofLabel) }
+    }
+
+    Try(json.parseJson) match {
+      case Success(JsObject(inputs)) => toLabels(inputs)
+      case Failure(reason: Throwable) => s"Workflow contains invalid labels JSON: ${reason.getMessage}".invalidNel
+      case _ => """Invalid workflow labels JSON. Expected a JsObject of "labelKey": "labelValue" values.""".invalidNel
+    }
+  }
+
   private def validateCoercedInputs(rawInputs: Map[String, JsValue],
                                     namespace: WdlNamespaceWithWorkflow): ErrorOr[WorkflowCoercedInputs] = {
     namespace.coerceRawInputs(rawInputs) match {
diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStore.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStore.scala
index 29190617e..d527378e3 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStore.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStore.scala
@@ -57,6 +57,7 @@ case class SqlWorkflowStore(sqlDatabase: WorkflowStoreSqlDatabase) extends Workf
       workflowStoreEntry.workflowDefinition.toRawString,
       workflowStoreEntry.workflowInputs.toRawString,
       workflowStoreEntry.workflowOptions.toRawString,
+      workflowStoreEntry.customLabels.toRawString,
       workflowStoreEntry.importsZipFile.map(b => b.getBytes(1, b.length.asInstanceOf[Int]))
     )
     WorkflowToStart(
@@ -67,13 +68,14 @@ case class SqlWorkflowStore(sqlDatabase: WorkflowStoreSqlDatabase) extends Workf
 
   private def toWorkflowStoreEntry(workflowSourceFiles: WorkflowSourceFilesCollection): WorkflowStoreEntry = {
     WorkflowStoreEntry(
-      WorkflowId.randomId().toString,
-      workflowSourceFiles.wdlSource.toClob,
-      workflowSourceFiles.inputsJson.toClob,
-      workflowSourceFiles.workflowOptionsJson.toClob,
-      WorkflowStoreState.Submitted.toString,
-      OffsetDateTime.now.toSystemTimestamp,
-      workflowSourceFiles.importsZipFileOption.map(new SerialBlob(_))
+      workflowExecutionUuid = WorkflowId.randomId().toString,
+      workflowDefinition = workflowSourceFiles.wdlSource.toClob,
+      workflowInputs = workflowSourceFiles.inputsJson.toClob,
+      workflowOptions = workflowSourceFiles.workflowOptionsJson.toClob,
+      customLabels = workflowSourceFiles.labelsJson.toClob,
+      workflowState = WorkflowStoreState.Submitted.toString,
+      submissionTime = OffsetDateTime.now.toSystemTimestamp,
+      importsZipFile = workflowSourceFiles.importsZipFileOption.map(new SerialBlob(_))
     )
   }
 
diff --git a/engine/src/main/scala/cromwell/webservice/CromwellApiHandler.scala b/engine/src/main/scala/cromwell/webservice/CromwellApiHandler.scala
index 8afd3fafa..7fb9e87e2 100644
--- a/engine/src/main/scala/cromwell/webservice/CromwellApiHandler.scala
+++ b/engine/src/main/scala/cromwell/webservice/CromwellApiHandler.scala
@@ -68,7 +68,7 @@ class CromwellApiHandler(requestHandlerActor: ActorRef) extends Actor with Workf
       context.parent ! RequestComplete((StatusCodes.Created, WorkflowSubmitResponse(id.toString, WorkflowSubmitted.toString)))
 
     case ApiHandlerWorkflowSubmitBatch(sources) => requestHandlerActor !
-      WorkflowStoreActor.BatchSubmitWorkflows(sources.map(x => WorkflowSourceFilesCollection(x.wdlSource,x.inputsJson,x.workflowOptionsJson,x.importsZipFileOption)))
+      WorkflowStoreActor.BatchSubmitWorkflows(sources.map(x => WorkflowSourceFilesCollection(x.wdlSource, x.inputsJson, x.workflowOptionsJson, x.labelsJson, x.importsZipFileOption)))
 
 
     case WorkflowStoreActor.WorkflowsBatchSubmittedToStore(ids) =>
diff --git a/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala b/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala
index 34113fd55..b11521434 100644
--- a/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala
+++ b/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala
@@ -127,34 +127,41 @@ trait CromwellApiService extends HttpService with PerRequestCreator {
       }
     }
 
-  case class PartialWorkflowSources(wdlSource: Option[WdlSource], workflowInputs: Seq[WdlJson], workflowInputsAux: Map[Int, WdlJson], workflowOptions: Option[WorkflowOptionsJson], zippedImports: Option[Array[Byte]])
+  case class PartialWorkflowSources
+  (
+    wdlSource: Option[WdlSource],
+    workflowInputs: Vector[WdlJson],
+    workflowInputsAux: Map[Int, WdlJson],
+    workflowOptions: Option[WorkflowOptionsJson],
+    customLabels: Option[WdlJson],
+    zippedImports: Option[Array[Byte]])
   object PartialWorkflowSources {
-    private def workflowInputs(bodyPart: BodyPart): Seq[WdlJson] = {
+    private def workflowInputs(bodyPart: BodyPart): Vector[WdlJson] = {
       import spray.json._
       bodyPart.entity.data.asString.parseJson match {
-        case JsArray(Seq(x, xs@_*)) => (List(x) ++ xs).map(_.compactPrint)
-        case JsArray(_) => Seq.empty
-        case v: JsValue => Seq(v.compactPrint)
+        case JsArray(Seq(x, xs@_*)) => (Vector(x) ++ xs).map(_.compactPrint)
+        case JsArray(_) => Vector.empty
+        case v: JsValue => Vector(v.compactPrint)
       }
     }
 
     def partialSourcesToSourceCollections(partialSources: Try[PartialWorkflowSources], allowNoInputs: Boolean): Try[Seq[WorkflowSourceFilesCollection]] = {
       partialSources flatMap {
-        case PartialWorkflowSources(Some(wdlSource), workflowInputs, workflowInputsAux, workflowOptions, wdlDependencies) =>
+        case PartialWorkflowSources(Some(wdlSource), workflowInputs, workflowInputsAux, workflowOptions, labels, wdlDependencies) =>
           //The order of addition allows for the expected override of colliding keys.
           val sortedInputAuxes = workflowInputsAux.toSeq.sortBy(_._1).map(x => Option(x._2))
           val wfInputs: Try[Seq[WdlJson]] = if (workflowInputs.isEmpty) {
-            if (allowNoInputs) Success(Seq("{}")) else Failure(new IllegalArgumentException("No inputs were provided"))
+            if (allowNoInputs) Success(Vector("{}")) else Failure(new IllegalArgumentException("No inputs were provided"))
           } else Success(workflowInputs map { workflowInputSet =>
             mergeMaps(Seq(Option(workflowInputSet)) ++ sortedInputAuxes).toString
           })
-          wfInputs.map(_.map(x => WorkflowSourceFilesCollection(wdlSource, x, workflowOptions.getOrElse("{}"), wdlDependencies)))
+          wfInputs.map(_.map(x => WorkflowSourceFilesCollection(wdlSource, x, workflowOptions.getOrElse("{}"), labels.getOrElse("{}"), wdlDependencies)))
         case other => Failure(new IllegalArgumentException(s"Incomplete workflow submission: $other"))
       }
     }
 
     def fromSubmitRoute(formData: MultipartFormData, allowNoInputs: Boolean): Try[Seq[WorkflowSourceFilesCollection]] = {
-      val partialSources = Try(formData.fields.foldLeft(PartialWorkflowSources(None, Seq.empty, Map.empty, None, None)) { (partialSources: PartialWorkflowSources, bodyPart: BodyPart) =>
+      val partialSources = Try(formData.fields.foldLeft(PartialWorkflowSources(None, Vector.empty, Map.empty, None, None, None)) { (partialSources: PartialWorkflowSources, bodyPart: BodyPart) =>
         if (bodyPart.name.contains("wdlSource")) {
           partialSources.copy(wdlSource = Some(bodyPart.entity.data.asString))
         } else if (bodyPart.name.contains("workflowInputs")) {
@@ -166,6 +173,8 @@ trait CromwellApiService extends HttpService with PerRequestCreator {
           partialSources.copy(workflowOptions = Some(bodyPart.entity.data.asString))
         } else if (bodyPart.name.contains("wdlDependencies")) {
           partialSources.copy(zippedImports = Some(bodyPart.entity.data.toByteArray))
+        } else if (bodyPart.name.contains("customLabels")) {
+          partialSources.copy(customLabels = Some(bodyPart.entity.data.asString))
         } else {
           throw new IllegalArgumentException(s"Unexpected body part name: ${bodyPart.name.getOrElse("None")}")
         }
diff --git a/engine/src/main/scala/cromwell/webservice/WorkflowJsonSupport.scala b/engine/src/main/scala/cromwell/webservice/WorkflowJsonSupport.scala
index b2afe311f..817cb8dcf 100644
--- a/engine/src/main/scala/cromwell/webservice/WorkflowJsonSupport.scala
+++ b/engine/src/main/scala/cromwell/webservice/WorkflowJsonSupport.scala
@@ -20,7 +20,7 @@ object WorkflowJsonSupport extends DefaultJsonProtocol {
   implicit val callOutputResponseProtocol = jsonFormat3(CallOutputResponse)
   implicit val engineStatsProtocol = jsonFormat2(EngineStatsActor.EngineStats)
   implicit val callAttempt = jsonFormat2(CallAttempt)
-  implicit val workflowSourceData = jsonFormat3(WorkflowSourceFilesWithoutImports)
+  implicit val workflowSourceData = jsonFormat4(WorkflowSourceFilesWithoutImports)
 
   implicit object fileJsonFormat extends RootJsonFormat[File] {
     override def write(obj: File) = JsString(obj.path.toAbsolutePath.toString)
@@ -30,7 +30,7 @@ object WorkflowJsonSupport extends DefaultJsonProtocol {
     }
   }
 
-  implicit val workflowSourceDataWithImports = jsonFormat4(WorkflowSourceFilesWithDependenciesZip)
+  implicit val workflowSourceDataWithImports = jsonFormat5(WorkflowSourceFilesWithDependenciesZip)
   implicit val errorResponse = jsonFormat3(FailureResponse)
   implicit val successResponse = jsonFormat3(SuccessResponse)
 
diff --git a/engine/src/test/scala/cromwell/CromwellTestKitSpec.scala b/engine/src/test/scala/cromwell/CromwellTestKitSpec.scala
index c30488027..845f0b9c5 100644
--- a/engine/src/test/scala/cromwell/CromwellTestKitSpec.scala
+++ b/engine/src/test/scala/cromwell/CromwellTestKitSpec.scala
@@ -326,11 +326,12 @@ abstract class CromwellTestKitSpec(val twms: TestWorkflowManagerSystem = new Cro
   def runWdl(sampleWdl: SampleWdl,
              runtime: String = "",
              workflowOptions: String = "{}",
+             customLabels: String = "{}",
              terminalState: WorkflowState = WorkflowSucceeded,
              config: Config = DefaultConfig,
              patienceConfig: PatienceConfig = defaultPatience)(implicit ec: ExecutionContext): Map[FullyQualifiedName, WdlValue] = {
     val rootActor = buildCromwellRootActor(config)
-    val sources = WorkflowSourceFilesWithoutImports(sampleWdl.wdlSource(runtime), sampleWdl.wdlJson, workflowOptions)
+    val sources = WorkflowSourceFilesWithoutImports(sampleWdl.wdlSource(runtime), sampleWdl.wdlJson, workflowOptions, customLabels)
     val workflowId = rootActor.underlyingActor.submitWorkflow(sources)
     eventually { verifyWorkflowState(rootActor.underlyingActor.serviceRegistryActor, workflowId, terminalState) } (config = patienceConfig, pos = implicitly[org.scalactic.source.Position])
     val outcome = getWorkflowOutputsFromMetadata(workflowId, rootActor.underlyingActor.serviceRegistryActor)
diff --git a/engine/src/test/scala/cromwell/RestartWorkflowSpec.scala b/engine/src/test/scala/cromwell/RestartWorkflowSpec.scala
index 5db1f3ebe..c8f701d4d 100644
--- a/engine/src/test/scala/cromwell/RestartWorkflowSpec.scala
+++ b/engine/src/test/scala/cromwell/RestartWorkflowSpec.scala
@@ -18,7 +18,8 @@ class RestartWorkflowSpec extends CromwellTestKitSpec with WorkflowDescriptorBui
                 |}
               """.stripMargin,
     inputsJson="{}",
-    workflowOptionsJson="{}"
+    workflowOptionsJson="{}",
+    labelsJson="{}"
   )
 
   "RestartWorkflowSpec" should {
diff --git a/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala b/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala
index 29e382623..8eb60d45f 100644
--- a/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala
+++ b/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala
@@ -34,7 +34,7 @@ class SimpleWorkflowActorSpec extends CromwellTestKitSpec with BeforeAndAfter {
                                  rawInputsOverride: String,
                                  workflowId: WorkflowId,
                                  matchers: Matcher*): TestableWorkflowActorAndMetadataPromise = {
-    val workflowSources = WorkflowSourceFilesWithoutImports(sampleWdl.wdlSource(), rawInputsOverride, "{}")
+    val workflowSources = WorkflowSourceFilesWithoutImports(sampleWdl.wdlSource(), rawInputsOverride, "{}", "{}")
     val promise = Promise[Unit]()
     val watchActor = system.actorOf(MetadataWatchActor.props(promise, matchers: _*), s"service-registry-$workflowId-${UUID.randomUUID()}")
     val supervisor = TestProbe()
diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActorSpec.scala
index 06f3be008..8ae601d85 100644
--- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActorSpec.scala
+++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActorSpec.scala
@@ -4,6 +4,7 @@ import akka.actor.Props
 import akka.testkit.TestDuration
 import com.typesafe.config.ConfigFactory
 import cromwell.CromwellTestKitSpec
+import cromwell.core.labels.{Label, Labels}
 import cromwell.core.{WorkflowId, WorkflowOptions, WorkflowSourceFilesWithoutImports}
 import cromwell.engine.backend.{BackendConfigurationEntry, CromwellBackends}
 import cromwell.engine.workflow.lifecycle.MaterializeWorkflowDescriptorActor.{MaterializeWorkflowDescriptorCommand, MaterializeWorkflowDescriptorFailureResponse, MaterializeWorkflowDescriptorSuccessResponse}
@@ -38,6 +39,8 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitSpec with Be
     """.stripMargin)
   val unstructuredFile = "fubar badness!"
   val validOptionsFile =""" { "write_to_cache": "true" } """
+  val validCustomLabelsFile="""{ "label1": "value1", "label2": "value2" }"""
+  val badCustomLabelsFile="""{ "Label1": "valu£1", "--label2": "valuevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevalue" }"""
 
   val validInputsJson = HelloWorld.rawInputs.toJson.toString()
   val wdlSourceWithDocker = HelloWorld.wdlSource(""" runtime { docker: "ubuntu:latest" } """)
@@ -55,7 +58,7 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitSpec with Be
   "MaterializeWorkflowDescriptorActor" should {
     "accept valid WDL, inputs and options files" in {
       val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviourActor, workflowId, importLocalFilesystem = false))
-      val sources = WorkflowSourceFilesWithoutImports(wdlSourceNoDocker, validInputsJson, validOptionsFile)
+      val sources = WorkflowSourceFilesWithoutImports(wdlSourceNoDocker, validInputsJson, validOptionsFile, validCustomLabelsFile)
       materializeWfActor ! MaterializeWorkflowDescriptorCommand(sources, minimumConf)
 
       within(Timeout) {
@@ -68,6 +71,7 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitSpec with Be
             wfDesc.backendDescriptor.knownValues.head shouldBe (("wf_hello.hello.addressee", WdlString("world")))
             wfDesc.getWorkflowOption(WorkflowOptions.WriteToCache) shouldBe Option("true")
             wfDesc.getWorkflowOption(WorkflowOptions.ReadFromCache) shouldBe None
+            wfDesc.backendDescriptor.customLabels shouldBe Labels("label1" -> "value1", "label2" -> "value2")
             // Default backend assignment is "Local":
             wfDesc.backendAssignments foreach {
               case (call, assignment) if call.task.name.equals("hello") => assignment shouldBe "Local"
@@ -106,7 +110,7 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitSpec with Be
           |}
         """.stripMargin
       val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviourActor, workflowId, importLocalFilesystem = false))
-      val sources = WorkflowSourceFilesWithoutImports(wdl, "{}", defaultDocker)
+      val sources = WorkflowSourceFilesWithoutImports(wdl, "{}", defaultDocker, validCustomLabelsFile)
       materializeWfActor ! MaterializeWorkflowDescriptorCommand(sources, minimumConf)
 
       within(Timeout) {
@@ -153,7 +157,7 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitSpec with Be
 
       // Run the test:
       val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviourActor, workflowId, cromwellBackends, importLocalFilesystem = false))
-      val sources = WorkflowSourceFilesWithoutImports(wdl, "{}", "{}")
+      val sources = WorkflowSourceFilesWithoutImports(wdl, "{}", "{}", validCustomLabelsFile)
       materializeWfActor ! MaterializeWorkflowDescriptorCommand(sources, differentDefaultBackendConf)
 
       within(Timeout) {
@@ -188,7 +192,7 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitSpec with Be
         """.stripMargin
 
       val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviourActor, workflowId, importLocalFilesystem = false))
-      val sources = WorkflowSourceFilesWithoutImports(wdl, "{}", "{}")
+      val sources = WorkflowSourceFilesWithoutImports(wdl, "{}", "{}", "{}")
       materializeWfActor ! MaterializeWorkflowDescriptorCommand(sources, differentDefaultBackendConf)
 
       within(Timeout) {
@@ -207,7 +211,7 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitSpec with Be
 
     "reject an invalid WDL source" in {
       val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviourActor, workflowId, importLocalFilesystem = false))
-      val sources = WorkflowSourceFilesWithoutImports(unstructuredFile, validInputsJson, validOptionsFile)
+      val sources = WorkflowSourceFilesWithoutImports(unstructuredFile, validInputsJson, validOptionsFile, validCustomLabelsFile)
       materializeWfActor ! MaterializeWorkflowDescriptorCommand(sources, minimumConf)
 
       within(Timeout) {
@@ -231,7 +235,7 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitSpec with Be
           |# no workflow foo { ... } block!!
         """.stripMargin
       val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviourActor, workflowId, importLocalFilesystem = false))
-      val sources = WorkflowSourceFilesWithoutImports(noWorkflowWdl, validInputsJson, validOptionsFile)
+      val sources = WorkflowSourceFilesWithoutImports(noWorkflowWdl, validInputsJson, validOptionsFile, validCustomLabelsFile)
       materializeWfActor ! MaterializeWorkflowDescriptorCommand(sources, minimumConf)
 
       within(Timeout) {
@@ -256,7 +260,7 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitSpec with Be
           |workflow foo {  }
         """.stripMargin
       val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviourActor, workflowId, importLocalFilesystem = false))
-      val badWdlSources = WorkflowSourceFilesWithoutImports(noWorkflowWdl, validInputsJson, validOptionsFile)
+      val badWdlSources = WorkflowSourceFilesWithoutImports(noWorkflowWdl, validInputsJson, validOptionsFile, validCustomLabelsFile)
       materializeWfActor ! MaterializeWorkflowDescriptorCommand(badWdlSources, minimumConf)
 
       within(Timeout) {
@@ -275,7 +279,7 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitSpec with Be
 
     "reject an invalid options file" in {
       val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviourActor, workflowId, importLocalFilesystem = false))
-      val sources = WorkflowSourceFilesWithoutImports(wdlSourceNoDocker, validInputsJson, unstructuredFile)
+      val sources = WorkflowSourceFilesWithoutImports(wdlSourceNoDocker, validInputsJson, unstructuredFile, validCustomLabelsFile)
       materializeWfActor ! MaterializeWorkflowDescriptorCommand(sources, minimumConf)
 
       within(Timeout) {
@@ -291,9 +295,52 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitSpec with Be
       system.stop(materializeWfActor)
     }
 
+    "reject an unstructured labels file" in {
+      val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviourActor, workflowId, importLocalFilesystem = false))
+      val sources = WorkflowSourceFilesWithoutImports(wdlSourceNoDocker, validInputsJson, validOptionsFile, unstructuredFile)
+      materializeWfActor ! MaterializeWorkflowDescriptorCommand(sources, minimumConf)
+
+      within(Timeout) {
+        expectMsgPF() {
+          case MaterializeWorkflowDescriptorFailureResponse(reason) =>
+            reason.getMessage should startWith(
+              """Workflow input processing failed:
+                |Workflow contains invalid labels JSON: Unexpected character 'u'""".stripMargin)
+          case MaterializeWorkflowDescriptorSuccessResponse(wfDesc) => fail("This materialization should not have succeeded!")
+          case unknown =>
+            fail(s"Unexpected materialization response: $unknown")
+        }
+      }
+
+      system.stop(materializeWfActor)
+    }
+
+    "reject invalid labels" in {
+      val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviourActor, workflowId, importLocalFilesystem = false))
+      val sources = WorkflowSourceFilesWithoutImports(wdlSourceNoDocker, validInputsJson, validOptionsFile, badCustomLabelsFile)
+      materializeWfActor ! MaterializeWorkflowDescriptorCommand(sources, minimumConf)
+
+      within(Timeout) {
+        expectMsgPF() {
+          case MaterializeWorkflowDescriptorFailureResponse(reason) =>
+            reason.getMessage should be(
+              s"""Workflow input processing failed:
+                |Invalid label: Label1 did not match the regex ${Label.LabelRegexPattern}
+                |Invalid label: valu£1 did not match the regex ${Label.LabelRegexPattern}
+                |Invalid label: --label2 did not match the regex ${Label.LabelRegexPattern}
+                |Invalid label: valuevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevalue was 65 characters. The maximum is 63""".stripMargin)
+          case MaterializeWorkflowDescriptorSuccessResponse(wfDesc) => fail("This materialization should not have succeeded!")
+          case unknown =>
+            fail(s"Unexpected materialization response: $unknown")
+        }
+      }
+
+      system.stop(materializeWfActor)
+    }
+
     "reject an invalid workflow inputs file" in {
       val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviourActor, workflowId, importLocalFilesystem = false))
-      val sources = WorkflowSourceFilesWithoutImports(wdlSourceNoDocker, unstructuredFile, validOptionsFile)
+      val sources = WorkflowSourceFilesWithoutImports(wdlSourceNoDocker, unstructuredFile, validOptionsFile, validCustomLabelsFile)
       materializeWfActor ! MaterializeWorkflowDescriptorCommand(sources, minimumConf)
 
       within(Timeout) {
@@ -312,7 +359,7 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitSpec with Be
     "reject requests if any required inputs are missing" in {
       val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviourActor, workflowId, importLocalFilesystem = false))
       val noInputsJson = "{}"
-      val badOptionsSources = WorkflowSourceFilesWithoutImports(wdlSourceNoDocker, noInputsJson, validOptionsFile)
+      val badOptionsSources = WorkflowSourceFilesWithoutImports(wdlSourceNoDocker, noInputsJson, validOptionsFile, validCustomLabelsFile)
       materializeWfActor ! MaterializeWorkflowDescriptorCommand(badOptionsSources, minimumConf)
 
       within(Timeout) {
@@ -338,7 +385,7 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitSpec with Be
           |}
         """.stripMargin
       val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviourActor, workflowId, importLocalFilesystem = false))
-      val sources = WorkflowSourceFilesWithoutImports(wdl, "{}", validOptionsFile)
+      val sources = WorkflowSourceFilesWithoutImports(wdl, "{}", validOptionsFile, validCustomLabelsFile)
       materializeWfActor ! MaterializeWorkflowDescriptorCommand(sources, minimumConf)
 
       within(Timeout) {
diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala
index 2add49473..eb312013c 100644
--- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala
+++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala
@@ -52,7 +52,7 @@ private[ejea] class PerTestHelper(implicit val system: ActorSystem) extends Mock
   call.parent_=(workflow)
   val jobDescriptorKey = BackendJobDescriptorKey(call, jobIndex, jobAttempt)
 
-  val backendWorkflowDescriptor = BackendWorkflowDescriptor(workflowId, null, null, null)
+  val backendWorkflowDescriptor = BackendWorkflowDescriptor(workflowId, null, null, null, null)
   val backendJobDescriptor = BackendJobDescriptor(backendWorkflowDescriptor, jobDescriptorKey, runtimeAttributes = Map.empty, inputDeclarations = Map.empty)
 
   var fetchCachedResultsActorCreations: ExpectOne[(CallCachingEntryId, Seq[TaskOutput])] = NothingYet
diff --git a/engine/src/test/scala/cromwell/subworkflowstore/SubWorkflowStoreSpec.scala b/engine/src/test/scala/cromwell/subworkflowstore/SubWorkflowStoreSpec.scala
index 79e41bd70..ed32f43c5 100644
--- a/engine/src/test/scala/cromwell/subworkflowstore/SubWorkflowStoreSpec.scala
+++ b/engine/src/test/scala/cromwell/subworkflowstore/SubWorkflowStoreSpec.scala
@@ -44,7 +44,7 @@ class SubWorkflowStoreSpec extends CromwellTestKitSpec with Matchers with Mockit
         override def tag: String = "foobar"
       }
 
-      workflowStoreService ! SubmitWorkflow(WorkflowSourceFilesWithoutImports("", "{}", "{}"))
+      workflowStoreService ! SubmitWorkflow(WorkflowSourceFilesWithoutImports("", "{}", "{}", "{}"))
       val rootWorkflowId = expectMsgType[WorkflowSubmittedToStore](10 seconds).workflowId
 
       // Query for non existing sub workflow
diff --git a/src/main/scala/cromwell/CromwellCommandLine.scala b/src/main/scala/cromwell/CromwellCommandLine.scala
index 9c2e4f5ae..3b1b55155 100644
--- a/src/main/scala/cromwell/CromwellCommandLine.scala
+++ b/src/main/scala/cromwell/CromwellCommandLine.scala
@@ -6,10 +6,11 @@ import better.files._
 import cats.data.Validated._
 import cats.syntax.cartesian._
 import cats.syntax.validated._
-import cromwell.core.{WorkflowSourceFilesWithoutImports, WorkflowSourceFilesCollection, WorkflowSourceFilesWithDependenciesZip}
+import cromwell.core.{WorkflowSourceFilesCollection, WorkflowSourceFilesWithDependenciesZip, WorkflowSourceFilesWithoutImports}
 import cromwell.util.FileUtil._
 import lenthall.exception.MessageAggregation
 import lenthall.validation.ErrorOr._
+import org.slf4j.LoggerFactory
 
 import scala.util.{Failure, Success, Try}
 
@@ -17,11 +18,6 @@ sealed abstract class CromwellCommandLine
 case object UsageAndExit extends CromwellCommandLine
 case object RunServer extends CromwellCommandLine
 case object VersionAndExit extends CromwellCommandLine
-final case class RunSingle(wdlPath: Path,
-                           sourceFiles: WorkflowSourceFilesCollection,
-                           inputsPath: Option[Path],
-                           optionsPath: Option[Path],
-                           metadataPath: Option[Path]) extends CromwellCommandLine
 
 object CromwellCommandLine {
   def apply(args: Seq[String]): CromwellCommandLine = {
@@ -34,27 +30,48 @@ object CromwellCommandLine {
   }
 }
 
+// We cannot initialize the logging until after we parse the command line in Main.scala. So we have to bundle up and pass back this information, just for logging.
+case class SingleRunPathParameters(wdlPath: Path, inputsPath: Option[Path], optionsPath: Option[Path], metadataPath: Option[Path], importPath: Option[Path], labelsPath: Option[Path]) {
+  def logMe(log: org.slf4j.Logger) = {
+    log.info(s"  WDL file: $wdlPath")
+    inputsPath foreach { i => log.info(s"  Inputs: $i") }
+    optionsPath foreach { o => log.info(s"  Workflow Options: $o") }
+    metadataPath foreach { m => log.info(s"  Workflow Metadata Output: $m") }
+    importPath foreach { i => log.info(s"  WDL import bundle: $i") }
+    labelsPath foreach { o => log.info(s"  Custom labels: $o") }
+  }
+}
+
+final case class RunSingle(sourceFiles: WorkflowSourceFilesCollection, paths: SingleRunPathParameters) extends CromwellCommandLine
+
 object RunSingle {
+
+  lazy val Log = LoggerFactory.getLogger("cromwell")
+
   def apply(args: Seq[String]): RunSingle = {
-    val wdlPath = Paths.get(args.head).toAbsolutePath
-    val inputsPath = argPath(args, 1, Option(".inputs"), checkDefaultExists = false)
-    val optionsPath = argPath(args, 2, Option(".options"), checkDefaultExists = true)
-    val metadataPath = argPath(args, 3, None)
-    val importPath = argPath(args, 4, None)
-
-    val wdl = readContent("WDL file", wdlPath)
-    val inputsJson = readJson("Inputs", inputsPath)
-    val optionsJson = readJson("Workflow Options", optionsPath)
-
-    val sourceFileCollection = importPath match {
-      case Some(p) => (wdl |@| inputsJson |@| optionsJson) map { (w, i, o) => WorkflowSourceFilesWithDependenciesZip.apply(w, i, o, Files.readAllBytes(p)) }
-      case None => (wdl |@| inputsJson |@| optionsJson) map WorkflowSourceFilesWithoutImports.apply
+    val pathParameters = SingleRunPathParameters(
+      wdlPath = Paths.get(args.head).toAbsolutePath,
+      inputsPath = argPath(args, 1, Option(".inputs"), checkDefaultExists = false),
+      optionsPath = argPath(args, 2, Option(".options"), checkDefaultExists = true),
+      metadataPath = argPath(args, 3, None),
+      importPath = argPath(args, 4, None),
+      labelsPath = argPath(args, 5, None)
+    )
+
+    val wdl = readContent("WDL file", pathParameters.wdlPath)
+    val inputsJson = readJson("Inputs", pathParameters.inputsPath)
+    val optionsJson = readJson("Workflow Options", pathParameters.optionsPath)
+    val labelsJson = readJson("Labels", pathParameters.labelsPath)
+
+    val sourceFileCollection = pathParameters.importPath match {
+      case Some(p) => (wdl |@| inputsJson |@| optionsJson |@| labelsJson) map { (w, i, o, l) => WorkflowSourceFilesWithDependenciesZip.apply(w, i, o, l, Files.readAllBytes(p)) }
+      case None => (wdl |@| inputsJson |@| optionsJson |@| labelsJson) map WorkflowSourceFilesWithoutImports.apply
     }
 
-    val runSingle = for {
+    val runSingle: ErrorOr[RunSingle] = for {
       sources <- sourceFileCollection
-      _ <- writeableMetadataPath(metadataPath)
-    } yield RunSingle(wdlPath, sources, inputsPath, optionsPath, metadataPath)
+      _ <- writeableMetadataPath(pathParameters.metadataPath)
+    } yield RunSingle(sources, pathParameters)
 
     runSingle match {
       case Valid(r) => r
diff --git a/src/main/scala/cromwell/Main.scala b/src/main/scala/cromwell/Main.scala
index 05fa645e5..6fab442cc 100644
--- a/src/main/scala/cromwell/Main.scala
+++ b/src/main/scala/cromwell/Main.scala
@@ -73,12 +73,8 @@ object Main extends App {
     implicit val actorSystem = CromwellSystem.actorSystem
 
     Log.info(s"RUN sub-command")
-    Log.info(s"  WDL file: ${commandLine.wdlPath}")
-    commandLine.inputsPath foreach { i => Log.info(s"  Inputs: $i") }
-    commandLine.optionsPath foreach { o => Log.info(s"  Workflow Options: $o") }
-    commandLine.metadataPath foreach { m => Log.info(s"  Workflow Metadata Output: $m") }
-
-    val runnerProps = SingleWorkflowRunnerActor.props(commandLine.sourceFiles, commandLine.metadataPath)
+    commandLine.paths.logMe(Log)
+    val runnerProps = SingleWorkflowRunnerActor.props(commandLine.sourceFiles, commandLine.paths.metadataPath)
 
     val runner = CromwellSystem.actorSystem.actorOf(runnerProps, "SingleWorkflowRunnerActor")
 
@@ -109,7 +105,7 @@ object Main extends App {
         |
         |Actions:
         |run <WDL file> [<JSON inputs file>] [<JSON workflow options>]
-        |  [<OUTPUT workflow metadata>] [<Zip of WDL Files>]
+        |  [<OUTPUT workflow metadata>] [<Zip of WDL Files>] [<JSON labels file>]
         |
         |  Given a WDL file and JSON file containing the value of the
         |  workflow inputs, this will run the workflow locally and
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala
index 290cb8f4e..538087674 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala
@@ -8,7 +8,7 @@ import com.google.api.services.genomics.Genomics
 import com.google.api.services.genomics.model._
 import cromwell.backend.BackendJobDescriptor
 import cromwell.backend.impl.jes.RunStatus.{Failed, Initializing, Running, Success}
-import cromwell.backend.impl.jes.labels.Labels
+import cromwell.core.labels.Labels
 import cromwell.core.ExecutionEvent
 import cromwell.core.logging.JobLogger
 import org.slf4j.LoggerFactory
@@ -62,22 +62,23 @@ object Run {
 
     lazy val labels: Labels = {
 
-      val hasSubworkflow = workflow.workflow.equals(workflow.rootWorkflow)
+      val subWorkflow = workflow.workflow
+      val subWorkflowLabels = if (!subWorkflow.equals(workflow.rootWorkflow))
+        Labels("cromwell-sub-workflow-name" -> subWorkflow.unqualifiedName)
+      else
+        Labels.empty
 
-      val subWorkflowName = if (hasSubworkflow) workflow.workflow.unqualifiedName else "n-a"
-      val subWorkflowLabels = Labels("cromwell-sub-workflow-name" -> subWorkflowName)
+      val alias = jobDescriptor.call.unqualifiedName
+      val aliasLabels = if (!alias.equals(jobDescriptor.call.task.name))
+        Labels("wdl-call-alias" -> alias)
+      else
+        Labels.empty
 
-      val rootWorkflowLabels = Labels(
+      Labels(
         "cromwell-workflow-id" -> s"cromwell-${workflow.rootWorkflowId}",
-        "cromwell-workflow-name" -> workflow.rootWorkflow.unqualifiedName
-      )
-
-      val callLabels = Labels(
-        "wdl-call-name" -> jobDescriptor.call.unqualifiedName,
+        "cromwell-workflow-name" -> workflow.rootWorkflow.unqualifiedName,
         "wdl-task-name" -> jobDescriptor.call.task.name
-      )
-
-      rootWorkflowLabels ++ subWorkflowLabels ++ callLabels
+      ) ++ subWorkflowLabels ++ aliasLabels ++ jobDescriptor.workflowDescriptor.customLabels
     }
 
     def runPipeline: String = {
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/labels/Label.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/labels/Label.scala
deleted file mode 100644
index 0e7e83101..000000000
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/labels/Label.scala
+++ /dev/null
@@ -1,54 +0,0 @@
-package cromwell.backend.impl.jes.labels
-
-case class Label private[labels](key: String, value: String)
-
-object Label {
-
-  // Yes, 63. Not a typo for 64.
-  // See 'labels' in https://cloud.google.com/genomics/reference/rpc/google.genomics.v1alpha2#google.genomics.v1alpha2.RunPipelineArgs
-  private val MAX_LABEL_LENGTH = 63
-
-  def validate(s: String) = "[a-z]([-a-z0-9]*[a-z0-9])?".r.pattern.matcher(s).matches && s.length <= MAX_LABEL_LENGTH
-
-  /**
-    * Change to meet the constraint:
-    *  - Must match the regex [a-z]([-a-z0-9]*[a-z0-9])?
-    *  - Must be between 1 and MAX_LABEL_LENGTH characters total
-    */
-  def safeName(mainText: String) = {
-
-    if (validate(mainText)) {
-      mainText
-    } else {
-      def appendSafe(current: String, nextChar: Char): String = {
-        nextChar match {
-          case c if c.isLetterOrDigit || c == '-' => current + c.toLower
-          case _ => current + '-'
-        }
-      }
-
-      val foldResult = mainText.toCharArray.foldLeft("")(appendSafe)
-
-      val startsValid = foldResult.headOption.exists(_.isLetter)
-      val endsValid = foldResult.lastOption.exists(_.isLetterOrDigit)
-
-      val validStart = if (startsValid) foldResult else "x--" + foldResult
-      val validStartAndEnd = if (endsValid) validStart else validStart + "--x"
-
-      val length = validStartAndEnd.length
-      val tooLong = length > MAX_LABEL_LENGTH
-
-      if (tooLong) {
-        val middleSeparator = "---"
-        val subSectionLength = (MAX_LABEL_LENGTH - middleSeparator.length) / 2
-        validStartAndEnd.substring(0, subSectionLength) + middleSeparator + validStartAndEnd.substring(length - subSectionLength, length)
-      } else {
-        validStartAndEnd
-      }
-    }
-  }
-
-  def safeLabel(key: String, value: String): Label = {
-    Label(safeName(key), safeName(value))
-  }
-}
diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala
index 605cc7201..08a821037 100644
--- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala
+++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala
@@ -16,6 +16,7 @@ import cromwell.backend.impl.jes.io.{DiskType, JesWorkingDisk}
 import cromwell.backend.impl.jes.statuspolling.JesApiQueryManager.DoPoll
 import cromwell.backend.standard.{DefaultStandardAsyncExecutionActorParams, StandardAsyncExecutionActorParams, StandardAsyncJob}
 import cromwell.core._
+import cromwell.core.labels.Labels
 import cromwell.core.logging.JobLogger
 import cromwell.core.path.PathImplicits._
 import cromwell.filesystems.gcs.auth.GoogleAuthMode.NoAuthMode
@@ -131,7 +132,8 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
       WorkflowId.randomId(),
       WdlNamespaceWithWorkflow.load(YoSup.replace("[PREEMPTIBLE]", s"preemptible: $preemptible"), Seq.empty[ImportResolver]).workflow,
       Inputs,
-      NoOptions
+      NoOptions,
+      Labels.empty
     )
 
     val job = workflowDescriptor.workflow.taskCalls.head
@@ -305,7 +307,8 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
       WorkflowId.randomId(),
       WdlNamespaceWithWorkflow.load(YoSup.replace("[PREEMPTIBLE]", ""), Seq.empty[ImportResolver]).workflow,
       inputs,
-      NoOptions
+      NoOptions,
+      Labels.empty
     )
 
     val call = workflowDescriptor.workflow.taskCalls.head
@@ -359,7 +362,8 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
       WorkflowId.randomId(),
       WdlNamespaceWithWorkflow.load(SampleWdl.CurrentDirectory.asWorkflowSources(DockerAndDiskRuntime).wdlSource, Seq.empty[ImportResolver]).workflow,
       inputs,
-      NoOptions
+      NoOptions,
+      Labels.empty
     )
 
     val job = workflowDescriptor.workflow.taskCalls.head
@@ -398,7 +402,8 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
       WorkflowId.randomId(),
       WdlNamespaceWithWorkflow.load(sampleWdl.asWorkflowSources(DockerAndDiskRuntime).wdlSource, Seq.empty[ImportResolver]).workflow,
       inputs,
-      NoOptions
+      NoOptions,
+      Labels.empty
     )
 
     val call = workflowDescriptor.workflow.findCallByName(callName).get.asInstanceOf[TaskCall]
@@ -459,7 +464,8 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
       WorkflowId.randomId(),
       WdlNamespaceWithWorkflow.load(SampleWdl.CurrentDirectory.asWorkflowSources(DockerAndDiskRuntime).wdlSource, Seq.empty[ImportResolver]).workflow,
       inputs,
-      NoOptions
+      NoOptions,
+      Labels.empty
     )
 
     val job = workflowDescriptor.workflow.taskCalls.head
@@ -487,7 +493,8 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
       WorkflowId.randomId(),
       WdlNamespaceWithWorkflow.load(SampleWdl.CurrentDirectory.asWorkflowSources(DockerAndDiskRuntime).wdlSource, Seq.empty[ImportResolver]).workflow,
       inputs,
-      NoOptions
+      NoOptions,
+      Labels.empty
     )
 
     val job = workflowDescriptor.workflow.taskCalls.head
@@ -531,7 +538,8 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
       WorkflowId.randomId(),
       WdlNamespaceWithWorkflow.load(SampleWdl.EmptyString.asWorkflowSources(DockerAndDiskRuntime).wdlSource, Seq.empty[ImportResolver]).workflow,
       Map.empty,
-      NoOptions
+      NoOptions,
+      Labels.empty
     )
 
     val call = workflowDescriptor.workflow.taskCalls.head
@@ -559,7 +567,8 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
       WorkflowId.randomId(),
       WdlNamespaceWithWorkflow.load(SampleWdl.EmptyString.asWorkflowSources(DockerAndDiskRuntime).wdlSource, Seq.empty[ImportResolver]).workflow,
       Map.empty,
-      WorkflowOptions.fromJsonString("""{"monitoring_script": "gs://path/to/script"}""").get
+      WorkflowOptions.fromJsonString("""{"monitoring_script": "gs://path/to/script"}""").get,
+      Labels.empty
     )
 
     val job = workflowDescriptor.workflow.taskCalls.head
@@ -580,7 +589,8 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
       WorkflowId.randomId(),
       WdlNamespaceWithWorkflow.load(SampleWdl.EmptyString.asWorkflowSources(DockerAndDiskRuntime).wdlSource, Seq.empty[ImportResolver]).workflow,
       Map.empty,
-      NoOptions
+      NoOptions,
+      Labels.empty
     )
 
     val job = workflowDescriptor.workflow.taskCalls.head
@@ -601,7 +611,8 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
       WdlNamespaceWithWorkflow.load(
         SampleWdl.HelloWorld.asWorkflowSources(""" runtime {docker: "ubuntu:latest"} """).wdlSource, Seq.empty[ImportResolver]).workflow,
       Map.empty,
-      WorkflowOptions.fromJsonString(""" {"jes_gcs_root": "gs://path/to/gcs_root"} """).get
+      WorkflowOptions.fromJsonString(""" {"jes_gcs_root": "gs://path/to/gcs_root"} """).get,
+      Labels.empty
     )
 
     val call = workflowDescriptor.workflow.findCallByName("hello").get.asInstanceOf[TaskCall]
@@ -632,7 +643,8 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
       WdlNamespaceWithWorkflow.load(
         new SampleWdl.ScatterWdl().asWorkflowSources(""" runtime {docker: "ubuntu:latest"} """).wdlSource, Seq.empty[ImportResolver]).workflow,
       Map.empty,
-      WorkflowOptions.fromJsonString(""" {"jes_gcs_root": "gs://path/to/gcs_root"} """).get
+      WorkflowOptions.fromJsonString(""" {"jes_gcs_root": "gs://path/to/gcs_root"} """).get,
+      Labels.empty
     )
 
     val call = workflowDescriptor.workflow.findCallByName("B").get.asInstanceOf[TaskCall]
diff --git a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkRuntimeAttributesSpec.scala b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkRuntimeAttributesSpec.scala
index fafec09e0..0eb0ddcec 100644
--- a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkRuntimeAttributesSpec.scala
+++ b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkRuntimeAttributesSpec.scala
@@ -2,6 +2,7 @@ package cromwell.backend.impl.spark
 
 import cromwell.backend.validation.RuntimeAttributesKeys._
 import cromwell.backend.{BackendWorkflowDescriptor, MemorySize}
+import cromwell.core.labels.Labels
 import cromwell.core.{WorkflowId, WorkflowOptions}
 import lenthall.util.TryUtil
 import org.scalatest.{Matchers, WordSpecLike}
@@ -92,7 +93,8 @@ class SparkRuntimeAttributesSpec extends WordSpecLike with Matchers {
       WorkflowId.randomId(),
       WdlNamespaceWithWorkflow.load(wdl.replaceAll("RUNTIME", runtime.format("appMainClass", "com.test.spark")), Seq.empty[ImportResolver]).workflow,
       inputs,
-      options
+      options,
+      Labels.empty
     )
   }
 

From 054a6d8383841067d9e86565903801a6a8f2ee17 Mon Sep 17 00:00:00 2001
From: Chris Llanwarne <cjllanwarne@gmail.com>
Date: Fri, 13 Jan 2017 11:55:16 -0500
Subject: [PATCH 16/84] write functions

---
 .../cromwell/backend/wdl/WriteFunctions.scala      | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/backend/src/main/scala/cromwell/backend/wdl/WriteFunctions.scala b/backend/src/main/scala/cromwell/backend/wdl/WriteFunctions.scala
index 4726c6947..cf7f85d78 100644
--- a/backend/src/main/scala/cromwell/backend/wdl/WriteFunctions.scala
+++ b/backend/src/main/scala/cromwell/backend/wdl/WriteFunctions.scala
@@ -4,6 +4,7 @@ import java.nio.file.Path
 
 import wdl4s.TsvSerializable
 import wdl4s.expression.WdlStandardLibraryFunctions
+import wdl4s.types._
 import wdl4s.values._
 
 import scala.util.{Failure, Try}
@@ -30,19 +31,26 @@ trait WriteFunctions { this: WdlStandardLibraryFunctions =>
     }
   }
 
-  private def writeToTsv(functionName: String, params: Seq[Try[WdlValue]], wdlClass: Class[_ <: WdlValue with TsvSerializable]) = {
+  private def writeToTsv[A <: WdlValue with TsvSerializable](functionName: String, params: Seq[Try[WdlValue]], defaultIfOptionalEmpty: A): Try[WdlFile] = {
+    val wdlClass = defaultIfOptionalEmpty.getClass
+    def castOrDefault(wdlValue: WdlValue): A = wdlValue match {
+      case WdlOptionalValue(_, None) => defaultIfOptionalEmpty
+      case WdlOptionalValue(_, Some(v)) => wdlClass.cast(v)
+      case _ => wdlClass.cast(wdlValue)
+    }
+
     for {
       singleArgument <- extractSingleArgument(functionName, params)
-      downcast <- Try(wdlClass.cast(singleArgument))
+      downcast <- Try(castOrDefault(singleArgument))
       tsvSerialized <- downcast.tsvSerialize
       file <- writeContent(wdlClass.getSimpleName.toLowerCase, tsvSerialized)
     } yield file
   }
 
-  override def write_lines(params: Seq[Try[WdlValue]]): Try[WdlFile] = writeToTsv("write_lines", params, classOf[WdlArray])
-  override def write_map(params: Seq[Try[WdlValue]]): Try[WdlFile] = writeToTsv("write_map", params, classOf[WdlMap])
-  override def write_object(params: Seq[Try[WdlValue]]): Try[WdlFile] = writeToTsv("write_object", params, classOf[WdlObject])
-  override def write_objects(params: Seq[Try[WdlValue]]): Try[WdlFile] = writeToTsv("write_objects", params, classOf[WdlArray])
-  override def write_tsv(params: Seq[Try[WdlValue]]): Try[WdlFile] = writeToTsv("write_tsv", params, classOf[WdlArray])
+  override def write_lines(params: Seq[Try[WdlValue]]): Try[WdlFile] = writeToTsv("write_lines", params, WdlArray(WdlArrayType(WdlStringType), List.empty[WdlValue]))
+  override def write_map(params: Seq[Try[WdlValue]]): Try[WdlFile] = writeToTsv("write_map", params, WdlMap(WdlMapType(WdlStringType, WdlStringType), Map.empty[WdlValue, WdlValue]))
+  override def write_object(params: Seq[Try[WdlValue]]): Try[WdlFile] = writeToTsv("write_object", params, WdlObject(Map.empty[String, WdlValue]))
+  override def write_objects(params: Seq[Try[WdlValue]]): Try[WdlFile] = writeToTsv("write_objects", params, WdlArray(WdlArrayType(WdlObjectType), List.empty[WdlObject]))
+  override def write_tsv(params: Seq[Try[WdlValue]]): Try[WdlFile] = writeToTsv("write_tsv", params, WdlArray(WdlArrayType(WdlStringType), List.empty[WdlValue]))
   override def write_json(params: Seq[Try[WdlValue]]): Try[WdlFile] = Failure(new NotImplementedError(s"write_json() not implemented yet"))
 }

From aa5dce65bbd4d12701b91e5e52ea9b0952662b9b Mon Sep 17 00:00:00 2001
From: Khalid Shakir <kshakir@broadinstitute.org>
Date: Sat, 14 Jan 2017 01:45:42 -0500
Subject: [PATCH 17/84] Changed "webservice.timeout" to
 "webservice.binding-timeout". Also no longer auto-merging the changelog, as
 we couldn't quite get auto-merging to work with our flow, requiring further
 manual intervention anyway.

---
 .gitattributes                                             | 3 ---
 CHANGELOG.md                                               | 5 ++---
 core/src/main/resources/reference.conf                     | 2 +-
 engine/src/main/scala/cromwell/server/CromwellServer.scala | 2 +-
 4 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/.gitattributes b/.gitattributes
index 49c97c85f..9fcf38aa0 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -1,6 +1,3 @@
-# This file can always be added to, line additions should never collide.
-CHANGELOG.MD merge=union
-
 # These files are text and should be normalized (Convert crlf => lf)
 *.scala text
 *.MD text
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 94edff1e4..8dcead856 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,9 +7,8 @@
 * Added ability to override the default zone(s) used by JES via the config structure by setting `genomics.default-zones` in the JES configuration
 * Added support for new WDL functions:
   * `length: (Array[X]) => Integer` - report the length of the specified array 
-* The cromwell server TCP binding timeout is now configurable via the config key `webservice.timeout`, defaulted to the
-  previous value `5s` (five seconds) via the reference.conf.
-  * `length: (Array[X]) => Integer` - report the length of the specified array.
+* The cromwell server TCP binding timeout is now configurable via the config key `webservice.binding-timeout`, defaulted
+  to the previous value `5s` (five seconds) via the reference.conf.
 
 ### Database schema changes
 * Added CUSTOM_LABELS as a field of WORKFLOW_STORE_ENTRY, to store workflow store entries.
diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf
index b60bac4ec..32feb16d3 100644
--- a/core/src/main/resources/reference.conf
+++ b/core/src/main/resources/reference.conf
@@ -8,7 +8,7 @@
 webservice {
   port = 8000
   interface = 0.0.0.0
-  timeout = 5s
+  binding-timeout = 5s
   instance.name = "reference"
 }
 
diff --git a/engine/src/main/scala/cromwell/server/CromwellServer.scala b/engine/src/main/scala/cromwell/server/CromwellServer.scala
index ac47e1ef6..56f87ef87 100644
--- a/engine/src/main/scala/cromwell/server/CromwellServer.scala
+++ b/engine/src/main/scala/cromwell/server/CromwellServer.scala
@@ -31,7 +31,7 @@ object CromwellServer {
 
     val interface = webserviceConf.getString("interface")
     val port = webserviceConf.getInt("port")
-    val timeout = webserviceConf.as[FiniteDuration]("timeout")
+    val timeout = webserviceConf.as[FiniteDuration]("binding-timeout")
     val futureBind = service.bind(interface, port)(implicitly, timeout, actorSystem, implicitly)
     futureBind andThen {
       case Success(_) =>

From cf0c1399809199f28e4c64abdf397afbf55a15ca Mon Sep 17 00:00:00 2001
From: Jeff Gentry <geoffjentry@gmail.com>
Date: Tue, 17 Jan 2017 22:04:40 -0500
Subject: [PATCH 18/84] Add a test for Timedout behavior in CromwellServerActor
 directly here instead of in the route tests (#1854)

---
 .../scala/cromwell/server/CromwellServerSpec.scala | 33 ++++++++++++++++++++++
 1 file changed, 33 insertions(+)
 create mode 100644 engine/src/test/scala/cromwell/server/CromwellServerSpec.scala

diff --git a/engine/src/test/scala/cromwell/server/CromwellServerSpec.scala b/engine/src/test/scala/cromwell/server/CromwellServerSpec.scala
new file mode 100644
index 000000000..01cd00236
--- /dev/null
+++ b/engine/src/test/scala/cromwell/server/CromwellServerSpec.scala
@@ -0,0 +1,33 @@
+package cromwell.server
+
+import akka.pattern.ask
+import akka.util.Timeout
+import com.typesafe.config.ConfigFactory
+import org.scalatest.concurrent.{PatienceConfiguration, ScalaFutures}
+import org.scalatest.{FlatSpec, Matchers}
+import org.specs2.mock.Mockito
+import spray.http.{ContentTypes, HttpRequest, HttpResponse, Timedout}
+
+import scala.concurrent.duration._
+
+class CromwellServerSpec extends FlatSpec with Matchers with Mockito with ScalaFutures {
+  implicit val timeout: Timeout = 5.seconds
+
+  it should "return 500 errors as Json" in {
+    val cromwellSystem = new CromwellSystem {}
+
+    val cromwellServerActor = cromwellSystem.actorSystem.actorOf(CromwellServerActor.props(ConfigFactory.empty()))
+    val response = cromwellServerActor.ask(Timedout(mock[HttpRequest]))
+
+    response.futureValue(PatienceConfiguration.Timeout(timeout.duration)) match {
+      case h: HttpResponse =>
+        h.entity.toOption match {
+          case Some(e) => e.contentType.toString() shouldBe ContentTypes.`application/json`.mediaType.value.toString
+          case None => fail()
+        }
+      case _ => fail()
+    }
+
+    cromwellSystem.shutdownActorSystem()
+  }
+}

From 198a58c669a8416c3aa64b2837db34461137addf Mon Sep 17 00:00:00 2001
From: Miguel Covarrubias <mcovarr@broadinstitute.org>
Date: Tue, 10 Jan 2017 18:13:59 -0500
Subject: [PATCH 19/84] Support non-root users.  Closes #472.

---
 README.md                                          |  2 +-
 .../backend/io/WorkflowPathsWithDocker.scala       |  2 +-
 .../cromwell/backend/wdl/WriteFunctions.scala      |  4 ++-
 .../scala/cromwell/backend/io/JobPathsSpec.scala   | 10 +++---
 .../cromwell/backend/io/WorkflowPathsSpec.scala    |  4 +--
 core/src/main/resources/reference.conf             |  7 +++--
 .../cromwell/core/logging/WorkflowLogger.scala     |  3 +-
 .../scala/cromwell/core/path/FileImplicits.scala   | 36 ++++++++++++++++++++++
 .../main/scala/cromwell/core/path/PathCopier.scala |  3 +-
 .../test/scala/cromwell/util/TestFileUtil.scala    | 10 ++++--
 .../workflow/lifecycle/CopyWorkflowLogsActor.scala |  3 +-
 .../localization/CachedResultLocalization.scala    |  3 +-
 .../cromwell/backend/sfs/SharedFileSystem.scala    |  7 +++--
 .../SharedFileSystemAsyncJobExecutionActor.scala   |  4 ++-
 .../sfs/SharedFileSystemInitializationActor.scala  |  4 +--
 .../SharedFileSystemJobExecutionActorSpec.scala    |  5 +--
 .../backend/sfs/SharedFileSystemSpec.scala         |  3 +-
 17 files changed, 83 insertions(+), 27 deletions(-)
 create mode 100644 core/src/main/scala/cromwell/core/path/FileImplicits.scala

diff --git a/README.md b/README.md
index f73226582..0390470dc 100644
--- a/README.md
+++ b/README.md
@@ -734,7 +734,7 @@ cd <container_call_root>
 echo $? > rc
 ```
 
-`<container_call_root>` would be equal to `<call_dir>` for non-Docker jobs, or it would be under `/root/<workflow_uuid>/call-<call_name>` if this is running in a Docker container.
+`<container_call_root>` would be equal to `<call_dir>` for non-Docker jobs, or it would be under `/cromwell-executions/<workflow_uuid>/call-<call_name>` if this is running in a Docker container.
 
 When running without docker, the subprocess command that the local backend will launch is:
 
diff --git a/backend/src/main/scala/cromwell/backend/io/WorkflowPathsWithDocker.scala b/backend/src/main/scala/cromwell/backend/io/WorkflowPathsWithDocker.scala
index 26e6ce95d..9c1706d96 100644
--- a/backend/src/main/scala/cromwell/backend/io/WorkflowPathsWithDocker.scala
+++ b/backend/src/main/scala/cromwell/backend/io/WorkflowPathsWithDocker.scala
@@ -7,7 +7,7 @@ import cromwell.backend.{BackendJobDescriptorKey, BackendWorkflowDescriptor}
 import cromwell.core.path.PathBuilder
 
 object WorkflowPathsWithDocker {
-  val DockerRoot: Path = Paths.get("/root")
+  val DockerRoot: Path = Paths.get("/cromwell-executions")
 }
 
 class WorkflowPathsWithDocker(val workflowDescriptor: BackendWorkflowDescriptor, val config: Config, val pathBuilders: List[PathBuilder] = WorkflowPaths.DefaultPathBuilders) extends WorkflowPaths {
diff --git a/backend/src/main/scala/cromwell/backend/wdl/WriteFunctions.scala b/backend/src/main/scala/cromwell/backend/wdl/WriteFunctions.scala
index cf7f85d78..293a5f05f 100644
--- a/backend/src/main/scala/cromwell/backend/wdl/WriteFunctions.scala
+++ b/backend/src/main/scala/cromwell/backend/wdl/WriteFunctions.scala
@@ -2,11 +2,13 @@ package cromwell.backend.wdl
 
 import java.nio.file.Path
 
+import cromwell.core.path.FileImplicits._
 import wdl4s.TsvSerializable
 import wdl4s.expression.WdlStandardLibraryFunctions
 import wdl4s.types._
 import wdl4s.values._
 
+import scala.language.existentials
 import scala.util.{Failure, Try}
 
 trait WriteFunctions { this: WdlStandardLibraryFunctions =>
@@ -17,7 +19,7 @@ trait WriteFunctions { this: WdlStandardLibraryFunctions =>
     */
   def writeDirectory: Path
 
-  private lazy val _writeDirectory = File(writeDirectory).createDirectories()
+  private lazy val _writeDirectory = File(writeDirectory).createPermissionedDirectories()
 
   def writeTempFile(path: String,prefix: String,suffix: String,content: String): String = throw new NotImplementedError("This method is not used anywhere and should be removed")
 
diff --git a/backend/src/test/scala/cromwell/backend/io/JobPathsSpec.scala b/backend/src/test/scala/cromwell/backend/io/JobPathsSpec.scala
index dffd64884..94073dd52 100644
--- a/backend/src/test/scala/cromwell/backend/io/JobPathsSpec.scala
+++ b/backend/src/test/scala/cromwell/backend/io/JobPathsSpec.scala
@@ -51,13 +51,13 @@ class JobPathsSpec extends FlatSpec with Matchers with BackendSpec {
     jobPaths.callExecutionRoot.toString shouldBe
       File(s"local-cromwell-executions/wf_hello/$id/call-hello/execution").pathAsString
     jobPaths.callDockerRoot.toString shouldBe
-      File(s"/root/wf_hello/$id/call-hello").pathAsString
+      File(s"/cromwell-executions/wf_hello/$id/call-hello").pathAsString
     jobPaths.callExecutionDockerRoot.toString shouldBe
-      File(s"/root/wf_hello/$id/call-hello/execution").pathAsString
+      File(s"/cromwell-executions/wf_hello/$id/call-hello/execution").pathAsString
     jobPaths.toDockerPath(Paths.get(s"local-cromwell-executions/wf_hello/$id/call-hello/execution/stdout")).toString shouldBe
-      File(s"/root/wf_hello/$id/call-hello/execution/stdout").pathAsString
-    jobPaths.toDockerPath(Paths.get("/root/dock/path")).toString shouldBe
-      File("/root/dock/path").pathAsString
+      File(s"/cromwell-executions/wf_hello/$id/call-hello/execution/stdout").pathAsString
+    jobPaths.toDockerPath(Paths.get("/cromwell-executions/dock/path")).toString shouldBe
+      File("/cromwell-executions/dock/path").pathAsString
 
     val jobKeySharded = BackendJobDescriptorKey(call, Option(0), 1)
     val jobPathsSharded = new JobPathsWithDocker(jobKeySharded, wd, backendConfig)
diff --git a/backend/src/test/scala/cromwell/backend/io/WorkflowPathsSpec.scala b/backend/src/test/scala/cromwell/backend/io/WorkflowPathsSpec.scala
index 36d774bf0..5cdec5500 100644
--- a/backend/src/test/scala/cromwell/backend/io/WorkflowPathsSpec.scala
+++ b/backend/src/test/scala/cromwell/backend/io/WorkflowPathsSpec.scala
@@ -21,7 +21,7 @@ class WorkflowPathsSpec extends FlatSpec with Matchers with BackendSpec {
     workflowPaths.workflowRoot.toString shouldBe
       File(s"local-cromwell-executions/wf_hello/$id").pathAsString
     workflowPaths.dockerWorkflowRoot.toString shouldBe
-      s"/root/wf_hello/$id"
+      s"/cromwell-executions/wf_hello/$id"
   }
 
   "WorkflowPaths" should "provide correct paths for a sub workflow" in {
@@ -59,6 +59,6 @@ class WorkflowPathsSpec extends FlatSpec with Matchers with BackendSpec {
     
     val workflowPaths = new WorkflowPathsWithDocker(subWd, backendConfig)
     workflowPaths.workflowRoot.toString shouldBe File(s"local-cromwell-executions/rootWorkflow/$rootWorkflowId/call-call1/shard-1/attempt-2/subWorkflow/$subWorkflowId").pathAsString
-    workflowPaths.dockerWorkflowRoot.toString shouldBe s"/root/rootWorkflow/$rootWorkflowId/call-call1/shard-1/attempt-2/subWorkflow/$subWorkflowId"
+    workflowPaths.dockerWorkflowRoot.toString shouldBe s"/cromwell-executions/rootWorkflow/$rootWorkflowId/call-call1/shard-1/attempt-2/subWorkflow/$subWorkflowId"
   }
 }
diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf
index 32feb16d3..32f88e9c1 100644
--- a/core/src/main/resources/reference.conf
+++ b/core/src/main/resources/reference.conf
@@ -163,9 +163,12 @@ backend {
       actor-factory = "cromwell.backend.impl.sfs.config.ConfigBackendLifecycleActorFactory"
       config {
         run-in-background = true
-        runtime-attributes = "String? docker"
+        runtime-attributes = """
+        String? docker
+        String? docker_user
+        """
         submit = "/bin/bash ${script}"
-        submit-docker = "docker run --rm -v ${cwd}:${docker_cwd} -i ${docker} /bin/bash ${docker_cwd}/execution/script"
+        submit-docker = """docker run --rm ${"--user " + docker_user} -v ${cwd}:${docker_cwd} -i ${docker} /bin/bash ${docker_cwd}/execution/script"""
 
         # Root directory where Cromwell writes job results.  This directory must be
         # visible and writeable by the Cromwell process as well as the jobs that Cromwell
diff --git a/core/src/main/scala/cromwell/core/logging/WorkflowLogger.scala b/core/src/main/scala/cromwell/core/logging/WorkflowLogger.scala
index 905362c2d..289dd901f 100644
--- a/core/src/main/scala/cromwell/core/logging/WorkflowLogger.scala
+++ b/core/src/main/scala/cromwell/core/logging/WorkflowLogger.scala
@@ -10,6 +10,7 @@ import ch.qos.logback.classic.{Level, LoggerContext}
 import ch.qos.logback.core.FileAppender
 import com.typesafe.config.{Config, ConfigFactory}
 import cromwell.core.WorkflowId
+import cromwell.core.path.FileImplicits._
 import net.ceedubs.ficus.Ficus._
 import org.slf4j.helpers.NOPLogger
 import org.slf4j.{Logger, LoggerFactory}
@@ -116,7 +117,7 @@ class WorkflowLogger(loggerName: String,
   import WorkflowLogger._
 
   lazy val workflowLogPath = workflowLogConfiguration.map(workflowLogConfigurationActual =>
-    File(workflowLogConfigurationActual.dir).createDirectories() / s"workflow.$workflowId.log").map(_.path)
+    File(workflowLogConfigurationActual.dir).createPermissionedDirectories() / s"workflow.$workflowId.log").map(_.path)
 
   lazy val fileLogger = workflowLogPath match {
     case Some(path) => makeFileLogger(path, Level.toLevel(sys.props.getOrElse("LOG_LEVEL", "debug")))
diff --git a/core/src/main/scala/cromwell/core/path/FileImplicits.scala b/core/src/main/scala/cromwell/core/path/FileImplicits.scala
new file mode 100644
index 000000000..0857402da
--- /dev/null
+++ b/core/src/main/scala/cromwell/core/path/FileImplicits.scala
@@ -0,0 +1,36 @@
+package cromwell.core.path
+
+import java.nio.file.FileAlreadyExistsException
+import java.nio.file.attribute.PosixFilePermission
+
+import better.files.File
+
+object FileImplicits {
+
+  implicit class EnhancedFile(val directory: File) extends AnyVal {
+    def createPermissionedDirectories(): directory.type = {
+      if (!directory.exists) {
+        directory.parent.createPermissionedDirectories()
+        try {
+          directory.createDirectory()
+          // When using PosixFilePermissions/FileAttributes with createDirectories, the umask Cromwell happens to be using
+          // affects the resulting directory permissions.  This is not the desired behavior, these directories should be
+          // world readable/writable/executable irrespective of the umask.
+          directory.addPermission(PosixFilePermission.OTHERS_READ)
+          directory.addPermission(PosixFilePermission.OTHERS_WRITE)
+          directory.addPermission(PosixFilePermission.OTHERS_EXECUTE)
+        }
+        catch {
+          // Race condition that's particularly likely with scatters.  Ignore.
+          case _: FileAlreadyExistsException =>
+          // The GCS filesystem does not support setting permissions and will throw an `UnsupportedOperationException`.
+          // Evaluating expressions like `write_lines` in a command block will cause the above permission-manipulating
+          // code to run against a GCS Path. Fortunately creating directories in GCS is also unnecessary, so this
+          // exception type is just ignored.
+          case _: UnsupportedOperationException =>
+        }
+      }
+      directory
+    }
+  }
+}
diff --git a/core/src/main/scala/cromwell/core/path/PathCopier.scala b/core/src/main/scala/cromwell/core/path/PathCopier.scala
index 58eb29068..8318f8202 100644
--- a/core/src/main/scala/cromwell/core/path/PathCopier.scala
+++ b/core/src/main/scala/cromwell/core/path/PathCopier.scala
@@ -6,6 +6,7 @@ import java.nio.file.Path
 import better.files._
 
 import scala.util.{Failure, Try}
+import FileImplicits._
 
 object PathCopier {
 
@@ -44,7 +45,7 @@ object PathCopier {
     * Copies from source to destination. NOTE: Copies are not atomic, and may create a partial copy.
     */
   def copy(sourceFilePath: Path, destinationFilePath: Path): Try[Unit] = {
-    Option(File(destinationFilePath).parent).foreach(_.createDirectories())
+    Option(File(destinationFilePath).parent).foreach(_.createPermissionedDirectories())
     Try {
       File(sourceFilePath).copyTo(destinationFilePath, overwrite = true)
 
diff --git a/core/src/test/scala/cromwell/util/TestFileUtil.scala b/core/src/test/scala/cromwell/util/TestFileUtil.scala
index fb3c9ea14..e02c485a3 100644
--- a/core/src/test/scala/cromwell/util/TestFileUtil.scala
+++ b/core/src/test/scala/cromwell/util/TestFileUtil.scala
@@ -1,18 +1,24 @@
 package cromwell.util
 
 import java.nio.file.Path
+import java.nio.file.attribute.PosixFilePermission
 
 import better.files._
 import wdl4s.values._
+import cromwell.core.path.FileImplicits._
 
 trait TestFileUtil {
   def createCannedFile(prefix: String, contents: String, dir: Option[Path] = None): Path = {
     val suffix = ".out"
-    File.newTemporaryFile(prefix, suffix, dir.map(File.apply)).write(contents).path
+    val tempFile = File.newTemporaryFile(prefix, suffix, dir.map(File.apply))
+    tempFile.createPermissionedDirectories()
+    tempFile.addPermission(PosixFilePermission.OTHERS_READ)
+    tempFile.addPermission(PosixFilePermission.OTHERS_WRITE)
+    tempFile.write(contents).path
   }
 
   def createFile(name: String, dir: Path, contents: String): Path = {
-    File(dir).createDirectories()./(name).write(contents).path
+    File(dir).createPermissionedDirectories()./(name).write(contents).path
   }
 }
 
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/CopyWorkflowLogsActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/CopyWorkflowLogsActor.scala
index 55ef429be..bb1a7a460 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/CopyWorkflowLogsActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/CopyWorkflowLogsActor.scala
@@ -9,6 +9,7 @@ import better.files._
 import cromwell.core.Dispatcher.IoDispatcher
 import cromwell.core._
 import cromwell.core.logging.WorkflowLogger
+import cromwell.core.path.FileImplicits._
 import cromwell.services.metadata.MetadataService.PutMetadataAction
 import cromwell.services.metadata.{MetadataEvent, MetadataKey, MetadataValue}
 
@@ -30,7 +31,7 @@ class CopyWorkflowLogsActor(serviceRegistryActor: ActorRef)
     with ActorLogging {
 
   def copyAndClean(src: Path, dest: Path) = {
-    File(dest).parent.createDirectories()
+    File(dest).parent.createPermissionedDirectories()
 
     File(src).copyTo(dest, overwrite = true)
     if (WorkflowLogger.isTemporary) {
diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/localization/CachedResultLocalization.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/localization/CachedResultLocalization.scala
index 127141c5b..f3c6ad632 100644
--- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/localization/CachedResultLocalization.scala
+++ b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/localization/CachedResultLocalization.scala
@@ -3,6 +3,7 @@ package cromwell.backend.impl.htcondor.caching.localization
 import java.nio.file.{Files, Path, Paths}
 
 import better.files.File
+import cromwell.core.path.FileImplicits._
 import cromwell.core.{JobOutput, _}
 import wdl4s.types.{WdlArrayType, WdlFileType}
 import wdl4s.values.{WdlArray, WdlSingleFile, WdlValue}
@@ -11,7 +12,7 @@ trait CachedResultLocalization {
   private[localization] def localizePathViaSymbolicLink(originalPath: Path, executionPath: Path): Path = {
     if (File(originalPath).isDirectory) throw new UnsupportedOperationException("Cannot localize directory with symbolic links.")
     else {
-      File(executionPath).parent.createDirectories()
+      File(executionPath).parent.createPermissionedDirectories()
       Files.createSymbolicLink(executionPath, originalPath.toAbsolutePath)
     }
   }
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala
index fc1c4168d..83d3e67bf 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala
@@ -9,6 +9,7 @@ import com.typesafe.config.Config
 import com.typesafe.scalalogging.StrictLogging
 import cromwell.backend.io.JobPaths
 import cromwell.core.CromwellFatalExceptionMarker
+import cromwell.core.path.FileImplicits._
 import cromwell.core.path.PathFactory
 import cromwell.core.path.PathFactory._
 import lenthall.util.TryUtil
@@ -47,7 +48,7 @@ object SharedFileSystem extends StrictLogging {
 
   private def localizePathViaCopy(originalPath: File, executionPath: File): Try[Unit] = {
     val action = Try {
-      executionPath.parent.createDirectories()
+      executionPath.parent.createPermissionedDirectories()
       val executionTmpPath = pathPlusSuffix(executionPath, "tmp")
       originalPath.copyTo(executionTmpPath, overwrite = true).moveTo(executionPath, overwrite = true)
     }.void
@@ -56,7 +57,7 @@ object SharedFileSystem extends StrictLogging {
 
   private def localizePathViaHardLink(originalPath: File, executionPath: File): Try[Unit] = {
     val action = Try {
-      executionPath.parent.createDirectories()
+      executionPath.parent.createPermissionedDirectories()
       executionPath.linkTo(originalPath, symbolic = false)
     }.void
     logOnFailure(action, "hard link")
@@ -67,7 +68,7 @@ object SharedFileSystem extends StrictLogging {
       else if (!originalPath.exists) Failure(new FileNotFoundException(originalPath.pathAsString))
       else {
         val action = Try {
-          executionPath.parent.createDirectories()
+          executionPath.parent.createPermissionedDirectories()
           executionPath.linkTo(originalPath, symbolic = true)
         }.void
         logOnFailure(action, "symbolic link")
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala
index 2c850aaf8..170d24437 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala
@@ -9,6 +9,7 @@ import cromwell.backend.io.JobPathsWithDocker
 import cromwell.backend.standard.{StandardAsyncExecutionActor, StandardAsyncJob, StandardInitializationData}
 import cromwell.backend.validation._
 import cromwell.backend.wdl.OutputEvaluator
+import cromwell.core.path.FileImplicits._
 import cromwell.core.path.PathFactory._
 import cromwell.core.path.{DefaultPathBuilder, PathBuilder}
 import cromwell.core.retry.SimpleExponentialBackoff
@@ -126,7 +127,7 @@ trait SharedFileSystemAsyncJobExecutionActor
   override def execute(): ExecutionHandle = {
     val script = instantiatedCommand
     jobLogger.info(s"`$script`")
-    File(jobPaths.callExecutionRoot).createDirectories()
+    File(jobPaths.callExecutionRoot).createPermissionedDirectories()
     val cwd = if (isDockerRun) jobPathsWithDocker.callExecutionDockerRoot else jobPaths.callExecutionRoot
     writeScript(script, cwd, backendEngineFunctions.findGlobOutputs(call, jobDescriptor))
     jobLogger.info(s"command: $processArgs")
@@ -179,6 +180,7 @@ trait SharedFileSystemAsyncJobExecutionActor
 
     val scriptBody =
       s"""|#!/bin/sh
+          |umask 0000
           |(
           |cd $cwd
           |INSTANTIATED_COMMAND
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemInitializationActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemInitializationActor.scala
index 6509ee3f6..4249c7bb4 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemInitializationActor.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemInitializationActor.scala
@@ -13,7 +13,7 @@ import net.ceedubs.ficus.Ficus._
 
 import scala.concurrent.Future
 import scala.util.Try
-
+import cromwell.core.path.FileImplicits._
 /**
   * Initializes a shared file system actor factory and creates initialization data to pass to the execution actors.
   *
@@ -56,7 +56,7 @@ class SharedFileSystemInitializationActor(params: DefaultInitializationActorPara
   override def beforeAll(): Future[Option[BackendInitializationData]] = {
     Future.fromTry(Try {
       publishWorkflowRoot(workflowPaths.workflowRoot.toString)
-      File(workflowPaths.workflowRoot).createDirectories()
+      File(workflowPaths.workflowRoot).createPermissionedDirectories()
       Option(initializationData)
     })
   }
diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala
index 82575378b..518971070 100644
--- a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala
+++ b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala
@@ -12,9 +12,10 @@ import cromwell.backend.io.TestWorkflows._
 import cromwell.backend.io.{JobPathsWithDocker, TestWorkflows}
 import cromwell.backend.sfs.TestLocalAsyncJobExecutionActor._
 import cromwell.backend.standard.StandardValidatedRuntimeAttributesBuilder
-import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptor, BackendJobDescriptorKey, BackendSpec, RuntimeAttributeDefinition}
+import cromwell.backend._
 import cromwell.core.Tags._
 import cromwell.core._
+import cromwell.core.path.FileImplicits._
 import cromwell.services.keyvalue.KeyValueServiceActor.{KvJobKey, KvPair, ScopedKey}
 import lenthall.exception.AggregatedException
 import org.scalatest.concurrent.PatienceConfiguration.Timeout
@@ -161,7 +162,7 @@ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSyst
     val backend = backendRef.underlyingActor
 
     val jobPaths = new JobPathsWithDocker(jobDescriptor.key, workflowDescriptor, ConfigFactory.empty)
-    File(jobPaths.callExecutionRoot).createDirectories()
+    File(jobPaths.callExecutionRoot).createPermissionedDirectories()
     File(jobPaths.stdout).write("Hello stubby ! ")
     File(jobPaths.stderr).touch()
 
diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemSpec.scala b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemSpec.scala
index 228943633..16ac18836 100644
--- a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemSpec.scala
+++ b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemSpec.scala
@@ -7,6 +7,7 @@ import com.typesafe.config.{Config, ConfigFactory}
 import cromwell.backend.BackendSpec
 import cromwell.core.CromwellFatalExceptionMarker
 import cromwell.core.path.DefaultPathBuilder
+import cromwell.core.path.FileImplicits._
 import org.scalatest.prop.TableDrivenPropertyChecks
 import org.scalatest.{FlatSpec, Matchers}
 import org.specs2.mock.Mockito
@@ -33,7 +34,7 @@ class SharedFileSystemSpec extends FlatSpec with Matchers with Mockito with Tabl
     val dest = if (fileInCallDir) orig else callDir./(orig.pathAsString.drop(1))
     orig.touch()
     if (fileAlreadyExists) {
-      dest.parent.createDirectories()
+      dest.parent.createPermissionedDirectories()
       dest.touch()
     }
 

From 1d8fb0396a8fa80a12b83d5d23ff71569ec3204a Mon Sep 17 00:00:00 2001
From: Jeff Gentry <geoffjentry@gmail.com>
Date: Thu, 19 Jan 2017 09:12:29 -0500
Subject: [PATCH 20/84] Change exit codes for usage and version to 0. Closes
 #1870 (#1873)

---
 src/main/scala/cromwell/Main.scala | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/scala/cromwell/Main.scala b/src/main/scala/cromwell/Main.scala
index 6fab442cc..445c227b7 100644
--- a/src/main/scala/cromwell/Main.scala
+++ b/src/main/scala/cromwell/Main.scala
@@ -131,7 +131,7 @@ object Main extends App {
         |
       """.stripMargin)
 
-    System.exit(1)
+    System.exit(0)
   }
 
   def versionAndExit(): Unit = {
@@ -141,6 +141,6 @@ object Main extends App {
          |cromwell: ${versionConf.getString("cromwell")}
        """.stripMargin
     )
-    System.exit(1)
+    System.exit(0)
   }
 }

From ff6c2dbaaef4302045635a8ffd8ab014f3eff8fc Mon Sep 17 00:00:00 2001
From: Thib <Horneth@users.noreply.github.com>
Date: Thu, 19 Jan 2017 10:12:47 -0500
Subject: [PATCH 21/84] Better JES errors + RC Closes #1848 (#1856)

* JES error mapping + populate RC even if JES job fails

* custom centaur branch

* PR comments

* oops

* centaur back to develop
---
 .../BackendWorkflowInitializationActor.scala       |  5 +-
 .../backend/async/KnownJobFailureException.scala   | 38 +++++++++++++---
 .../standard/StandardAsyncExecutionActor.scala     | 25 +++++-----
 .../engine/workflow/WorkflowManagerActor.scala     |  8 +++-
 .../main/scala/cromwell/jobstore/SqlJobStore.scala |  3 +-
 .../main/scala/cromwell/jobstore/jobstore_.scala   |  5 +-
 .../jes/JesAsyncBackendJobExecutionActor.scala     | 22 ++++-----
 .../main/scala/cromwell/backend/impl/jes/Run.scala |  2 +-
 .../cromwell/backend/impl/jes/RunStatus.scala      | 14 +++++-
 .../backend/impl/jes/errors/JesError.scala         | 53 ++++++++++++++++++++++
 .../impl/jes/errors/JesKnownJobFailure.scala       | 25 ++++++++++
 .../cromwell/backend/impl/jes/io/package.scala     |  4 ++
 .../jes/JesAsyncBackendJobExecutionActorSpec.scala | 20 ++++----
 .../impl/jes/JesInitializationActorSpec.scala      |  7 +--
 .../jes/statuspolling/JesPollingActorSpec.scala    |  2 +-
 .../SharedFileSystemJobExecutionActorSpec.scala    |  2 +-
 16 files changed, 183 insertions(+), 52 deletions(-)
 create mode 100644 supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/errors/JesError.scala
 create mode 100644 supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/errors/JesKnownJobFailure.scala

diff --git a/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala b/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala
index 626674f19..5bf1d05fd 100644
--- a/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala
+++ b/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala
@@ -4,6 +4,7 @@ import akka.actor.{ActorLogging, ActorRef}
 import akka.event.LoggingReceive
 import cromwell.backend.BackendLifecycleActor._
 import cromwell.backend.BackendWorkflowInitializationActor._
+import cromwell.backend.async.{RuntimeAttributeValidationFailure, RuntimeAttributeValidationFailures}
 import cromwell.backend.validation.ContinueOnReturnCodeValidation
 import cromwell.core.{WorkflowMetadataKeys, WorkflowOptions}
 import cromwell.services.metadata.MetadataService.PutMetadataAction
@@ -129,13 +130,13 @@ trait BackendWorkflowInitializationActor extends BackendWorkflowLifecycleActor w
             val value = task.runtimeAttributes.attrs.get(attributeName) orElse defaultRuntimeAttribute(attributeName)
             attributeName -> ((value, validator(value)))
           } collect {
-            case (name, (value, false)) => s"Task ${task.name} has an invalid runtime attribute $name = ${value map { _.valueString} getOrElse "!! NOT FOUND !!"}"
+            case (name, (value, false)) => RuntimeAttributeValidationFailure(task.name, name, value)
           }
         }
 
         calls map { _.task } flatMap badRuntimeAttrsForTask match {
           case errors if errors.isEmpty => Future.successful(())
-          case errors => Future.failed(new IllegalArgumentException(errors.mkString(". ")))
+          case errors => Future.failed(RuntimeAttributeValidationFailures(errors.toList))
         }
       case Failure(t) => Future.failed(t)
     }
diff --git a/backend/src/main/scala/cromwell/backend/async/KnownJobFailureException.scala b/backend/src/main/scala/cromwell/backend/async/KnownJobFailureException.scala
index e43895e22..53130ada9 100644
--- a/backend/src/main/scala/cromwell/backend/async/KnownJobFailureException.scala
+++ b/backend/src/main/scala/cromwell/backend/async/KnownJobFailureException.scala
@@ -2,16 +2,18 @@ package cromwell.backend.async
 
 import java.nio.file.Path
 
-sealed abstract class KnownJobFailureException extends Exception {
-  def stderrPath: Path
-  def jobTag: String
+import lenthall.exception.ThrowableAggregation
+import wdl4s.values.WdlValue
+
+abstract class KnownJobFailureException extends Exception {
+  def stderrPath: Option[Path]
 }
 
-final case class WrongReturnCode(jobTag: String, returnCode: Int, stderrPath: Path) extends KnownJobFailureException {
+final case class WrongReturnCode(jobTag: String, returnCode: Int, stderrPath: Option[Path]) extends KnownJobFailureException {
   override def getMessage = s"Job $jobTag exited with return code $returnCode which has not been declared as a valid return code. See 'continueOnReturnCode' runtime attribute for more details."
 }
 
-final case class ReturnCodeIsNotAnInt(jobTag: String, returnCode: String, stderrPath: Path) extends KnownJobFailureException {
+final case class ReturnCodeIsNotAnInt(jobTag: String, returnCode: String, stderrPath: Option[Path]) extends KnownJobFailureException {
   override def getMessage = {
     if (returnCode.isEmpty)
       s"The return code file for job $jobTag was empty."
@@ -20,6 +22,30 @@ final case class ReturnCodeIsNotAnInt(jobTag: String, returnCode: String, stderr
   }
 }
 
-final case class StderrNonEmpty(jobTag: String, stderrLength: Long, stderrPath: Path) extends KnownJobFailureException {
+final case class StderrNonEmpty(jobTag: String, stderrLength: Long, stderrPath: Option[Path]) extends KnownJobFailureException {
   override def getMessage = s"stderr for job $jobTag has length $stderrLength and 'failOnStderr' runtime attribute was true."
 }
+
+
+object RuntimeAttributeValidationFailure {
+  def apply(jobTag: String,
+            runtimeAttributeName: String,
+            runtimeAttributeValue: Option[WdlValue]): RuntimeAttributeValidationFailure = RuntimeAttributeValidationFailure(jobTag, runtimeAttributeName, runtimeAttributeValue, None)
+}
+
+final case class RuntimeAttributeValidationFailure private (jobTag: String,
+                                                   runtimeAttributeName: String,
+                                                   runtimeAttributeValue: Option[WdlValue],
+                                                   stderrPath: Option[Path]) extends KnownJobFailureException {
+  override def getMessage = s"Task $jobTag has an invalid runtime attribute $runtimeAttributeName = ${runtimeAttributeValue map { _.valueString} getOrElse "!! NOT FOUND !!"}"
+}
+
+final case class RuntimeAttributeValidationFailures(throwables: List[RuntimeAttributeValidationFailure]) extends KnownJobFailureException with ThrowableAggregation {
+  override def exceptionContext = "Runtime validation failed"
+  override val stderrPath: Option[Path] = None
+}
+
+final case class JobAlreadyFailedInJobStore(jobTag: String, originalErrorMessage: String) extends KnownJobFailureException {
+  override def stderrPath: Option[Path] = None
+  override def getMessage = originalErrorMessage
+}
diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala b/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala
index 0d97cbbb9..95ebc8a11 100644
--- a/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala
+++ b/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala
@@ -1,5 +1,7 @@
 package cromwell.backend.standard
 
+import java.nio.file.Path
+
 import akka.actor.{Actor, ActorLogging, ActorRef}
 import akka.event.LoggingReceive
 import better.files.File
@@ -239,8 +241,9 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
     * @return The execution handle.
     */
   def handleExecutionFailure(runStatus: StandardAsyncRunStatus,
-                             handle: StandardAsyncPendingExecutionHandle): ExecutionHandle = {
-    FailedNonRetryableExecutionHandle(new Exception(s"Task failed for unknown reason: $runStatus"), None)
+                             handle: StandardAsyncPendingExecutionHandle,
+                             returnCode: Option[Int]): ExecutionHandle = {
+    FailedNonRetryableExecutionHandle(new Exception(s"Task failed for unknown reason: $runStatus"), returnCode)
   }
 
   context.become(standardReceiveBehavior(None) orElse receive)
@@ -371,12 +374,13 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
   def handleExecutionResult(status: StandardAsyncRunStatus,
                             oldHandle: StandardAsyncPendingExecutionHandle): ExecutionHandle = {
     try {
-      if (isSuccess(status)) {
 
+      lazy val returnCodeAsString: Try[String] = Try(File(jobPaths.returnCode).contentAsString)
+      lazy val returnCodeAsInt: Try[Int] = returnCodeAsString.map(_.trim.toInt)
+      lazy val stderrAsOption: Option[Path] = Option(jobPaths.stderr)
+      
+      if (isSuccess(status)) {
         lazy val stderrLength: Try[Long] = Try(File(jobPaths.stderr).size)
-        lazy val returnCodeAsString: Try[String] = Try(File(jobPaths.returnCode).contentAsString)
-        lazy val returnCodeAsInt: Try[Int] = returnCodeAsString.map(_.trim.toInt)
-        
         (stderrLength, returnCodeAsString, returnCodeAsInt) match {
             // Failed to get stderr size -> Retry
           case (Failure(exception), _, _) =>
@@ -388,23 +392,22 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
             oldHandle
             // Failed to convert return code content to Int -> Fail
           case (_, _, Failure(exception)) =>
-            FailedNonRetryableExecutionHandle(ReturnCodeIsNotAnInt(jobDescriptor.key.tag, returnCodeAsString.get, jobPaths.stderr))
+            FailedNonRetryableExecutionHandle(ReturnCodeIsNotAnInt(jobDescriptor.key.tag, returnCodeAsString.get, stderrAsOption))
             // Stderr is not empty and failOnStdErr is true -> Fail
           case (Success(length), _, _) if failOnStdErr && length.intValue > 0 =>
-            FailedNonRetryableExecutionHandle(StderrNonEmpty(jobDescriptor.key.tag, length, jobPaths.stderr), returnCodeAsInt.toOption)
+            FailedNonRetryableExecutionHandle(StderrNonEmpty(jobDescriptor.key.tag, length, stderrAsOption), returnCodeAsInt.toOption)
             // Return code is abort code -> Abort
           case (_, _, Success(rc)) if isAbort(rc) =>
             AbortedExecutionHandle
             // Return code is not valid -> Fail
           case (_, _, Success(rc)) if !continueOnReturnCode.continueFor(rc) =>
-            FailedNonRetryableExecutionHandle(WrongReturnCode(jobDescriptor.key.tag, returnCodeAsInt.get, jobPaths.stderr), returnCodeAsInt.toOption)
+            FailedNonRetryableExecutionHandle(WrongReturnCode(jobDescriptor.key.tag, returnCodeAsInt.get, stderrAsOption), returnCodeAsInt.toOption)
             // Otherwise -> Succeed
           case (_, _, Success(rc)) => 
             handleExecutionSuccess(status, oldHandle, rc)
         }
-
       } else {
-        handleExecutionFailure(status, oldHandle)
+        handleExecutionFailure(status, oldHandle, returnCodeAsInt.toOption)
       }
     } catch {
       case e: Exception =>
diff --git a/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala b/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala
index 766a54f66..7b870407c 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala
@@ -15,6 +15,7 @@ import cromwell.engine.workflow.workflowstore.{WorkflowStoreActor, WorkflowStore
 import cromwell.jobstore.JobStoreActor.{JobStoreWriteFailure, JobStoreWriteSuccess, RegisterWorkflowCompleted}
 import cromwell.services.metadata.MetadataService._
 import cromwell.webservice.EngineStatsActor
+import lenthall.exception.ThrowableAggregation
 import net.ceedubs.ficus.Ficus._
 import org.apache.commons.lang3.exception.ExceptionUtils
 
@@ -301,11 +302,14 @@ class WorkflowManagerActor(params: WorkflowManagerActorParams)
     context.system.scheduler.scheduleOnce(newWorkflowPollRate, self, RetrieveNewWorkflows)(context.dispatcher)
   }
 
-  private def expandFailureReasons(reasons: Seq[Throwable]) = {
+  private def expandFailureReasons(reasons: Seq[Throwable]): String = {
     import cromwell.core.path.PathImplicits._
+    
     reasons map {
+      case reason: ThrowableAggregation => expandFailureReasons(reason.throwables.toSeq)
       case reason: KnownJobFailureException =>
-        reason.getMessage + "\n" + s"Check the content of stderr for potential additional information: ${reason.stderrPath.toRealString}"
+        val stderrMessage = reason.stderrPath map { path => s"\nCheck the content of stderr for potential additional information: ${path.toRealString}" } getOrElse ""
+        reason.getMessage + stderrMessage
       case reason =>
       reason.getMessage + "\n" + ExceptionUtils.getStackTrace(reason)
     } mkString "\n"
diff --git a/engine/src/main/scala/cromwell/jobstore/SqlJobStore.scala b/engine/src/main/scala/cromwell/jobstore/SqlJobStore.scala
index 2ef0614c0..085f30e06 100644
--- a/engine/src/main/scala/cromwell/jobstore/SqlJobStore.scala
+++ b/engine/src/main/scala/cromwell/jobstore/SqlJobStore.scala
@@ -1,6 +1,7 @@
 package cromwell.jobstore
 
 import cromwell.Simpletons._
+import cromwell.backend.async.JobAlreadyFailedInJobStore
 import cromwell.core.ExecutionIndex._
 import cromwell.core.WorkflowId
 import cromwell.core.simpleton.WdlValueBuilder
@@ -63,7 +64,7 @@ class SqlJobStore(sqlDatabase: JobStoreSqlDatabase) extends JobStore {
             val jobOutputs = WdlValueBuilder.toJobOutputs(taskOutputs, simpletons)
             JobResultSuccess(returnCode, jobOutputs)
           case JobStoreEntry(_, _, _, _, false, returnCode, Some(exceptionMessage), Some(retryable), _) =>
-            JobResultFailure(returnCode, new Exception(exceptionMessage), retryable)
+            JobResultFailure(returnCode, JobAlreadyFailedInJobStore(jobStoreKey.tag, exceptionMessage), retryable)
           case bad =>
             throw new Exception(s"Invalid contents of JobStore table: $bad")
         }
diff --git a/engine/src/main/scala/cromwell/jobstore/jobstore_.scala b/engine/src/main/scala/cromwell/jobstore/jobstore_.scala
index 921183d35..5ba67b462 100644
--- a/engine/src/main/scala/cromwell/jobstore/jobstore_.scala
+++ b/engine/src/main/scala/cromwell/jobstore/jobstore_.scala
@@ -2,7 +2,10 @@ package cromwell.jobstore
 
 import cromwell.core.{WorkflowId, _}
 
-case class JobStoreKey(workflowId: WorkflowId, callFqn: String, index: Option[Int], attempt: Int)
+case class JobStoreKey(workflowId: WorkflowId, callFqn: String, index: Option[Int], attempt: Int) {
+  private lazy val indexString = index map { _.toString } getOrElse "NA"
+  lazy val tag = s"$workflowId:$callFqn:$indexString:$attempt"
+}
 
 sealed trait JobResult
 case class JobResultSuccess(returnCode: Option[Int], jobOutputs: CallOutputs) extends JobResult
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala
index dc24e4c3d..a59259677 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala
@@ -258,7 +258,7 @@ class JesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn
           |mv $rcTmpPath $rcPath
           |""".stripMargin.replace("INSTANTIATED_COMMAND", command)
 
-    File(jesCallPaths.script).write(fileContent)
+    jesCallPaths.script.writeAsText(fileContent)
     ()
   }
 
@@ -405,7 +405,7 @@ class JesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn
     errorMessage.substring(0, errorMessage.indexOf(':')).toInt
   }
 
-  private def preempted(errorCode: Int, errorMessage: List[String]): Boolean = {
+  private def preempted(errorCode: Int, errorMessage: Option[String]): Boolean = {
     def isPreemptionCode(code: Int) = code == 13 || code == 14
 
     try {
@@ -418,12 +418,13 @@ class JesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn
   }
 
   override def handleExecutionFailure(runStatus: RunStatus,
-                                      handle: StandardAsyncPendingExecutionHandle): ExecutionHandle = {
+                                      handle: StandardAsyncPendingExecutionHandle,
+                                      returnCode: Option[Int]): ExecutionHandle = {
     val failed = runStatus match {
       case failedStatus: RunStatus.Failed => failedStatus
-      case unknown =>
-        throw new RuntimeException(s"handleExecutionFailure not called with RunStatus.Failed. Instead got $unknown")
+      case unknown => throw new RuntimeException(s"handleExecutionFailure not called with RunStatus.Failed. Instead got $unknown")
     }
+    
     val errorCode = failed.errorCode
     val errorMessage = failed.errorMessage
 
@@ -442,19 +443,16 @@ class JesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn
           s"""$preemptedMsg The call will be restarted with another preemptible VM (max preemptible attempts number is $maxPreemption).
              |Error code $errorCode. Message: $errorMessage""".stripMargin
         )
-        FailedRetryableExecutionHandle(e, None)
+        FailedRetryableExecutionHandle(e, returnCode)
       } else {
         val e = PreemptedException(
           s"""$preemptedMsg The maximum number of preemptible attempts ($maxPreemption) has been reached. The call will be restarted with a non-preemptible VM.
              |Error code $errorCode. Message: $errorMessage)""".stripMargin)
-        FailedRetryableExecutionHandle(e, None)
+        FailedRetryableExecutionHandle(e, returnCode)
       }
     } else {
-      val id = workflowDescriptor.id
-      val name = jobDescriptor.call.unqualifiedName
-      val message = if (errorMessage.isEmpty) "null" else errorMessage.mkString(", ")
-      val exception = new RuntimeException(s"Task $id:$name failed: error code $errorCode. Message: $message")
-      FailedNonRetryableExecutionHandle(exception, None)
+      val exception = failed.toFailure(jobPaths.jobKey.tag, Option(jobPaths.stderr))
+      FailedNonRetryableExecutionHandle(exception, returnCode)
     }
   }
 
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala
index 538087674..2de9ebc8e 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala
@@ -127,7 +127,7 @@ object Run {
       // If there's an error, generate a Failed status. Otherwise, we were successful!
       Option(op.getError) match {
         case None => Success(eventList, machineType, zone, instanceName)
-        case Some(error) => Failed(error.getCode, Option(error.getMessage).toList, eventList, machineType, zone, instanceName)
+        case Some(error) => Failed(error.getCode, Option(error.getMessage), eventList, machineType, zone, instanceName)
       }
     } else if (op.hasStarted) {
       Running
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/RunStatus.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/RunStatus.scala
index d0609573e..45684bb11 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/RunStatus.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/RunStatus.scala
@@ -1,5 +1,8 @@
 package cromwell.backend.impl.jes
 
+import java.nio.file.Path
+
+import cromwell.backend.impl.jes.errors.JesError
 import cromwell.core.ExecutionEvent
 
 sealed trait RunStatus {
@@ -27,9 +30,18 @@ object RunStatus {
     override def toString = "Success"
   }
 
-  final case class Failed(errorCode: Int, errorMessage: List[String], eventList: Seq[ExecutionEvent], machineType: Option[String], zone: Option[String], instanceName: Option[String])
+  final case class Failed(errorCode: Int, errorMessage: Option[String], eventList: Seq[ExecutionEvent], machineType: Option[String], zone: Option[String], instanceName: Option[String])
     extends TerminalRunStatus {
+    
+    private def unknownError(jobTag: String) = {
+      new RuntimeException(s"Task $jobTag failed: error code $errorCode. Message: $errorMessage")
+    }
+    
     // Don't want to include errorMessage or code in the snappy status toString:
     override def toString = "Failed"
+    
+    def toFailure(jobTag: String, stderrPath: Option[Path]): Exception = {
+      JesError.fromFailedStatus(this, jobTag, stderrPath) getOrElse unknownError(jobTag)
+    }
   }
 }
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/errors/JesError.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/errors/JesError.scala
new file mode 100644
index 000000000..2952ef70e
--- /dev/null
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/errors/JesError.scala
@@ -0,0 +1,53 @@
+package cromwell.backend.impl.jes.errors
+
+import java.nio.file.Path
+
+import cromwell.backend.impl.jes.RunStatus
+
+import scala.util.Try
+
+object JesError {
+  /**
+    * List of known and mapped JES errors
+    */
+  private val KnownErrors = List(
+    FailedToDelocalize
+  )
+
+  /**
+    * The error message looks like "10: Something bad happened"
+    * Extract 10 as an inner error code and then the rest of the message
+    */
+  private def splitInnerCodeAndMessage(errorMessage: String): Option[(Int, String)] = Try {
+    val sep = errorMessage.indexOf(':')
+    val innerCode = errorMessage.substring(0, sep).toInt
+    val message = errorMessage.substring(sep + 1, errorMessage.length - 1).trim
+    (innerCode, message)
+  }.toOption
+  
+  def fromFailedStatus(failedStatus: RunStatus.Failed, jobTag: String, stderrPath: Option[Path]): Option[JesKnownJobFailure] = {
+    def lookupError(innerCode: Int, message: String) = {
+      val search = KnownErrors.toStream map { _.toFailureOption(failedStatus.errorCode, innerCode, message, jobTag, stderrPath) } find { _.isDefined }
+      search.flatten
+    }
+    
+    failedStatus.errorMessage flatMap splitInnerCodeAndMessage flatMap Function.tupled(lookupError)
+  }
+}
+
+sealed abstract class JesError(val outerCode: Int, val innerCode: Int, val messageStart: String) {
+  def toFailureOption(errorOuterCode: Int, errorInnerCode: Int, errorMessage: String, jobTag: String, stderrPath: Option[Path]): Option[JesKnownJobFailure] = {
+    if (
+        errorOuterCode == outerCode &&
+        errorInnerCode == innerCode &&
+        errorMessage.startsWith(messageStart)
+    ) Option(toJobFailure(errorMessage, jobTag, stderrPath))
+    else None
+  }
+  
+  protected def toJobFailure(message: String, jobTag: String, stderrPath: Option[Path]): JesKnownJobFailure
+}
+
+private case object FailedToDelocalize extends JesError(5, 10, "Failed to delocalize files") {
+  def toJobFailure(message: String, jobTag: String, stderrPath: Option[Path]) = FailedToDelocalizeFailure(message, jobTag, stderrPath)
+}
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/errors/JesKnownJobFailure.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/errors/JesKnownJobFailure.scala
new file mode 100644
index 000000000..1ca9a771f
--- /dev/null
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/errors/JesKnownJobFailure.scala
@@ -0,0 +1,25 @@
+package cromwell.backend.impl.jes.errors
+
+import java.nio.file.Path
+
+import cromwell.backend.async.KnownJobFailureException
+import cromwell.core.path.PathImplicits._
+
+sealed trait JesKnownJobFailure extends KnownJobFailureException
+
+case class FailedToDelocalizeFailure(message: String, jobTag: String, stderrPath: Option[Path]) extends JesKnownJobFailure {
+  lazy val stderrMessage = stderrPath map { p =>
+    s"3) Look into the stderr (${p.toRealString}) file for evidence that some of the output files the command is expected to create were not created."
+  } getOrElse ""
+  
+  lazy val missingFilesMessage = if (message.contains("No URLs matched")) {
+    s"""It appears that some of the expected output files for task $jobTag did not exist when the command exited.
+       |A few things to try
+       |1) Check that the output section in your WDL is correct. Remember that all output files declared in a task must exist when the command exits.
+       |2) Check that the return code is available and is valid with respect to your command expected exit code
+       |$stderrMessage
+     """.stripMargin
+  } else ""
+  
+  override def getMessage = missingFilesMessage + message
+}
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/package.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/package.scala
index 2a2ae8ac2..acfc61f0a 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/package.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/package.scala
@@ -12,6 +12,10 @@ package object io {
     def writeAsJson(content: String): File = {
       Files.write(path, content.getBytes, CloudStorageOptions.withMimeType("application/json"))
     }
+
+    def writeAsText(content: String): File = {
+      Files.write(path, content.getBytes, CloudStorageOptions.withMimeType("text/plain"))
+    }
   }
 
   private [jes] def isFatalJesException(t: Throwable): Boolean = t match {
diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala
index 08a821037..56ccc1ced 100644
--- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala
+++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala
@@ -160,7 +160,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
 
   private def runAndFail(attempt: Int, preemptible: Int, errorCode: Int, innerErrorCode: Int): BackendJobExecutionResponse = {
 
-    val runStatus = Failed(errorCode, List(s"$innerErrorCode: I seen some things man"), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance"))
+    val runStatus = Failed(errorCode, Option(s"$innerErrorCode: I seen some things man"), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance"))
     val statusPoller = TestProbe()
 
     val promise = Promise[BackendJobExecutionResponse]()
@@ -222,7 +222,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
     val handle = mock[JesPendingExecutionHandle]
     implicit val ec = system.dispatcher
 
-    val failedStatus = Failed(10, List("14: VM XXX shut down unexpectedly."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance"))
+    val failedStatus = Failed(10, Option("14: VM XXX shut down unexpectedly."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance"))
     val executionResult = Await.result(jesBackend.executionResult(failedStatus, handle), 2.seconds)
     executionResult.isInstanceOf[FailedNonRetryableExecutionHandle] shouldBe true
     val failedHandle = executionResult.asInstanceOf[FailedNonRetryableExecutionHandle]
@@ -235,7 +235,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
     val handle = mock[JesPendingExecutionHandle]
     implicit val ec = system.dispatcher
 
-    val failedStatus = Failed(10, List("14: VM XXX shut down unexpectedly."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance"))
+    val failedStatus = Failed(10, Option("14: VM XXX shut down unexpectedly."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance"))
     val executionResult = Await.result(jesBackend.executionResult(failedStatus, handle), 2.seconds)
     executionResult.isInstanceOf[FailedRetryableExecutionHandle] shouldBe true
     val retryableHandle = executionResult.asInstanceOf[FailedRetryableExecutionHandle]
@@ -251,7 +251,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
     val handle = mock[JesPendingExecutionHandle]
     implicit val ec = system.dispatcher
 
-    val failedStatus = Failed(10, List("14: VM XXX shut down unexpectedly."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance"))
+    val failedStatus = Failed(10, Option("14: VM XXX shut down unexpectedly."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance"))
     val executionResult = Await.result(jesBackend.executionResult(failedStatus, handle), 2.seconds)
     executionResult.isInstanceOf[FailedRetryableExecutionHandle] shouldBe true
     val retryableHandle = executionResult.asInstanceOf[FailedRetryableExecutionHandle]
@@ -268,22 +268,22 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
     implicit val ec = system.dispatcher
 
     Await.result(jesBackend.executionResult(
-      Failed(10, List("15: Other type of error."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")), handle), 2.seconds
+      Failed(10, Option("15: Other type of error."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")), handle), 2.seconds
     ).isInstanceOf[FailedNonRetryableExecutionHandle] shouldBe true
     Await.result(jesBackend.executionResult(
-      Failed(11, List("14: Wrong errorCode."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")), handle), 2.seconds
+      Failed(11, Option("14: Wrong errorCode."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")), handle), 2.seconds
     ).isInstanceOf[FailedNonRetryableExecutionHandle] shouldBe true
     Await.result(jesBackend.executionResult(
-      Failed(10, List("Weird error message."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")), handle), 2.seconds
+      Failed(10, Option("Weird error message."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")), handle), 2.seconds
     ).isInstanceOf[FailedNonRetryableExecutionHandle] shouldBe true
     Await.result(jesBackend.executionResult(
-      Failed(10, List("UnparsableInt: Even weirder error message."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")), handle), 2.seconds
+      Failed(10, Option("UnparsableInt: Even weirder error message."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")), handle), 2.seconds
     ).isInstanceOf[FailedNonRetryableExecutionHandle] shouldBe true
     Await.result(jesBackend.executionResult(
-      Failed(10, List.empty, Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")), handle), 2.seconds
+      Failed(10, Option.empty, Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")), handle), 2.seconds
     ).isInstanceOf[FailedNonRetryableExecutionHandle] shouldBe true
     Await.result(jesBackend.executionResult(
-      Failed(10, List("Operation canceled at"), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")), handle), 2.seconds
+      Failed(10, Option("Operation canceled at"), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")), handle), 2.seconds
     ) shouldBe AbortedExecutionHandle
 
     actorRef.stop()
diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala
index 844fe73d8..d810e7f8a 100644
--- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala
+++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala
@@ -5,6 +5,7 @@ import java.util.UUID
 import akka.testkit._
 import com.typesafe.config.{Config, ConfigFactory}
 import cromwell.backend.BackendWorkflowInitializationActor.{InitializationFailed, InitializationSuccess, Initialize}
+import cromwell.backend.async.RuntimeAttributeValidationFailures
 import cromwell.backend.impl.jes.authentication.GcsLocalizing
 import cromwell.backend.{BackendConfigurationDescriptor, BackendSpec, BackendWorkflowDescriptor}
 import cromwell.core.Tags.IntegrationTest
@@ -173,9 +174,9 @@ class JesInitializationActorSpec extends TestKitSuite("JesInitializationActorSpe
       expectMsgPF() {
         case InitializationFailed(failure) =>
           failure match {
-            case exception: IllegalArgumentException =>
-              if (!exception.getMessage.equals("Task hello has an invalid runtime attribute docker = !! NOT FOUND !!"))
-                fail("Exception message does not contains 'Runtime attribute validation failed'.")
+            case exception: RuntimeAttributeValidationFailures =>
+              if (!exception.getMessage.equals("Runtime validation failed:\nTask hello has an invalid runtime attribute docker = !! NOT FOUND !!"))
+                fail("Exception message is not equal to 'Runtime validation failed:\nTask hello has an invalid runtime attribute docker = !! NOT FOUND !!'.")
           }
       }
     }
diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActorSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActorSpec.scala
index aa29b0df3..05b655c8b 100644
--- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActorSpec.scala
+++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActorSpec.scala
@@ -61,7 +61,7 @@ class JesPollingActorSpec extends TestKitSuite("JesPollingActor") with FlatSpecL
     jpActor.underlyingActor.callbackResponses :+= CallbackFailure
 
     val successStatus = RunStatus.Success(Seq.empty[ExecutionEvent], None, None, None)
-    val failureStatus = RunStatus.Failed(-1, List.empty[String], Seq.empty[ExecutionEvent], None, None, None)
+    val failureStatus = RunStatus.Failed(-1, Option.empty[String], Seq.empty[ExecutionEvent], None, None, None)
     jpActor.underlyingActor.operationStatusResponses :+= successStatus
     jpActor.underlyingActor.operationStatusResponses :+= failureStatus
 
diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala
index 518971070..cfbd2348d 100644
--- a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala
+++ b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala
@@ -56,7 +56,7 @@ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSyst
 
   it should "send back an execution failure if the task fails" in {
     val expectedResponse =
-      JobFailedNonRetryableResponse(mock[BackendJobDescriptorKey], WrongReturnCode("wf_goodbye.goodbye:NA:1", 1, Paths.get("")), Option(1))
+      JobFailedNonRetryableResponse(mock[BackendJobDescriptorKey], WrongReturnCode("wf_goodbye.goodbye:NA:1", 1, None), Option(1))
     val workflow = TestWorkflow(buildWorkflowDescriptor(GoodbyeWorld), emptyBackendConfig, expectedResponse)
     val backend = createBackend(jobDescriptorFromSingleCallWorkflow(workflow.workflowDescriptor, Map.empty, WorkflowOptions.empty, runtimeAttributeDefinitions), workflow.config)
     testWorkflow(workflow, backend)

From 5da6fae54b8850f26dbad088249cfa0535eee5f9 Mon Sep 17 00:00:00 2001
From: Miguel Covarrubias <mcovarr@broadinstitute.org>
Date: Thu, 19 Jan 2017 12:07:07 -0500
Subject: [PATCH 22/84] Happy Birthday Ruchi! Also, fix prefix by upgrading the
 wdl4s dependency.

---
 project/Dependencies.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/project/Dependencies.scala b/project/Dependencies.scala
index 31909fc5e..8711f0c08 100644
--- a/project/Dependencies.scala
+++ b/project/Dependencies.scala
@@ -2,7 +2,7 @@ import sbt._
 
 object Dependencies {
   lazy val lenthallV = "0.20"
-  lazy val wdl4sV = "0.9-369e288-SNAP"
+  lazy val wdl4sV = "0.9-9e4b343-SNAP"
   lazy val sprayV = "1.3.3"
   /*
   spray-json is an independent project from the "spray suite"

From a70e5776314530c4073d048fec928c2a7c6b9067 Mon Sep 17 00:00:00 2001
From: Khalid Shakir <kshakir@broadinstitute.org>
Date: Thu, 19 Jan 2017 18:14:20 -0500
Subject: [PATCH 23/84] Re-adding Standard (JES & SFS) backendStatus to
 metadata.

---
 .../backend/standard/StandardAsyncExecutionActor.scala       | 12 +-----------
 .../engine/workflow/SingleWorkflowRunnerActorSpec.scala      |  4 ++--
 .../backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala |  4 +++-
 3 files changed, 6 insertions(+), 14 deletions(-)

diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala b/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala
index 95ebc8a11..24b906494 100644
--- a/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala
+++ b/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala
@@ -150,14 +150,6 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
   }
 
   /**
-    * Maps the status to a status string that should be stored in the metadata.
-    *
-    * @param runStatus The run status.
-    * @return Some() string that should be stored, or None if nothing should be stored in the metadata.
-    */
-  def statusString(runStatus: StandardAsyncRunStatus): Option[String] = None
-
-  /**
     * Returns true when a job is complete, either successfully or unsuccessfully.
     *
     * @param runStatus The run status.
@@ -324,9 +316,7 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
        */
       val prevStateName = previousStatus.map(_.toString).getOrElse("-")
       jobLogger.info(s"$tag Status change from $prevStateName to $status")
-      statusString(status) foreach { statusMetadata =>
-        tellMetadata(Map(CallMetadataKeys.BackendStatus -> statusMetadata))
-      }
+      tellMetadata(Map(CallMetadataKeys.BackendStatus -> status))
     }
 
     status match {
diff --git a/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala
index 6dec787d5..90f3c0fa5 100644
--- a/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala
+++ b/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala
@@ -141,7 +141,7 @@ class SingleWorkflowRunnerActorWithMetadataSpec extends SingleWorkflowRunnerActo
       inputs should have size numInputs
       call.get("executionStatus").toStringValue should be("Done")
       call.get("backend").toStringValue should be("Local")
-      call.get("backendStatus") should be(empty)
+      call.get("backendStatus").toStringValue should be("Done")
       call.get("outputs").toFields should have size numOutputs
       val callStart = call.get("start").toOffsetDateTime
       callStart should be >= workflowStart
@@ -212,7 +212,7 @@ class SingleWorkflowRunnerActorWithMetadataOnFailureSpec extends SingleWorkflowR
       inputs should have size 0
       call.get("executionStatus").toStringValue should be("Failed")
       call.get("backend").toStringValue should be("Local")
-      call.get("backendStatus") should be(empty)
+      call.get("backendStatus").toStringValue should be("Done")
       call.get("outputs") shouldBe empty
       val callStart = call.get("start").toOffsetDateTime
       callStart should be >= workflowStart
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala
index 170d24437..09cac5a8e 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala
@@ -19,7 +19,9 @@ import wdl4s.values.{WdlArray, WdlFile, WdlGlobFile, WdlMap, WdlValue}
 import scala.concurrent.duration._
 import scala.util.{Failure, Success, Try}
 
-case class SharedFileSystemRunStatus(returnCodeFileExists: Boolean)
+case class SharedFileSystemRunStatus(returnCodeFileExists: Boolean) {
+  override def toString: String = if (returnCodeFileExists) "Done" else "WaitingForReturnCodeFile"
+}
 
 object SharedFileSystemAsyncJobExecutionActor {
   val JobIdKey = "sfs_job_id"

From 1ab0df1e4225fb272ecc7c19292038b6c1d51de4 Mon Sep 17 00:00:00 2001
From: Thib <Horneth@users.noreply.github.com>
Date: Fri, 20 Jan 2017 13:24:27 -0500
Subject: [PATCH 24/84] Some scripts to bring up a cromwell server w/ mysql
 using docker compose (#1878)

* add some docker-compose scripts
---
 .gitignore                                         |  1 +
 scripts/docker-compose-mysql/README.md             | 32 ++++++++++
 .../compose/cromwell/Dockerfile                    |  9 +++
 .../compose/cromwell/app-config/application.conf   | 55 +++++++++++++++++
 .../compose/mysql/init/init_user.sql               |  4 ++
 scripts/docker-compose-mysql/docker-compose.yml    | 22 +++++++
 .../jes-cromwell/docker-compose.yml                |  9 +++
 .../jes-cromwell/jes-config/application.conf       | 68 ++++++++++++++++++++++
 8 files changed, 200 insertions(+)
 create mode 100644 scripts/docker-compose-mysql/README.md
 create mode 100644 scripts/docker-compose-mysql/compose/cromwell/Dockerfile
 create mode 100644 scripts/docker-compose-mysql/compose/cromwell/app-config/application.conf
 create mode 100644 scripts/docker-compose-mysql/compose/mysql/init/init_user.sql
 create mode 100644 scripts/docker-compose-mysql/docker-compose.yml
 create mode 100644 scripts/docker-compose-mysql/jes-cromwell/docker-compose.yml
 create mode 100644 scripts/docker-compose-mysql/jes-cromwell/jes-config/application.conf

diff --git a/.gitignore b/.gitignore
index f9bb0a4f5..7915ca4d0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,3 +8,4 @@ cromwell-workflow-logs
 cromwell-test-workflow-logs
 local-cromwell-executions
 native
+scripts/docker-compose-mysql/compose/mysql/data
diff --git a/scripts/docker-compose-mysql/README.md b/scripts/docker-compose-mysql/README.md
new file mode 100644
index 000000000..c6ea57d08
--- /dev/null
+++ b/scripts/docker-compose-mysql/README.md
@@ -0,0 +1,32 @@
+# Cromwell server on MySQL Database
+
+Uses docker-compose to link together a Cromwell docker image (built locally with `sbt docker` or available on [dockerhub](https://hub.docker.com/r/broadinstitute/cromwell/)) with a MySQL docker image.
+To change the version of Cromwell used, change the tag in `compose/cromwell/Dockerfile`
+
+## Local
+
+`docker-compose up` from this directory will start a Cromwell server running on a mysql instance with local backend.
+The default configuration file used can be found at `compose/cromwell/app-config/application.conf`
+To override it, simply mount a volume containing your custom `application.conf` to `/app-config` (see `jes-cromwell/docker-compose.yml` for an example)
+
+## JES
+
+The `jes-cromwell` directory is an example of how to customize the original compose file with a configuration file and environment variables.
+It uses the application default credentials of the host machine. To use it make sure your gcloud is up to date and that your [application-default credentials](https://developers.google.com/identity/protocols/application-default-credentials) are set up.
+Then run `docker-compose -f docker-compose.yml -f jes-cromwell/docker-compose.yml up` to start a Cromwell server with a JES backend on MySQL.
+
+## MySQL
+
+The data directory in the MySQL container is mounted to `compose/mysql/data`, which allows the data to survive a `docker-compose down`.
+To disable this feature, simply remove the `./compose/mysql/data:/var/lib/mysql` line in the volume section of `docker-compose.yml`.
+Note that in such case, the data will still be preserved by a `docker-compose stop` that stops the container but doesn't delete it.
+
+## Notes
+
+To run Cromwell in the background, add `-d` at the end of the command:
+`docker-compose up -d`
+
+To then see the logs for a specific service, run `docker-compose logs -f <service>`. 
+For example `docker-compose logs -f cromwell`.
+
+For more information about docker compose: [Docker compose doc](https://docs.docker.com/compose/)
diff --git a/scripts/docker-compose-mysql/compose/cromwell/Dockerfile b/scripts/docker-compose-mysql/compose/cromwell/Dockerfile
new file mode 100644
index 000000000..6bf9fd88b
--- /dev/null
+++ b/scripts/docker-compose-mysql/compose/cromwell/Dockerfile
@@ -0,0 +1,9 @@
+FROM broadinstitute/cromwell:develop
+
+RUN git clone https://github.com/vishnubob/wait-for-it.git
+RUN mkdir cromwell-working-dir
+WORKDIR cromwell-working-dir
+
+COPY ./app-config /app-config
+
+ENTRYPOINT ["/bin/sh", "-c"]
diff --git a/scripts/docker-compose-mysql/compose/cromwell/app-config/application.conf b/scripts/docker-compose-mysql/compose/cromwell/app-config/application.conf
new file mode 100644
index 000000000..2869bc5fd
--- /dev/null
+++ b/scripts/docker-compose-mysql/compose/cromwell/app-config/application.conf
@@ -0,0 +1,55 @@
+call-caching {
+  enabled = false
+}
+
+backend {
+  default = "Local"
+  providers {
+    Local {
+      actor-factory = "cromwell.backend.impl.sfs.config.ConfigBackendLifecycleActorFactory"
+      config {
+        run-in-background = true
+        runtime-attributes = "String? docker Int? max_runtime = 2"
+        submit = "/bin/bash ${script}"
+        submit-docker = "docker run --rm -v ${cwd}:${docker_cwd} -i ${docker} /bin/bash < ${script}"
+
+        # Root directory where Cromwell writes job results. This directory must be
+        # visible and writeable by the Cromwell process as well as the jobs that Cromwell
+        # launches.
+        root: "cromwell-executions"
+
+        filesystems {
+          local {
+            localization: [
+              "soft-link", "hard-link", "copy"
+            ]
+
+            caching {
+              duplication-strategy: [
+                "soft-link"
+              ]
+
+              # Possible values: file, path
+              # "file" will compute an md5 hash of the file content.
+              # "path" will compute an md5 hash of the file path. This strategy will only be effective if the duplication-strategy (above) is set to "soft-link",
+              # in order to allow for the original file path to be hashed.
+              hashing-strategy: "path"
+
+              # When true, will check if a sibling file with the same name and the .md5 extension exists, and if it does, use the content of this file as a hash.
+              # If false or the md5 does not exist, will proceed with the above-defined hashing strategy.
+              check-sibling-md5: false
+            }
+          }
+        }
+      }
+    }
+  }
+}
+
+database {
+  db.url = "jdbc:mysql://mysql-db/cromwell_db?useSSL=false&rewriteBatchedStatements=true"
+  db.user = "cromwell"
+  db.password = "cromwell"
+  db.driver = "com.mysql.jdbc.Driver"
+  driver = "slick.driver.MySQLDriver$"
+}
diff --git a/scripts/docker-compose-mysql/compose/mysql/init/init_user.sql b/scripts/docker-compose-mysql/compose/mysql/init/init_user.sql
new file mode 100644
index 000000000..b07938189
--- /dev/null
+++ b/scripts/docker-compose-mysql/compose/mysql/init/init_user.sql
@@ -0,0 +1,4 @@
+CREATE USER 'cromwell'@'localhost' IDENTIFIED BY 'cromwell';
+GRANT ALL PRIVILEGES ON cromwell_db.* TO 'cromwell'@'localhost' WITH GRANT OPTION;
+CREATE USER 'cromwell'@'%' IDENTIFIED BY 'cromwell';
+GRANT ALL PRIVILEGES ON cromwell_db.* TO 'cromwell'@'%' WITH GRANT OPTION;
diff --git a/scripts/docker-compose-mysql/docker-compose.yml b/scripts/docker-compose-mysql/docker-compose.yml
new file mode 100644
index 000000000..c1a8e21f3
--- /dev/null
+++ b/scripts/docker-compose-mysql/docker-compose.yml
@@ -0,0 +1,22 @@
+version: '2'
+services:
+  cromwell:
+    build: 
+      context: ./compose/cromwell
+    volumes:
+      - ./cromwell-executions:/cromwell-working-dir/cromwell-executions
+    command: ["/wait-for-it/wait-for-it.sh mysql-db:3306 -t 120 -- java -Dconfig.file=/app-config/application.conf -jar /app/cromwell.jar server"]
+    links:
+      - mysql-db
+    ports:
+      - "80:8000"
+  mysql-db:
+    image: "mysql:5.7"
+    environment:
+      - MYSQL_ROOT_PASSWORD=cromwell
+      - MYSQL_DATABASE=cromwell_db
+    volumes:
+      - ./compose/mysql/init:/docker-entrypoint-initdb.d
+      - ./compose/mysql/data:/var/lib/mysql
+    ports:
+      - "3307:3306"
diff --git a/scripts/docker-compose-mysql/jes-cromwell/docker-compose.yml b/scripts/docker-compose-mysql/jes-cromwell/docker-compose.yml
new file mode 100644
index 000000000..c7f0e1cee
--- /dev/null
+++ b/scripts/docker-compose-mysql/jes-cromwell/docker-compose.yml
@@ -0,0 +1,9 @@
+version: '2'
+services:
+  cromwell:
+    volumes:
+      - $HOME/.config/gcloud:/root/.config/gcloud
+      - ./jes-cromwell/jes-config:/app-config
+    environment:
+      - GOOGLE_APPLICATION_CREDENTIALS=/root/.config/gcloud/application_default_credentials.json
+
diff --git a/scripts/docker-compose-mysql/jes-cromwell/jes-config/application.conf b/scripts/docker-compose-mysql/jes-cromwell/jes-config/application.conf
new file mode 100644
index 000000000..aab9925ff
--- /dev/null
+++ b/scripts/docker-compose-mysql/jes-cromwell/jes-config/application.conf
@@ -0,0 +1,68 @@
+system.new-workflow-poll-rate=1
+
+google {
+
+  application-name = "cromwell"
+
+  auths = [
+    {
+      name = "application-default"
+      scheme = "application_default"
+    }
+  ]
+}
+
+engine {
+  filesystems {
+     gcs {
+       auth = "application-default"
+     }
+  }
+}
+
+call-caching {
+  enabled = false
+}
+
+backend {
+  default = "Jes"
+  providers {
+    Jes {
+          actor-factory = "cromwell.backend.impl.jes.JesBackendLifecycleActorFactory"
+          config {
+            // Google project
+            project = "broad-dsde-cromwell-dev"
+
+            // Base bucket for workflow executions
+            root = "gs://tjeandet-cromwell-execs"
+
+            // Polling for completion backs-off gradually for slower-running jobs.
+            // This is the maximum polling interval (in seconds):
+            maximum-polling-interval = 600
+
+            genomics {
+              // A reference to an auth defined in the `google` stanza at the top.  This auth is used to create
+              // Pipelines and manipulate auth JSONs.
+              auth = "application-default"
+              // Endpoint for APIs, no reason to change this unless directed by Google.
+              endpoint-url = "https://genomics.googleapis.com/"
+            }
+
+            filesystems {
+              gcs {
+                // A reference to a potentially different auth for manipulating files via engine functions.
+                auth = "application-default"
+              }
+            }
+          }
+    }
+  }
+}
+
+database {
+  db.url = "jdbc:mysql://mysql-db/cromwell_db?useSSL=false&rewriteBatchedStatements=true"
+  db.user = "cromwell"
+  db.password = "cromwell"
+  db.driver = "com.mysql.jdbc.Driver"
+  driver = "slick.driver.MySQLDriver$"
+}

From 5213592fc7d15c38edeedd222751bad0559ce14f Mon Sep 17 00:00:00 2001
From: Miguel Covarrubias <mcovarr@broadinstitute.org>
Date: Fri, 20 Jan 2017 13:36:27 -0500
Subject: [PATCH 25/84] Add missing docs

---
 CHANGELOG.md | 8 +++++++-
 README.md    | 1 +
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8dcead856..15f868eb7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,9 +6,15 @@
 * Cromwell now applies default labels automatically to JES pipeline runs.
 * Added ability to override the default zone(s) used by JES via the config structure by setting `genomics.default-zones` in the JES configuration
 * Added support for new WDL functions:
-  * `length: (Array[X]) => Integer` - report the length of the specified array 
+  * `length: (Array[X]) => Integer` - report the length of the specified array
+  * `prefix: (String, Array[X]) => Array[String]` - generate an array consisting of each element of the input array prefixed
+     by a specified `String`.  The input array can have elements of any primitive type, the return array will always have
+     type `Array[String]`.
 * The cromwell server TCP binding timeout is now configurable via the config key `webservice.binding-timeout`, defaulted
   to the previous value `5s` (five seconds) via the reference.conf.
+* Cromwell's Config (Shared Filesystem) backend now supports invocation of commands which run in a Docker image as a non-root user.
+  The non-root user could either be the default user for a given Docker image (e.g. specified in a Dockerfile via a `USER` directive),
+  or the Config backend could pass an optional `"-u username"` as part of the `submit-docker` command.
 
 ### Database schema changes
 * Added CUSTOM_LABELS as a field of WORKFLOW_STORE_ENTRY, to store workflow store entries.
diff --git a/README.md b/README.md
index 0390470dc..3072d32e8 100644
--- a/README.md
+++ b/README.md
@@ -382,6 +382,7 @@ For many examples on how to use WDL see [the WDL site](https://github.com/broadi
   * [Pair(X,Y) zip(X,Y)](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#pairxy-zipxy)
   * [Pair(X,Y) cross(X,Y)](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#pairxy-crossxy)
   * [Integer length(Array\[X\])](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#integer-lengtharrayx)
+  * [Array\[String\] prefix(String, Array\[X\])](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#arraystring-prefixstring-arrayx)
 * [Data Types & Serialization](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#data-types--serialization)
   * [Serialization of Task Inputs](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#serialization-of-task-inputs)
     * [Primitive Types](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#primitive-types)

From da35cadd70da97d4a65120000254531d0ec34235 Mon Sep 17 00:00:00 2001
From: Ruchi <rmunshi@broadinstitute.org>
Date: Fri, 20 Jan 2017 14:12:17 -0500
Subject: [PATCH 26/84] Develop - Update detritus value data type closes #1862
 (#1869)

* Update detritus value data type to longtext

* add changeset to changelog

* convert clob to raw string
---
 database/migration/src/main/resources/changelog.xml    |  1 +
 .../resources/changesets/embiggen_detritus_value.xml   | 18 ++++++++++++++++++
 .../tables/CallCachingDetritusEntryComponent.scala     |  4 +++-
 .../database/sql/tables/CallCachingDetritusEntry.scala |  4 +++-
 .../lifecycle/execution/callcaching/CallCache.scala    |  3 ++-
 .../callcaching/FetchCachedResultsActor.scala          |  3 ++-
 6 files changed, 29 insertions(+), 4 deletions(-)
 create mode 100644 database/migration/src/main/resources/changesets/embiggen_detritus_value.xml

diff --git a/database/migration/src/main/resources/changelog.xml b/database/migration/src/main/resources/changelog.xml
index 1deb8d50c..cbbfe50f0 100644
--- a/database/migration/src/main/resources/changelog.xml
+++ b/database/migration/src/main/resources/changelog.xml
@@ -55,6 +55,7 @@
     <include file="changesets/sub_workflow_store.xml" relativeToChangelogFile="true" />
     <include file="changesets/workflow_store_imports_file.xml" relativeToChangelogFile="true" />
     <include file="changesets/workflow_store_labels_file.xml" relativeToChangelogFile="true" />
+    <include file="changesets/embiggen_detritus_value.xml" relativeToChangelogFile="true" />
 </databaseChangeLog>
 <!--
 
diff --git a/database/migration/src/main/resources/changesets/embiggen_detritus_value.xml b/database/migration/src/main/resources/changesets/embiggen_detritus_value.xml
new file mode 100644
index 000000000..096f17a4c
--- /dev/null
+++ b/database/migration/src/main/resources/changesets/embiggen_detritus_value.xml
@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
+
+    <property name="clob.type" value="LONGTEXT" dbms="mysql"/>
+    <property name="clob.type" value="LONGVARCHAR" dbms="hsqldb"/>
+
+    <changeSet author="rmunshi" id="embiggen_call_caching_detritus_entry">
+        <modifyDataType tableName="CALL_CACHING_DETRITUS_ENTRY"
+                        columnName="DETRITUS_VALUE"
+                        newDataType="${clob.type}"/>
+        <comment>
+            The Call Caching Detritus Entry version of the embiggener.
+        </comment>
+    </changeSet>
+
+</databaseChangeLog>
diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingDetritusEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingDetritusEntryComponent.scala
index b3afe6ac0..eae065b1c 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingDetritusEntryComponent.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingDetritusEntryComponent.scala
@@ -1,5 +1,7 @@
 package cromwell.database.slick.tables
 
+import java.sql.Clob
+
 import cromwell.database.sql.tables.CallCachingDetritusEntry
 
 trait CallCachingDetritusEntryComponent {
@@ -14,7 +16,7 @@ trait CallCachingDetritusEntryComponent {
 
     def detritusKey = column[String]("DETRITUS_KEY")
 
-    def detritusValue = column[String]("DETRITUS_VALUE")
+    def detritusValue = column[Clob]("DETRITUS_VALUE")
 
     def callCachingEntryId = column[Int]("CALL_CACHING_ENTRY_ID")
 
diff --git a/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingDetritusEntry.scala b/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingDetritusEntry.scala
index 976d4ce13..9e325d5c1 100644
--- a/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingDetritusEntry.scala
+++ b/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingDetritusEntry.scala
@@ -1,9 +1,11 @@
 package cromwell.database.sql.tables
 
+import java.sql.Clob
+
 case class CallCachingDetritusEntry
 (
   detritusKey: String,
-  detritusValue: String,
+  detritusValue: Clob,
   callCachingEntryId: Option[Int] = None,
   callCachingDetritusId: Option[Int] = None
 )
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala
index 53448d397..9a149155e 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala
@@ -9,6 +9,7 @@ import cromwell.core.WorkflowId
 import cromwell.core.callcaching.HashResult
 import cromwell.core.simpleton.WdlValueSimpleton
 import cromwell.core.path.PathImplicits._
+import cromwell.database.sql.SqlConverters._
 import cromwell.database.sql._
 import cromwell.database.sql.joins.CallCachingJoin
 import cromwell.database.sql.tables.{CallCachingDetritusEntry, CallCachingEntry, CallCachingHashEntry, CallCachingSimpletonEntry}
@@ -54,7 +55,7 @@ class CallCache(database: CallCachingSqlDatabase) {
 
     val jobDetritusToInsert: Iterable[CallCachingDetritusEntry] = {
       jobDetritus map {
-        case (fileName, filePath) => CallCachingDetritusEntry(fileName, filePath.toRealString)
+        case (fileName, filePath) => CallCachingDetritusEntry(fileName, filePath.toRealString.toClob)
       }
     }
 
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/FetchCachedResultsActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/FetchCachedResultsActor.scala
index ab00d5856..194561b8c 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/FetchCachedResultsActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/FetchCachedResultsActor.scala
@@ -4,6 +4,7 @@ import akka.actor.{Actor, ActorLogging, ActorRef, Props}
 import cromwell.Simpletons._
 import cromwell.core.Dispatcher.EngineDispatcher
 import cromwell.core.simpleton.WdlValueSimpleton
+import cromwell.database.sql.SqlConverters._
 import cromwell.engine.workflow.lifecycle.execution.callcaching.FetchCachedResultsActor.{CachedOutputLookupFailed, CachedOutputLookupSucceeded}
 
 import scala.concurrent.ExecutionContext
@@ -30,7 +31,7 @@ class FetchCachedResultsActor(cacheResultId: CallCachingEntryId, replyTo: ActorR
       case Success(Some(result)) =>
         val simpletons = result.callCachingSimpletonEntries map toSimpleton
         val jobDetritusFiles = result.callCachingDetritusEntries map { jobDetritusEntry =>
-          jobDetritusEntry.detritusKey -> jobDetritusEntry.detritusValue
+          jobDetritusEntry.detritusKey -> jobDetritusEntry.detritusValue.toRawString
         }
         val sourceCacheDetails = Seq(result.callCachingEntry.workflowExecutionUuid,
                                     result.callCachingEntry.callFullyQualifiedName,

From 4c88c70f2da90e040b2701b762787cbea482d4ca Mon Sep 17 00:00:00 2001
From: Khalid Shakir <kshakir@broadinstitute.org>
Date: Mon, 23 Jan 2017 13:05:08 -0500
Subject: [PATCH 27/84] Disabled type annotation inspections in IntelliJ.
 Synced common scala .gitignore across repositories.

---
 .gitignore                                   | 16 ++++++++++++----
 .idea/inspectionProfiles/Project_Default.xml |  6 ++++++
 2 files changed, 18 insertions(+), 4 deletions(-)
 create mode 100644 .idea/inspectionProfiles/Project_Default.xml

diff --git a/.gitignore b/.gitignore
index 7915ca4d0..21f16ea18 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,11 +1,19 @@
-.idea
-log
-target
+# common scala config
+*~
+.DS_Store
 .artifactory
+.idea/*
+!/.idea/inspectionProfiles/
+.idea/inspectionProfiles/*
+!/.idea/inspectionProfiles/Project_Default.xml
+target
+
+# custom config
 cromwell-executions
 cromwell-test-executions
-cromwell-workflow-logs
 cromwell-test-workflow-logs
+cromwell-workflow-logs
 local-cromwell-executions
+log
 native
 scripts/docker-compose-mysql/compose/mysql/data
diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml
new file mode 100644
index 000000000..b15d7f2ad
--- /dev/null
+++ b/.idea/inspectionProfiles/Project_Default.xml
@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <profile version="1.0">
+    <option name="myName" value="Project Default" />
+    <inspection_tool class="TypeAnnotation" enabled="false" level="WARNING" enabled_by_default="false" />
+  </profile>
+</component>
\ No newline at end of file

From 88b8aee5b3332e4994d7db172045015c58c8cca9 Mon Sep 17 00:00:00 2001
From: Jeff Gentry <geoffjentry@gmail.com>
Date: Mon, 23 Jan 2017 17:03:47 -0500
Subject: [PATCH 28/84] Rework structure of workflow store and metadata,
 provides roughly 5-fold performance increase under load prior to DB gumming
 up (#1836)

---
 CHANGELOG.md                                       |   1 +
 README.md                                          |   2 +-
 core/src/main/resources/reference.conf             |  16 +-
 .../database/slick/MetadataSlickDatabase.scala     |   2 +-
 .../slick/tables/MetadataEntryComponent.scala      |   2 +-
 .../workflow/SingleWorkflowRunnerActor.scala       |   6 +-
 .../engine/workflow/WorkflowManagerActor.scala     |  10 +-
 .../workflowstore/WorkflowStoreActor.scala         | 245 +-------
 .../workflowstore/WorkflowStoreEngineActor.scala   | 166 ++++++
 .../workflowstore/WorkflowStoreSubmitActor.scala   | 102 ++++
 .../cromwell/webservice/CromwellApiHandler.scala   |  10 +-
 .../cromwell/webservice/CromwellApiService.scala   |  21 +-
 .../test/scala/cromwell/CromwellTestKitSpec.scala  |   2 +-
 .../cromwell/engine/WorkflowStoreActorSpec.scala   |   2 +
 .../subworkflowstore/SubWorkflowStoreSpec.scala    |   3 +-
 .../webservice/CromwellApiServiceSpec.scala        | 623 ++++++++-------------
 project/Dependencies.scala                         |   2 +-
 .../metadata/impl/MetadataDatabaseAccess.scala     |   4 +-
 .../metadata/impl/MetadataServiceActor.scala       |   9 +-
 .../metadata/impl/WriteMetadataActor.scala         |  94 +++-
 .../metadata/impl/MetadataServiceActorSpec.scala   |  51 +-
 .../services/metadata/impl/MetadataValueSpec.scala |  11 +
 .../metadata/impl/WriteMetadataActorSpec.scala     |  44 ++
 23 files changed, 718 insertions(+), 710 deletions(-)
 create mode 100644 engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreEngineActor.scala
 create mode 100644 engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreSubmitActor.scala
 create mode 100644 services/src/test/scala/cromwell/services/metadata/impl/MetadataValueSpec.scala
 create mode 100644 services/src/test/scala/cromwell/services/metadata/impl/WriteMetadataActorSpec.scala

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 15f868eb7..220372fa3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -15,6 +15,7 @@
 * Cromwell's Config (Shared Filesystem) backend now supports invocation of commands which run in a Docker image as a non-root user.
   The non-root user could either be the default user for a given Docker image (e.g. specified in a Dockerfile via a `USER` directive),
   or the Config backend could pass an optional `"-u username"` as part of the `submit-docker` command.
+* For MySQL users, a massive scalability improvement via batched DB writing of internal metadata events. Note that one must add `rewriteBatchedStatements=true` to their JDBC URL in their config in order to take advantage of this
 
 ### Database schema changes
 * Added CUSTOM_LABELS as a field of WORKFLOW_STORE_ENTRY, to store workflow store entries.
diff --git a/README.md b/README.md
index 3072d32e8..554dcbb3e 100644
--- a/README.md
+++ b/README.md
@@ -465,7 +465,7 @@ database {
   driver = "slick.driver.MySQLDriver$"
   db {
     driver = "com.mysql.jdbc.Driver"
-    url = "jdbc:mysql://host/cromwell"
+    url = "jdbc:mysql://host/cromwell?rewriteBatchedStatements=true"
     user = "user"
     password = "pass"
     connectionTimeout = 5000
diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf
index 32f88e9c1..bf98ab431 100644
--- a/core/src/main/resources/reference.conf
+++ b/core/src/main/resources/reference.conf
@@ -397,6 +397,20 @@ services {
     config {
       # Set this value to "Inf" to turn off metadata summary refresh.  The default value is currently "2 seconds".
       # metadata-summary-refresh-interval = "Inf"
+      # For higher scale environments, e.g. many workflows and/or jobs, DB write performance for metadata events
+      # can improved by writing to the database in batches. Increasing this value can dramatically improve overall
+      # performance but will both lead to a higher memory usage as well as increase the risk that metadata events
+      # might not have been persisted in the event of a Cromwell crash.
+      #
+      # For normal usage the default value of 1 (effectively no batching) should be fine but for larger/production
+      # environments we recommend a value of at least 500. There'll be no one size fits all number here so we recommend
+      # benchmarking performance and tuning the value to match your environment
+      # db-batch-size = 1
+      #
+      # Periodically the stored metadata events will be forcibly written to the DB regardless of if the batch size
+      # has been reached. This is to prevent situations where events wind up never being written to an incomplete batch
+      # with no new events being generated. The default value is currently 5 seconds
+      # db-flush-rate = 5 seconds
     }
   }
 }
@@ -414,7 +428,7 @@ database {
   #driver = "slick.driver.MySQLDriver$"
   #db {
   #  driver = "com.mysql.jdbc.Driver"
-  #  url = "jdbc:mysql://host/cromwell"
+  #  url = "jdbc:mysql://host/cromwell?rewriteBatchedStatements=true"
   #  user = "user"
   #  password = "pass"
   #  connectionTimeout = 5000
diff --git a/database/sql/src/main/scala/cromwell/database/slick/MetadataSlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/MetadataSlickDatabase.scala
index efc27b1f6..521581011 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/MetadataSlickDatabase.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/MetadataSlickDatabase.scala
@@ -15,7 +15,7 @@ trait MetadataSlickDatabase extends MetadataSqlDatabase {
 
   override def addMetadataEntries(metadataEntries: Iterable[MetadataEntry])
                                  (implicit ec: ExecutionContext): Future[Unit] = {
-    val action = dataAccess.metadataEntryIdsAutoInc ++= metadataEntries
+    val action = DBIO.seq(metadataEntries.grouped(2000).map(dataAccess.metadataEntries ++= _).toSeq:_*)
     runTransaction(action).map(_ => ())
   }
 
diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/MetadataEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/MetadataEntryComponent.scala
index 90c1898fc..4cc960213 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/tables/MetadataEntryComponent.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/tables/MetadataEntryComponent.scala
@@ -57,7 +57,7 @@ trait MetadataEntryComponent {
       (workflowExecutionUuid, metadataKey, callFullyQualifiedName, jobIndex, jobAttempt), unique = false)
   }
 
-  protected val metadataEntries = TableQuery[MetadataEntries]
+  val metadataEntries = TableQuery[MetadataEntries]
 
   val metadataEntryIdsAutoInc = metadataEntries returning metadataEntries.map(_.metadataEntryId)
 
diff --git a/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala b/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala
index 98b51c50a..49685bf7e 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala
@@ -13,7 +13,7 @@ import cromwell.core._
 import cromwell.core.Dispatcher.EngineDispatcher
 import cromwell.engine.workflow.SingleWorkflowRunnerActor._
 import cromwell.engine.workflow.WorkflowManagerActor.RetrieveNewWorkflows
-import cromwell.engine.workflow.workflowstore.{InMemoryWorkflowStore, WorkflowStoreActor}
+import cromwell.engine.workflow.workflowstore.{InMemoryWorkflowStore, WorkflowStoreEngineActor, WorkflowStoreSubmitActor}
 import cromwell.engine.workflow.workflowstore.WorkflowStoreActor.SubmitWorkflow
 import cromwell.jobstore.EmptyJobStoreActor
 import cromwell.server.CromwellRootActor
@@ -56,7 +56,7 @@ class SingleWorkflowRunnerActor(source: WorkflowSourceFilesCollection, metadataO
   }
 
   when (SubmittedWorkflow) {
-    case Event(WorkflowStoreActor.WorkflowSubmittedToStore(id), SubmittedSwraData(replyTo)) =>
+    case Event(WorkflowStoreSubmitActor.WorkflowSubmittedToStore(id), SubmittedSwraData(replyTo)) =>
       log.info(s"$Tag: Workflow submitted UUID($id)")
       // Since we only have a single workflow, force the WorkflowManagerActor's hand in case the polling rate is long
       workflowManagerActor ! RetrieveNewWorkflows
@@ -103,7 +103,7 @@ class SingleWorkflowRunnerActor(source: WorkflowSourceFilesCollection, metadataO
 
   whenUnhandled {
     // Handle failures for all failure responses generically.
-    case Event(r: WorkflowStoreActor.WorkflowAbortFailed, data) => failAndFinish(r.reason, data)
+    case Event(r: WorkflowStoreEngineActor.WorkflowAbortFailed, data) => failAndFinish(r.reason, data)
     case Event(Failure(e), data) => failAndFinish(e, data)
     case Event(Status.Failure(e), data) => failAndFinish(e, data)
     case Event(RequestComplete((_, snap)), data) => failAndFinish(new RuntimeException(s"Unexpected API completion message: $snap"), data)
diff --git a/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala b/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala
index 7b870407c..bc2019f40 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala
@@ -11,7 +11,7 @@ import cromwell.core.{WorkflowAborted, WorkflowId}
 import cromwell.engine.backend.BackendSingletonCollection
 import cromwell.engine.workflow.WorkflowActor._
 import cromwell.engine.workflow.WorkflowManagerActor._
-import cromwell.engine.workflow.workflowstore.{WorkflowStoreActor, WorkflowStoreState}
+import cromwell.engine.workflow.workflowstore.{WorkflowStoreActor, WorkflowStoreEngineActor, WorkflowStoreState}
 import cromwell.jobstore.JobStoreActor.{JobStoreWriteFailure, JobStoreWriteSuccess, RegisterWorkflowCompleted}
 import cromwell.services.metadata.MetadataService._
 import cromwell.webservice.EngineStatsActor
@@ -166,11 +166,11 @@ class WorkflowManagerActor(params: WorkflowManagerActorParams)
       val maxNewWorkflows = maxWorkflowsToLaunch min (maxWorkflowsRunning - stateData.workflows.size)
       params.workflowStore ! WorkflowStoreActor.FetchRunnableWorkflows(maxNewWorkflows)
       stay()
-    case Event(WorkflowStoreActor.NoNewWorkflowsToStart, stateData) =>
+    case Event(WorkflowStoreEngineActor.NoNewWorkflowsToStart, stateData) =>
       log.debug("WorkflowStore provided no new workflows to start")
       scheduleNextNewWorkflowPoll()
       stay()
-    case Event(WorkflowStoreActor.NewWorkflowsToStart(newWorkflows), stateData) =>
+    case Event(WorkflowStoreEngineActor.NewWorkflowsToStart(newWorkflows), stateData) =>
       val newSubmissions = newWorkflows map submitWorkflow
       log.info("Retrieved {} workflows from the WorkflowStoreActor", newSubmissions.toList.size)
       scheduleNextNewWorkflowPoll()
@@ -189,7 +189,7 @@ class WorkflowManagerActor(params: WorkflowManagerActorParams)
         case None =>
           // All cool, if we got this far the workflow ID was found in the workflow store so this workflow must have never
           // made it to the workflow manager.
-          replyTo ! WorkflowStoreActor.WorkflowAborted(id)
+          replyTo ! WorkflowStoreEngineActor.WorkflowAborted(id)
           stay()
       }
     case Event(AbortAllWorkflowsCommand, data) if data.workflows.isEmpty =>
@@ -214,7 +214,7 @@ class WorkflowManagerActor(params: WorkflowManagerActorParams)
         params.jobStoreActor ! RegisterWorkflowCompleted(workflowId)
         if (toState.workflowState == WorkflowAborted) {
           val replyTo = abortingWorkflowToReplyTo(workflowId)
-          replyTo ! WorkflowStoreActor.WorkflowAborted(workflowId)
+          replyTo ! WorkflowStoreEngineActor.WorkflowAborted(workflowId)
           abortingWorkflowToReplyTo -= workflowId
         } else {
           params.workflowStore ! WorkflowStoreActor.RemoveWorkflow(workflowId)
diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreActor.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreActor.scala
index 27b9f2b6c..bebd40823 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreActor.scala
@@ -1,242 +1,33 @@
 package cromwell.engine.workflow.workflowstore
 
-import java.time.OffsetDateTime
-
-import akka.actor.{ActorLogging, ActorRef, LoggingFSM, Props}
+import akka.actor.{Actor, ActorLogging, ActorRef, Props}
 import cats.data.NonEmptyList
 import cromwell.core._
 import cromwell.core.Dispatcher.EngineDispatcher
-import cromwell.engine.workflow.WorkflowManagerActor
-import cromwell.engine.workflow.WorkflowManagerActor.WorkflowNotFoundException
-import cromwell.engine.workflow.workflowstore.WorkflowStoreActor._
-import cromwell.engine.workflow.workflowstore.WorkflowStoreState.StartableState
-import cromwell.services.metadata.MetadataService.{MetadataPutAcknowledgement, PutMetadataAction}
-import cromwell.services.metadata.{MetadataEvent, MetadataKey, MetadataValue}
-import lenthall.util.TryUtil
-import org.apache.commons.lang3.exception.ExceptionUtils
-
-import scala.concurrent.{ExecutionContext, Future}
-import scala.util.{Failure, Success, Try}
-
-case class WorkflowStoreActor(store: WorkflowStore, serviceRegistryActor: ActorRef)
-  extends LoggingFSM[WorkflowStoreActorState, WorkflowStoreActorData] with ActorLogging {
-
-  implicit val ec: ExecutionContext = context.dispatcher
-
-  startWith(Unstarted, WorkflowStoreActorData(None, List.empty))
-  self ! InitializerCommand
 
-  when(Unstarted) {
-    case Event(InitializerCommand, _) =>
-      val work = store.initialize map { _ =>
-        log.debug("Workflow store initialization successful")
-      }
-      addWorkCompletionHooks(InitializerCommand, work)
-      goto(Working) using stateData.withCurrentCommand(InitializerCommand, sender)
-    case Event(x: WorkflowStoreActorCommand, _) =>
-      stay using stateData.withPendingCommand(x, sender)
-  }
-
-  when(Idle) {
-    case Event(cmd: WorkflowStoreActorCommand, _) =>
-      if (stateData.currentOperation.nonEmpty || stateData.pendingOperations.nonEmpty) {
-        log.error("Non-empty WorkflowStoreActorData when in Idle state: {}", stateData)
-      }
-      startNewWork(cmd, sender, stateData.withCurrentCommand(cmd, sender))
-  }
-
-  when(Working) {
-    case Event(WorkDone, data) =>
-      val newData = data.pop
-      newData.currentOperation match {
-        case None => goto(Idle) using newData
-        case Some(WorkflowStoreActorCommandWithSender(cmd, sndr)) => startNewWork(cmd, sndr, newData)
-      }
-    case Event(cmd: WorkflowStoreActorCommand, data) => stay using data.withPendingCommand(cmd, sender)
-  }
+final case class WorkflowStoreActor(store: WorkflowStore, serviceRegistryActor: ActorRef) extends Actor with ActorLogging {
+  import WorkflowStoreActor._
 
-  whenUnhandled {
-    case Event(MetadataPutAcknowledgement(_), _) =>
-      stay // Ignored
-    case Event(msg, _) =>
-      log.warning("Unexpected message to WorkflowStoreActor in state {} with data {}: {}", stateName, stateData, msg)
-      stay
-  }
-
-  onTransition {
-    case fromState -> toState =>
-      log.debug("WorkflowStore moving from {} (using {}) to {} (using {})", fromState, stateData, toState, nextStateData)
-  }
-
-  private def startNewWork(command: WorkflowStoreActorCommand, sndr: ActorRef, nextData: WorkflowStoreActorData) = {
-    val work: Future[Any] = command match {
-      case cmd @ SubmitWorkflow(sourceFiles) =>
-        storeWorkflowSources(NonEmptyList.of(sourceFiles)) map { ids =>
-          val id = ids.head
-          registerSubmissionWithMetadataService(id, sourceFiles)
-          sndr ! WorkflowSubmittedToStore(id)
-          log.info("Workflow {} submitted.", id)
-        }
-      case cmd @ BatchSubmitWorkflows(sources) =>
-        storeWorkflowSources(sources) map { ids =>
-          val assignedSources = ids.toList.zip(sources.toList)
-          assignedSources foreach { case (id, sourceFiles) => registerSubmissionWithMetadataService(id, sourceFiles) }
-          sndr ! WorkflowsBatchSubmittedToStore(ids)
-          log.info("Workflows {} submitted.", ids.toList.mkString(", "))
-        }
-      case cmd @ FetchRunnableWorkflows(n) =>
-        newWorkflowMessage(n) map { nwm =>
-          nwm match {
-            case NewWorkflowsToStart(workflows) => log.info("{} new workflows fetched", workflows.toList.size)
-            case NoNewWorkflowsToStart => log.debug("No workflows fetched")
-            case _ => log.error("Unexpected response from newWorkflowMessage({}): {}", n, nwm)
-          }
-          sndr ! nwm
-        }
-      case cmd @ AbortWorkflow(id, manager) =>
-        store.remove(id) map { removed =>
-          if (removed) {
-            log.debug(s"Workflow $id aborted and removed from the workflow store.")
-            manager ! WorkflowManagerActor.AbortWorkflowCommand(id, sndr)
-          } else {
-            sndr ! WorkflowAbortFailed(id, new WorkflowNotFoundException(s"Couldn't abort $id because no workflow with that ID is in progress"))
-          }
-        }
-      case cmd @ RemoveWorkflow(id) =>
-        store.remove(id) map { removed =>
-          if (removed) {
-            log.debug("Workflow {} removed from store successfully.", id)
-          } else {
-            log.warning(s"Attempted to remove ID {} from the WorkflowStore but it didn't exist", id)
-          }
-        }
-      case oops =>
-        log.error("Unexpected type of start work command: {}", oops.getClass.getSimpleName)
-        Future.successful(self ! WorkDone)
-    }
-    addWorkCompletionHooks(command, work)
-    goto(Working) using nextData
-  }
-
-  private def storeWorkflowSources(sources: NonEmptyList[WorkflowSourceFilesCollection]): Future[NonEmptyList[WorkflowId]] = {
-    for {
-      processedSources <- Future.fromTry(processSources(sources, _.asPrettyJson))
-      workflowIds <- store.add(processedSources)
-    } yield workflowIds
-  }
-
-  private def processSources(sources: NonEmptyList[WorkflowSourceFilesCollection],
-                             processOptions: WorkflowOptions => WorkflowOptionsJson):
-  Try[NonEmptyList[WorkflowSourceFilesCollection]] = {
-    val nelTries: NonEmptyList[Try[WorkflowSourceFilesCollection]] = sources map processSource(processOptions)
-    val seqTries: Seq[Try[WorkflowSourceFilesCollection]] = nelTries.toList
-    val trySeqs: Try[Seq[WorkflowSourceFilesCollection]] = TryUtil.sequence(seqTries)
-    val tryNel: Try[NonEmptyList[WorkflowSourceFilesCollection]] = trySeqs.map(seq => NonEmptyList.fromList(seq.toList).get)
-    tryNel
-  }
+  lazy val workflowStoreSubmitActor: ActorRef = context.actorOf(WorkflowStoreSubmitActor.props(store, serviceRegistryActor))
+  lazy val workflowStoreEngineActor: ActorRef = context.actorOf(WorkflowStoreEngineActor.props(store, serviceRegistryActor))
 
-  /**
-    * Runs processing on workflow source files before they are stored.
-    *
-    * @param processOptions How to process the workflow options
-    * @param source         Original workflow source
-    * @return Attempted updated workflow source
-    */
-  private def processSource(processOptions: WorkflowOptions => WorkflowOptionsJson)
-                           (source: WorkflowSourceFilesCollection): Try[WorkflowSourceFilesCollection] = {
-    for {
-      processedWorkflowOptions <- WorkflowOptions.fromJsonString(source.workflowOptionsJson)
-    } yield source.copyOptions(processOptions(processedWorkflowOptions))
-  }
-
-  private def addWorkCompletionHooks[A](command: WorkflowStoreActorCommand, work: Future[A]) = {
-    work.onComplete {
-      case Success(_) =>
-        self ! WorkDone
-      case Failure(t) =>
-        log.error("Error occurred during {}: {} because {}", command.getClass.getSimpleName, t.toString, ExceptionUtils.getStackTrace(t))
-        self ! WorkDone
-    }
-  }
-
-  /**
-    * Fetches at most n workflows, and builds the correct response message based on if there were any workflows or not
-    */
-  private def newWorkflowMessage(maxWorkflows: Int): Future[WorkflowStoreActorResponse] = {
-    def fetchRunnableWorkflowsIfNeeded(maxWorkflowsInner: Int, state: StartableState) = {
-      if (maxWorkflows > 0) {
-        store.fetchRunnableWorkflows(maxWorkflowsInner, state)
-      } else {
-        Future.successful(List.empty[WorkflowToStart])
-      }
-    }
-
-    val runnableWorkflows = for {
-      restartableWorkflows <- fetchRunnableWorkflowsIfNeeded(maxWorkflows, WorkflowStoreState.Restartable)
-      submittedWorkflows <- fetchRunnableWorkflowsIfNeeded(maxWorkflows - restartableWorkflows.size, WorkflowStoreState.Submitted)
-    } yield restartableWorkflows ++ submittedWorkflows
-
-    runnableWorkflows map {
-      case x :: xs => NewWorkflowsToStart(NonEmptyList.of(x, xs: _*))
-      case _ => NoNewWorkflowsToStart
-    }
-  }
-
-  /**
-    * Takes the workflow id and sends it over to the metadata service w/ default empty values for inputs/outputs
-    */
-  private def registerSubmissionWithMetadataService(id: WorkflowId, originalSourceFiles: WorkflowSourceFilesCollection): Unit = {
-    val sourceFiles = processSource(_.clearEncryptedValues)(originalSourceFiles).get
-
-    val submissionEvents = List(
-      MetadataEvent(MetadataKey(id, None, WorkflowMetadataKeys.SubmissionTime), MetadataValue(OffsetDateTime.now.toString)),
-      MetadataEvent.empty(MetadataKey(id, None, WorkflowMetadataKeys.Inputs)),
-      MetadataEvent.empty(MetadataKey(id, None, WorkflowMetadataKeys.Outputs)),
-
-      MetadataEvent(MetadataKey(id, None, WorkflowMetadataKeys.SubmissionSection, WorkflowMetadataKeys.SubmissionSection_Workflow), MetadataValue(sourceFiles.wdlSource)),
-      MetadataEvent(MetadataKey(id, None, WorkflowMetadataKeys.SubmissionSection, WorkflowMetadataKeys.SubmissionSection_Inputs), MetadataValue(sourceFiles.inputsJson)),
-      MetadataEvent(MetadataKey(id, None, WorkflowMetadataKeys.SubmissionSection, WorkflowMetadataKeys.SubmissionSection_Options), MetadataValue(sourceFiles.workflowOptionsJson))
-    )
-
-    serviceRegistryActor ! PutMetadataAction(submissionEvents)
+  override def receive = {
+    case cmd: WorkflowStoreActorSubmitCommand => workflowStoreSubmitActor forward cmd
+    case cmd: WorkflowStoreActorEngineCommand => workflowStoreEngineActor forward cmd
   }
 }
 
 object WorkflowStoreActor {
-
-  private[workflowstore] case class WorkflowStoreActorCommandWithSender(command: WorkflowStoreActorCommand, sender: ActorRef)
-
-  private[workflowstore] case class WorkflowStoreActorData(currentOperation: Option[WorkflowStoreActorCommandWithSender], pendingOperations: List[WorkflowStoreActorCommandWithSender]) {
-    def withCurrentCommand(command: WorkflowStoreActorCommand, sender: ActorRef) = this.copy(currentOperation = Option(WorkflowStoreActorCommandWithSender(command, sender)))
-    def withPendingCommand(newCommand: WorkflowStoreActorCommand, sender: ActorRef) = this.copy(pendingOperations = this.pendingOperations :+ WorkflowStoreActorCommandWithSender(newCommand, sender))
-    def pop = {
-      if (pendingOperations.isEmpty) { WorkflowStoreActorData(None, List.empty) }
-      else { WorkflowStoreActorData(Option(pendingOperations.head), pendingOperations.tail) }
-    }
-  }
-
-  private[workflowstore] sealed trait WorkflowStoreActorState
-  private[workflowstore] case object Unstarted extends WorkflowStoreActorState
-  private[workflowstore] case object Working extends WorkflowStoreActorState
-  private[workflowstore] case object Idle extends WorkflowStoreActorState
-
-  sealed trait WorkflowStoreActorCommand
-  final case class SubmitWorkflow(source: WorkflowSourceFilesCollection) extends WorkflowStoreActorCommand
-  final case class BatchSubmitWorkflows(sources: NonEmptyList[WorkflowSourceFilesCollection]) extends WorkflowStoreActorCommand
-  final case class FetchRunnableWorkflows(n: Int) extends WorkflowStoreActorCommand
-  final case class RemoveWorkflow(id: WorkflowId) extends WorkflowStoreActorCommand
-  final case class AbortWorkflow(id: WorkflowId, manager: ActorRef) extends WorkflowStoreActorCommand
-
-  private case object InitializerCommand extends WorkflowStoreActorCommand
-  private case object WorkDone
-
-  sealed trait WorkflowStoreActorResponse
-  final case class WorkflowSubmittedToStore(workflowId: WorkflowId) extends WorkflowStoreActorResponse
-  final case class WorkflowsBatchSubmittedToStore(workflowIds: NonEmptyList[WorkflowId]) extends WorkflowStoreActorResponse
-  case object NoNewWorkflowsToStart extends WorkflowStoreActorResponse
-  final case class NewWorkflowsToStart(workflows: NonEmptyList[WorkflowToStart]) extends WorkflowStoreActorResponse
-  final case class WorkflowAborted(workflowId: WorkflowId) extends WorkflowStoreActorResponse
-  final case class WorkflowAbortFailed(workflowId: WorkflowId, reason: Throwable) extends WorkflowStoreActorResponse
+  sealed trait WorkflowStoreActorEngineCommand
+  final case class FetchRunnableWorkflows(n: Int) extends WorkflowStoreActorEngineCommand
+  final case class RemoveWorkflow(id: WorkflowId) extends WorkflowStoreActorEngineCommand
+  final case class AbortWorkflow(id: WorkflowId, manager: ActorRef) extends WorkflowStoreActorEngineCommand
+  case object InitializerCommand extends WorkflowStoreActorEngineCommand
+  case object WorkDone extends WorkflowStoreActorEngineCommand
+
+  sealed trait WorkflowStoreActorSubmitCommand
+  final case class SubmitWorkflow(source: WorkflowSourceFilesCollection) extends WorkflowStoreActorSubmitCommand
+  final case class BatchSubmitWorkflows(sources: NonEmptyList[WorkflowSourceFilesCollection]) extends WorkflowStoreActorSubmitCommand
 
   def props(workflowStoreDatabase: WorkflowStore, serviceRegistryActor: ActorRef) = {
     Props(WorkflowStoreActor(workflowStoreDatabase, serviceRegistryActor)).withDispatcher(EngineDispatcher)
diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreEngineActor.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreEngineActor.scala
new file mode 100644
index 000000000..180138753
--- /dev/null
+++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreEngineActor.scala
@@ -0,0 +1,166 @@
+package cromwell.engine.workflow.workflowstore
+
+import akka.actor.{ActorLogging, ActorRef, LoggingFSM, Props}
+import cats.data.NonEmptyList
+import cromwell.core.Dispatcher._
+import cromwell.engine.workflow.WorkflowManagerActor
+import cromwell.engine.workflow.WorkflowManagerActor.WorkflowNotFoundException
+import cromwell.engine.workflow.workflowstore.WorkflowStoreEngineActor.WorkflowStoreActorState
+import cromwell.engine.workflow.workflowstore.WorkflowStoreState.StartableState
+import WorkflowStoreEngineActor._
+import cromwell.core.WorkflowId
+import cromwell.engine.workflow.workflowstore.WorkflowStoreActor._
+import cromwell.services.metadata.MetadataService.MetadataPutAcknowledgement
+import org.apache.commons.lang3.exception.ExceptionUtils
+
+import scala.concurrent.{ExecutionContext, Future}
+import scala.util.{Failure, Success}
+
+final case class WorkflowStoreEngineActor(store: WorkflowStore, serviceRegistryActor: ActorRef)
+  extends LoggingFSM[WorkflowStoreActorState, WorkflowStoreActorData] with ActorLogging {
+
+  implicit val ec: ExecutionContext = context.dispatcher
+
+  startWith(Unstarted, WorkflowStoreActorData(None, List.empty))
+  self ! InitializerCommand
+
+  when(Unstarted) {
+    case Event(InitializerCommand, _) =>
+      val work = store.initialize map { _ =>
+        log.debug("Workflow store initialization successful")
+      }
+      addWorkCompletionHooks(InitializerCommand, work)
+      goto(Working) using stateData.withCurrentCommand(InitializerCommand, sender)
+    case Event(x: WorkflowStoreActorEngineCommand, _) =>
+      stay using stateData.withPendingCommand(x, sender)
+  }
+
+  when(Idle) {
+    case Event(cmd: WorkflowStoreActorEngineCommand, _) =>
+      if (stateData.currentOperation.nonEmpty || stateData.pendingOperations.nonEmpty) {
+        log.error("Non-empty WorkflowStoreActorData when in Idle state: {}", stateData)
+      }
+      startNewWork(cmd, sender, stateData.withCurrentCommand(cmd, sender))
+  }
+
+  when(Working) {
+    case Event(WorkDone, data) =>
+      val newData = data.pop
+      newData.currentOperation match {
+        case None => goto(Idle) using newData
+        case Some(WorkflowStoreActorCommandWithSender(cmd, sndr)) => startNewWork(cmd, sndr, newData)
+      }
+    case Event(cmd: WorkflowStoreActorEngineCommand, data) => stay using data.withPendingCommand(cmd, sender)
+  }
+
+  whenUnhandled {
+    case Event(MetadataPutAcknowledgement(_), _) =>
+      stay // Ignored
+    case Event(msg, _) =>
+      log.warning("Unexpected message to WorkflowStoreActor in state {} with data {}: {}", stateName, stateData, msg)
+      stay
+  }
+
+  onTransition {
+    case fromState -> toState =>
+      log.debug("WorkflowStore moving from {} (using {}) to {} (using {})", fromState, stateData, toState, nextStateData)
+  }
+
+  private def startNewWork(command: WorkflowStoreActorEngineCommand, sndr: ActorRef, nextData: WorkflowStoreActorData) = {
+    val work: Future[Any] = command match {
+      case cmd @ FetchRunnableWorkflows(n) =>
+        newWorkflowMessage(n) map { nwm =>
+          nwm match {
+            case NewWorkflowsToStart(workflows) => log.info("{} new workflows fetched", workflows.toList.size)
+            case NoNewWorkflowsToStart => log.debug("No workflows fetched")
+            case _ => log.error("Unexpected response from newWorkflowMessage({}): {}", n, nwm)
+          }
+          sndr ! nwm
+        }
+      case cmd @ AbortWorkflow(id, manager) =>
+        store.remove(id) map { removed =>
+          if (removed) {
+            log.debug(s"Workflow $id aborted and removed from the workflow store.")
+            manager ! WorkflowManagerActor.AbortWorkflowCommand(id, sndr)
+          } else {
+            sndr ! WorkflowAbortFailed(id, new WorkflowNotFoundException(s"Couldn't abort $id because no workflow with that ID is in progress"))
+          }
+        }
+      case cmd @ RemoveWorkflow(id) =>
+        store.remove(id) map { removed =>
+          if (removed) {
+            log.debug("Workflow {} removed from store successfully.", id)
+          } else {
+            log.warning(s"Attempted to remove ID {} from the WorkflowStore but it didn't exist", id)
+          }
+        }
+      case oops =>
+        log.error("Unexpected type of start work command: {}", oops.getClass.getSimpleName)
+        Future.successful(self ! WorkDone)
+    }
+    addWorkCompletionHooks(command, work)
+    goto(Working) using nextData
+  }
+
+  private def addWorkCompletionHooks[A](command: WorkflowStoreActorEngineCommand, work: Future[A]) = {
+    work.onComplete {
+      case Success(_) =>
+        self ! WorkDone
+      case Failure(t) =>
+        log.error("Error occurred during {}: {} because {}", command.getClass.getSimpleName, t.toString, ExceptionUtils.getStackTrace(t))
+        self ! WorkDone
+    }
+  }
+
+  /**
+    * Fetches at most n workflows, and builds the correct response message based on if there were any workflows or not
+    */
+  private def newWorkflowMessage(maxWorkflows: Int): Future[WorkflowStoreEngineActorResponse] = {
+    def fetchRunnableWorkflowsIfNeeded(maxWorkflowsInner: Int, state: StartableState) = {
+      if (maxWorkflows > 0) {
+        store.fetchRunnableWorkflows(maxWorkflowsInner, state)
+      } else {
+        Future.successful(List.empty[WorkflowToStart])
+      }
+    }
+
+    val runnableWorkflows = for {
+      restartableWorkflows <- fetchRunnableWorkflowsIfNeeded(maxWorkflows, WorkflowStoreState.Restartable)
+      submittedWorkflows <- fetchRunnableWorkflowsIfNeeded(maxWorkflows - restartableWorkflows.size, WorkflowStoreState.Submitted)
+    } yield restartableWorkflows ++ submittedWorkflows
+
+    runnableWorkflows map {
+      case x :: xs => NewWorkflowsToStart(NonEmptyList.of(x, xs: _*))
+      case _ => NoNewWorkflowsToStart
+    }
+  }
+}
+
+object WorkflowStoreEngineActor {
+  def props(workflowStoreDatabase: WorkflowStore, serviceRegistryActor: ActorRef) = {
+    Props(WorkflowStoreEngineActor(workflowStoreDatabase, serviceRegistryActor)).withDispatcher(EngineDispatcher)
+  }
+
+  sealed trait WorkflowStoreEngineActorResponse
+  case object NoNewWorkflowsToStart extends WorkflowStoreEngineActorResponse
+  final case class NewWorkflowsToStart(workflows: NonEmptyList[WorkflowToStart]) extends WorkflowStoreEngineActorResponse
+  final case class WorkflowAborted(workflowId: WorkflowId) extends WorkflowStoreEngineActorResponse
+  final case class WorkflowAbortFailed(workflowId: WorkflowId, reason: Throwable) extends WorkflowStoreEngineActorResponse
+
+
+  final case class WorkflowStoreActorCommandWithSender(command: WorkflowStoreActorEngineCommand, sender: ActorRef)
+
+  final case class WorkflowStoreActorData(currentOperation: Option[WorkflowStoreActorCommandWithSender], pendingOperations: List[WorkflowStoreActorCommandWithSender]) {
+    def withCurrentCommand(command: WorkflowStoreActorEngineCommand, sender: ActorRef) = this.copy(currentOperation = Option(WorkflowStoreActorCommandWithSender(command, sender)))
+    def withPendingCommand(newCommand: WorkflowStoreActorEngineCommand, sender: ActorRef) = this.copy(pendingOperations = this.pendingOperations :+ WorkflowStoreActorCommandWithSender(newCommand, sender))
+    def pop = {
+      if (pendingOperations.isEmpty) { WorkflowStoreActorData(None, List.empty) }
+      else { WorkflowStoreActorData(Option(pendingOperations.head), pendingOperations.tail) }
+    }
+  }
+
+  sealed trait WorkflowStoreActorState
+  case object Unstarted extends WorkflowStoreActorState
+  case object Working extends WorkflowStoreActorState
+  case object Idle extends WorkflowStoreActorState
+}
diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreSubmitActor.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreSubmitActor.scala
new file mode 100644
index 000000000..b9381863e
--- /dev/null
+++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreSubmitActor.scala
@@ -0,0 +1,102 @@
+package cromwell.engine.workflow.workflowstore
+
+import java.time.OffsetDateTime
+
+import akka.actor.{Actor, ActorLogging, ActorRef, Props}
+import cats.data.NonEmptyList
+import cromwell.core.Dispatcher._
+import cromwell.core._
+import cromwell.engine.workflow.workflowstore.WorkflowStoreActor._
+import cromwell.engine.workflow.workflowstore.WorkflowStoreSubmitActor.{WorkflowSubmittedToStore, WorkflowsBatchSubmittedToStore}
+import cromwell.services.metadata.{MetadataEvent, MetadataKey, MetadataValue}
+import cromwell.services.metadata.MetadataService.PutMetadataAction
+
+import scala.concurrent.{ExecutionContext, Future}
+import scala.util.{Failure, Success}
+
+final case class WorkflowStoreSubmitActor(store: WorkflowStore, serviceRegistryActor: ActorRef) extends Actor with ActorLogging {
+  implicit val ec: ExecutionContext = context.dispatcher
+
+  override def receive = {
+    case cmd: SubmitWorkflow =>
+      val sndr = sender()
+      storeWorkflowSources(NonEmptyList.of(cmd.source)) foreach { ids =>
+        val id = ids.head
+        registerSubmissionWithMetadataService(id, cmd.source)
+        sndr ! WorkflowSubmittedToStore(id)
+        log.info("Workflow {} submitted.", id)
+      }
+    case cmd: BatchSubmitWorkflows =>
+      val sndr = sender()
+      storeWorkflowSources(cmd.sources) foreach { ids =>
+        val assignedSources = ids.toList.zip(cmd.sources.toList)
+        assignedSources foreach { case (id, sourceFiles) => registerSubmissionWithMetadataService(id, sourceFiles) }
+        sndr ! WorkflowsBatchSubmittedToStore(ids)
+        log.info("Workflows {} submitted.", ids.toList.mkString(", "))
+      }
+  }
+
+  private def storeWorkflowSources(sources: NonEmptyList[WorkflowSourceFilesCollection]): Future[NonEmptyList[WorkflowId]] = {
+    for {
+      processedSources <- processSources(sources, _.asPrettyJson)
+      workflowIds <- store.add(processedSources)
+    } yield workflowIds
+  }
+
+  private def processSources(sources: NonEmptyList[WorkflowSourceFilesCollection],
+                             processOptions: WorkflowOptions => WorkflowOptionsJson): Future[NonEmptyList[WorkflowSourceFilesCollection]] = {
+    val nelFutures: NonEmptyList[Future[WorkflowSourceFilesCollection]] = sources map processSource(processOptions)
+    val listFutures: List[Future[WorkflowSourceFilesCollection]] = nelFutures.toList
+    val futureLists: Future[List[WorkflowSourceFilesCollection]] = Future.sequence(listFutures)
+    futureLists.map(seq => NonEmptyList.fromList(seq).get)
+  }
+
+  /**
+    * Runs processing on workflow source files before they are stored.
+    *
+    * @param processOptions How to process the workflow options
+    * @param source         Original workflow source
+    * @return Attempted updated workflow source
+    */
+  private def processSource(processOptions: WorkflowOptions => WorkflowOptionsJson)
+                           (source: WorkflowSourceFilesCollection): Future[WorkflowSourceFilesCollection] = {
+    val options = Future {
+      WorkflowOptions.fromJsonString(source.workflowOptionsJson)
+    }.flatMap {
+      case Success(s) => Future.successful(s)
+      case Failure(regrets) => Future.failed(regrets)
+    }
+
+    options map {o => source.copyOptions(processOptions(o)) }
+  }
+
+  /**
+    * Takes the workflow id and sends it over to the metadata service w/ default empty values for inputs/outputs
+    */
+  private def registerSubmissionWithMetadataService(id: WorkflowId, originalSourceFiles: WorkflowSourceFilesCollection): Unit = {
+    processSource(_.clearEncryptedValues)(originalSourceFiles) foreach { sourceFiles =>
+      val submissionEvents = List(
+        MetadataEvent(MetadataKey(id, None, WorkflowMetadataKeys.SubmissionTime), MetadataValue(OffsetDateTime.now.toString)),
+        MetadataEvent.empty(MetadataKey(id, None, WorkflowMetadataKeys.Inputs)),
+        MetadataEvent.empty(MetadataKey(id, None, WorkflowMetadataKeys.Outputs)),
+
+        MetadataEvent(MetadataKey(id, None, WorkflowMetadataKeys.SubmissionSection, WorkflowMetadataKeys.SubmissionSection_Workflow), MetadataValue(sourceFiles.wdlSource)),
+        MetadataEvent(MetadataKey(id, None, WorkflowMetadataKeys.SubmissionSection, WorkflowMetadataKeys.SubmissionSection_Inputs), MetadataValue(sourceFiles.inputsJson)),
+        MetadataEvent(MetadataKey(id, None, WorkflowMetadataKeys.SubmissionSection, WorkflowMetadataKeys.SubmissionSection_Options), MetadataValue(sourceFiles.workflowOptionsJson))
+      )
+
+      serviceRegistryActor ! PutMetadataAction(submissionEvents)
+    }
+  }
+}
+
+object WorkflowStoreSubmitActor {
+  def props(workflowStoreDatabase: WorkflowStore, serviceRegistryActor: ActorRef) = {
+    Props(WorkflowStoreSubmitActor(workflowStoreDatabase, serviceRegistryActor)).withDispatcher(ApiDispatcher)
+  }
+
+  sealed trait WorkflowStoreSubmitActorResponse
+  final case class WorkflowSubmittedToStore(workflowId: WorkflowId) extends WorkflowStoreSubmitActorResponse
+  final case class WorkflowsBatchSubmittedToStore(workflowIds: NonEmptyList[WorkflowId]) extends WorkflowStoreSubmitActorResponse
+}
+
diff --git a/engine/src/main/scala/cromwell/webservice/CromwellApiHandler.scala b/engine/src/main/scala/cromwell/webservice/CromwellApiHandler.scala
index 7fb9e87e2..c1e92690d 100644
--- a/engine/src/main/scala/cromwell/webservice/CromwellApiHandler.scala
+++ b/engine/src/main/scala/cromwell/webservice/CromwellApiHandler.scala
@@ -8,7 +8,7 @@ import cromwell.core._
 import cromwell.core.Dispatcher.ApiDispatcher
 import cromwell.engine.workflow.WorkflowManagerActor
 import cromwell.engine.workflow.WorkflowManagerActor.WorkflowNotFoundException
-import cromwell.engine.workflow.workflowstore.WorkflowStoreActor
+import cromwell.engine.workflow.workflowstore.{WorkflowStoreActor, WorkflowStoreEngineActor, WorkflowStoreSubmitActor}
 import cromwell.webservice.PerRequest.RequestComplete
 import cromwell.webservice.metadata.WorkflowQueryPagination
 import spray.http.{StatusCodes, Uri}
@@ -53,9 +53,9 @@ class CromwellApiHandler(requestHandlerActor: ActorRef) extends Actor with Workf
     case ApiHandlerEngineStats => requestHandlerActor ! WorkflowManagerActor.EngineStatsCommand
     case stats: EngineStatsActor.EngineStats => context.parent ! RequestComplete((StatusCodes.OK, stats))
     case ApiHandlerWorkflowAbort(id, manager) => requestHandlerActor ! WorkflowStoreActor.AbortWorkflow(id, manager)
-    case WorkflowStoreActor.WorkflowAborted(id) =>
+    case WorkflowStoreEngineActor.WorkflowAborted(id) =>
       context.parent ! RequestComplete((StatusCodes.OK, WorkflowAbortResponse(id.toString, WorkflowAborted.toString)))
-    case WorkflowStoreActor.WorkflowAbortFailed(_, e) =>
+    case WorkflowStoreEngineActor.WorkflowAbortFailed(_, e) =>
       error(e) {
         case _: IllegalStateException => RequestComplete((StatusCodes.Forbidden, APIResponse.error(e)))
         case _: WorkflowNotFoundException => RequestComplete((StatusCodes.NotFound, APIResponse.error(e)))
@@ -64,14 +64,14 @@ class CromwellApiHandler(requestHandlerActor: ActorRef) extends Actor with Workf
 
     case ApiHandlerWorkflowSubmit(source) => requestHandlerActor ! WorkflowStoreActor.SubmitWorkflow(source)
 
-    case WorkflowStoreActor.WorkflowSubmittedToStore(id) =>
+    case WorkflowStoreSubmitActor.WorkflowSubmittedToStore(id) =>
       context.parent ! RequestComplete((StatusCodes.Created, WorkflowSubmitResponse(id.toString, WorkflowSubmitted.toString)))
 
     case ApiHandlerWorkflowSubmitBatch(sources) => requestHandlerActor !
       WorkflowStoreActor.BatchSubmitWorkflows(sources.map(x => WorkflowSourceFilesCollection(x.wdlSource, x.inputsJson, x.workflowOptionsJson, x.labelsJson, x.importsZipFileOption)))
 
 
-    case WorkflowStoreActor.WorkflowsBatchSubmittedToStore(ids) =>
+    case WorkflowStoreSubmitActor.WorkflowsBatchSubmittedToStore(ids) =>
       val responses = ids map { id => WorkflowSubmitResponse(id.toString, WorkflowSubmitted.toString) }
       context.parent ! RequestComplete((StatusCodes.OK, responses.toList))
   }
diff --git a/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala b/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala
index b11521434..8f690a80c 100644
--- a/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala
+++ b/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala
@@ -25,9 +25,9 @@ trait SwaggerService extends SwaggerUiResourceHttpService {
 }
 
 trait CromwellApiService extends HttpService with PerRequestCreator {
-  val workflowManagerActor: ActorRef
-  val workflowStoreActor: ActorRef
-  val serviceRegistryActor: ActorRef
+  def workflowManagerActor: ActorRef
+  def workflowStoreActor: ActorRef
+  def serviceRegistryActor: ActorRef
 
   def toMap(someInput: Option[String]): Map[String, JsValue] = {
     import spray.json._
@@ -53,14 +53,19 @@ trait CromwellApiService extends HttpService with PerRequestCreator {
       perRequest(requestContext, metadataBuilderProps, message)
   }
 
-  private def failBadRequest(t: Throwable, statusCode: StatusCode = StatusCodes.BadRequest) = respondWithMediaType(`application/json`) {
+  def handleQueryMetadataRequest(parameters: Seq[(String, String)]): Route = {
+    requestContext =>
+      perRequest(requestContext, metadataBuilderProps, WorkflowQuery(requestContext.request.uri, parameters))
+  }
+
+  protected def failBadRequest(t: Throwable, statusCode: StatusCode = StatusCodes.BadRequest) = respondWithMediaType(`application/json`) {
     complete((statusCode, APIResponse.fail(t).toJson.prettyPrint))
   }
 
   val workflowRoutes = queryRoute ~ queryPostRoute ~ workflowOutputsRoute ~ submitRoute ~ submitBatchRoute ~
     workflowLogsRoute ~ abortRoute ~ metadataRoute ~ timingRoute ~ statusRoute ~ backendRoute ~ statsRoute ~ versionRoute
 
-  private def withRecognizedWorkflowId(possibleWorkflowId: String)(recognizedWorkflowId: WorkflowId => Route): Route = {
+  protected def withRecognizedWorkflowId(possibleWorkflowId: String)(recognizedWorkflowId: WorkflowId => Route): Route = {
     def callback(requestContext: RequestContext) = new ValidationCallback {
       // The submitted value is malformed as a UUID and therefore not possibly recognized.
       override def onMalformed(possibleWorkflowId: String): Unit = {
@@ -102,8 +107,7 @@ trait CromwellApiService extends HttpService with PerRequestCreator {
     path("workflows" / Segment / "query") { version =>
       parameterSeq { parameters =>
         get {
-          requestContext =>
-            perRequest(requestContext, metadataBuilderProps, WorkflowQuery(requestContext.request.uri, parameters))
+          handleQueryMetadataRequest(parameters)
         }
       }
     }
@@ -112,8 +116,7 @@ trait CromwellApiService extends HttpService with PerRequestCreator {
     path("workflows" / Segment / "query") { version =>
       entity(as[Seq[Map[String, String]]]) { parameterMap =>
         post {
-          requestContext =>
-            perRequest(requestContext, metadataBuilderProps, WorkflowQuery(requestContext.request.uri, parameterMap.flatMap(_.toSeq)))
+          handleQueryMetadataRequest(parameterMap.flatMap(_.toSeq))
         }
       }
     }
diff --git a/engine/src/test/scala/cromwell/CromwellTestKitSpec.scala b/engine/src/test/scala/cromwell/CromwellTestKitSpec.scala
index 845f0b9c5..c8c526b68 100644
--- a/engine/src/test/scala/cromwell/CromwellTestKitSpec.scala
+++ b/engine/src/test/scala/cromwell/CromwellTestKitSpec.scala
@@ -15,7 +15,7 @@ import cromwell.engine.backend.BackendConfigurationEntry
 import cromwell.engine.workflow.WorkflowManagerActor.RetrieveNewWorkflows
 import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheReadActor.{CacheLookupRequest, CacheResultMatchesForHashes}
 import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.CallCacheHashes
-import cromwell.engine.workflow.workflowstore.WorkflowStoreActor.WorkflowSubmittedToStore
+import cromwell.engine.workflow.workflowstore.WorkflowStoreSubmitActor.WorkflowSubmittedToStore
 import cromwell.engine.workflow.workflowstore.{InMemoryWorkflowStore, WorkflowStoreActor}
 import cromwell.jobstore.JobStoreActor.{JobStoreWriteSuccess, JobStoreWriterCommand}
 import cromwell.server.{CromwellRootActor, CromwellSystem}
diff --git a/engine/src/test/scala/cromwell/engine/WorkflowStoreActorSpec.scala b/engine/src/test/scala/cromwell/engine/WorkflowStoreActorSpec.scala
index d6719fe85..e59c3991d 100644
--- a/engine/src/test/scala/cromwell/engine/WorkflowStoreActorSpec.scala
+++ b/engine/src/test/scala/cromwell/engine/WorkflowStoreActorSpec.scala
@@ -4,6 +4,8 @@ import cats.data.NonEmptyList
 import cromwell.CromwellTestKitSpec
 import cromwell.core.{WorkflowId, WorkflowSourceFilesCollection}
 import cromwell.engine.workflow.workflowstore.WorkflowStoreActor._
+import cromwell.engine.workflow.workflowstore.WorkflowStoreEngineActor.{NewWorkflowsToStart, NoNewWorkflowsToStart}
+import cromwell.engine.workflow.workflowstore.WorkflowStoreSubmitActor.{WorkflowSubmittedToStore, WorkflowsBatchSubmittedToStore}
 import cromwell.engine.workflow.workflowstore._
 import cromwell.services.metadata.MetadataQuery
 import cromwell.services.metadata.MetadataService.{GetMetadataQueryAction, MetadataLookupResponse}
diff --git a/engine/src/test/scala/cromwell/subworkflowstore/SubWorkflowStoreSpec.scala b/engine/src/test/scala/cromwell/subworkflowstore/SubWorkflowStoreSpec.scala
index ed32f43c5..5101cee46 100644
--- a/engine/src/test/scala/cromwell/subworkflowstore/SubWorkflowStoreSpec.scala
+++ b/engine/src/test/scala/cromwell/subworkflowstore/SubWorkflowStoreSpec.scala
@@ -13,7 +13,8 @@ import scala.concurrent.duration._
 import SubWorkflowStoreSpec._
 import akka.testkit.TestProbe
 import cromwell.database.sql.tables.SubWorkflowStoreEntry
-import cromwell.engine.workflow.workflowstore.WorkflowStoreActor.{SubmitWorkflow, WorkflowSubmittedToStore}
+import cromwell.engine.workflow.workflowstore.WorkflowStoreActor.SubmitWorkflow
+import cromwell.engine.workflow.workflowstore.WorkflowStoreSubmitActor.WorkflowSubmittedToStore
 import cromwell.engine.workflow.workflowstore.{SqlWorkflowStore, WorkflowStoreActor}
 
 import scala.language.postfixOps
diff --git a/engine/src/test/scala/cromwell/webservice/CromwellApiServiceSpec.scala b/engine/src/test/scala/cromwell/webservice/CromwellApiServiceSpec.scala
index 18f985af6..b360b8e09 100644
--- a/engine/src/test/scala/cromwell/webservice/CromwellApiServiceSpec.scala
+++ b/engine/src/test/scala/cromwell/webservice/CromwellApiServiceSpec.scala
@@ -1,121 +1,77 @@
 package cromwell.webservice
 
-import java.time.OffsetDateTime
-import java.util.UUID
-
-import akka.actor.{Actor, Props}
-import akka.pattern.ask
-import akka.util.Timeout
-import com.typesafe.config.ConfigFactory
-import cromwell.CromwellTestKitSpec
-import cromwell.core._
-import cromwell.engine.workflow.workflowstore.WorkflowStoreActor
-import cromwell.engine.workflow.workflowstore.WorkflowStoreActor.{WorkflowAborted => _, _}
-import cromwell.server.{CromwellServerActor, CromwellSystem}
+import akka.actor.{Actor, ActorSystem, Props}
+import cromwell.core.{WorkflowId, WorkflowMetadataKeys, WorkflowSubmitted, WorkflowSucceeded}
+
+import scala.util.{Failure, Success, Try}
+import cromwell.engine.workflow.workflowstore.WorkflowStoreActor.{AbortWorkflow, BatchSubmitWorkflows, SubmitWorkflow}
+import cromwell.engine.workflow.workflowstore.WorkflowStoreEngineActor
+import cromwell.engine.workflow.workflowstore.WorkflowStoreEngineActor.WorkflowAbortFailed
 import cromwell.services.metadata.MetadataService._
-import cromwell.services.metadata._
-import cromwell.services.metadata.impl.MetadataSummaryRefreshActor.MetadataSummarySuccess
-import cromwell.util.SampleWdl.HelloWorld
-import org.scalatest.concurrent.{PatienceConfiguration, ScalaFutures}
 import org.scalatest.{FlatSpec, Matchers}
-import org.specs2.mock.Mockito
-import spray.http.{DateTime => _, _}
+import spray.http._
 import spray.json.DefaultJsonProtocol._
-import spray.json._
+import spray.json.{JsString, _}
+import spray.routing._
+import cromwell.engine.workflow.workflowstore.WorkflowStoreSubmitActor.{WorkflowSubmittedToStore, WorkflowsBatchSubmittedToStore}
+import cromwell.util.SampleWdl.HelloWorld
+import spray.httpx.SprayJsonSupport._
 import spray.testkit.ScalatestRouteTest
 
-object MockWorkflowStoreActor {
-  val submittedWorkflowId = WorkflowId(UUID.randomUUID())
-  val runningWorkflowId = WorkflowId.randomId()
-  val unknownId = WorkflowId.randomId()
-  val submittedScatterWorkflowId = WorkflowId.randomId()
-  val abortedWorkflowId = WorkflowId.randomId()
-}
-
-class MockWorkflowStoreActor extends Actor {
-  import MockWorkflowStoreActor.submittedWorkflowId
-
-  override def receive = {
-    case SubmitWorkflow(source) => sender ! WorkflowSubmittedToStore(submittedWorkflowId)
-    case BatchSubmitWorkflows(sources) =>
-      val response = WorkflowsBatchSubmittedToStore(sources map { _ => submittedWorkflowId })
-      sender ! response
-    case AbortWorkflow(id, manager) =>
-      val message = id match {
-        case MockWorkflowStoreActor.runningWorkflowId =>
-          WorkflowStoreActor.WorkflowAborted(id)
-        case MockWorkflowStoreActor.abortedWorkflowId =>
-          WorkflowAbortFailed(id, new IllegalStateException(s"Workflow ID '$id' is in terminal state 'Aborted' and cannot be aborted."))
-      }
-      sender ! message
-  }
-}
-
-class CromwellApiServiceSpec extends FlatSpec with CromwellApiService with ScalatestRouteTest with Matchers
-  with ScalaFutures with Mockito {
-  import spray.httpx.SprayJsonSupport._
-
-  // BUG: Must be called once to statically initialize the backends, otherwise this Spec won't run if run alone.
-  new CromwellSystem {}
-
-  import akka.testkit._
-
-  import scala.concurrent.duration._
-
-  // The submit route takes a bit longer than the default 1 s while things initialize, when this spec is run by itself
-  implicit val defaultTimeout = RouteTestTimeout(30.seconds.dilated)
-
-  override def actorRefFactory = system
-  override val serviceRegistryActor = CromwellTestKitSpec.ServiceRegistryActorInstance
-
-  override val workflowStoreActor = actorRefFactory.actorOf(Props(new MockWorkflowStoreActor()))
-  override val workflowManagerActor = actorRefFactory.actorOf(Props.empty)
+class CromwellApiServiceSpec extends FlatSpec with ScalatestRouteTest with Matchers {
+  import CromwellApiServiceSpec._
 
+  val cromwellApiService = new MockApiService()
   val version = "v1"
 
-  def publishMetadata(events: Seq[MetadataEvent]): Unit = {
-    val timeout: Timeout = 5.seconds.dilated
-
-    import akka.pattern.ask
-    val putResult = serviceRegistryActor.ask(PutMetadataAction(events))(timeout)
-    putResult.futureValue(PatienceConfiguration.Timeout(timeout.duration)) shouldBe a[MetadataPutAcknowledgement]
-    ()
+  behavior of "REST API /status endpoint"
+  it should "return 200 for get of a known workflow id" in {
+    val workflowId = MockApiService.ExistingWorkflowId
+    Get(s"/workflows/$version/$workflowId/status") ~>
+      cromwellApiService.statusRoute ~>
+      check {
+          status should be(StatusCodes.OK)
+          val result = responseAs[JsObject]
+          result.fields(WorkflowMetadataKeys.Status) should be(JsString("Submitted"))
+      }
   }
 
-  def forceSummary(): Unit = {
-    val timeout: Timeout = 5.seconds.dilated
-    val summaryResult = serviceRegistryActor.ask(RefreshSummary)(timeout)
+    it should "return 404 for get of unknown workflow" in {
+      val workflowId = MockApiService.UnrecognizedWorkflowId
 
-    val askResult = summaryResult.futureValue(PatienceConfiguration.Timeout(timeout.duration))
-    askResult match {
-      case MetadataSummarySuccess =>
-      case _ =>
-        fail()
+      Get(s"/workflows/$version/$workflowId/status") ~>
+        cromwellApiService.statusRoute ~>
+        check {
+          assertResult(StatusCodes.NotFound) {
+            status
+          }
+        }
     }
 
-  }
-
-  behavior of "REST API /status endpoint"
-
-  it should "return 500 errors as Json" in {
-    val apiActor = TestActorRef(new CromwellServerActor(ConfigFactory.empty()))
-    val probe = TestProbe()
-    probe.send(apiActor, Timedout(mock[HttpRequest]))
-    probe.expectMsgPF(defaultTimeout.duration) {
-      case response: HttpResponse =>
-        response.entity.toOption shouldBe defined
-        response.entity.toOption.get.contentType.toString() shouldBe ContentTypes.`application/json`.mediaType.value.toString
+     it should "return 400 for get of a malformed workflow id's status" in {
+      Get(s"/workflows/$version/foobar/status") ~>
+        cromwellApiService.statusRoute ~>
+        check {
+          assertResult(StatusCodes.BadRequest) {
+            status
+          }
+          assertResult(
+            """{
+               |  "status": "fail",
+               |  "message": "Invalid workflow ID: 'foobar'."
+               |}""".stripMargin
+          ) {
+            responseAs[String]
+          }
+        }
     }
 
-    system.stop(apiActor)
-    system.stop(probe.ref)
-  }
-
-  it should "return 404 for get of unknown workflow" in {
-    val workflowId = WorkflowId.randomId()
+  behavior of "REST API /abort endpoint"
+  it should "return 404 for abort of unknown workflow" in {
+    val workflowId = MockApiService.UnrecognizedWorkflowId
 
-    Get(s"/workflows/$version/$workflowId/status") ~>
-      statusRoute ~>
+    Post(s"/workflows/$version/$workflowId/abort") ~>
+      cromwellApiService.abortRoute ~>
       check {
         assertResult(StatusCodes.NotFound) {
           status
@@ -123,132 +79,69 @@ class CromwellApiServiceSpec extends FlatSpec with CromwellApiService with Scala
       }
   }
 
-   it should "return 400 for get of a malformed workflow id's status" in {
-    Get(s"/workflows/$version/foobar/status") ~>
-      statusRoute ~>
+  it should "return 400 for abort of a malformed workflow id" in {
+    Post(s"/workflows/$version/foobar/abort") ~>
+      cromwellApiService.abortRoute ~>
       check {
         assertResult(StatusCodes.BadRequest) {
           status
         }
         assertResult(
           """{
-             |  "status": "fail",
-             |  "message": "Invalid workflow ID: 'foobar'."
-             |}""".stripMargin
+            |  "status": "fail",
+            |  "message": "Invalid workflow ID: 'foobar'."
+            |}""".stripMargin
         ) {
           responseAs[String]
         }
       }
   }
 
-  private def publishStatusAndSubmission(workflowId: WorkflowId, state: WorkflowState): Unit = {
-    val events = Seq(
-      MetadataEvent(MetadataKey(workflowId, None, WorkflowMetadataKeys.SubmissionTime), MetadataValue(OffsetDateTime.now())),
-      MetadataEvent(MetadataKey(workflowId, None, WorkflowMetadataKeys.Status), MetadataValue(state))
-    )
-    publishMetadata(events)
-  }
-
-  it should "return 200 for get of a known workflow id" in {
-    val workflowId = WorkflowId.randomId()
-    publishStatusAndSubmission(workflowId, WorkflowSubmitted)
-
-    Get(s"/workflows/$version/$workflowId/status") ~>
-      statusRoute ~>
+    it should "return 403 for abort of a workflow in a terminal state" in {
+      Post(s"/workflows/$version/${MockApiService.AbortedWorkflowId}/abort") ~>
+      cromwellApiService.abortRoute ~>
       check {
-        status should be(StatusCodes.OK)
-        val result = responseAs[JsObject]
-        result.fields(WorkflowMetadataKeys.Status) should be(JsString("Submitted"))
-      }
-  }
-
-  behavior of "REST API /abort endpoint"
-
-  it should "return 404 for abort of unknown workflow" in {
-    Post(s"/workflows/$version/${MockWorkflowStoreActor.unknownId}/abort") ~>
-      abortRoute ~>
-      check {
-        assertResult(StatusCodes.NotFound) {
+        assertResult(StatusCodes.Forbidden) {
           status
         }
         assertResult(
           s"""{
-              |  "status": "fail",
-              |  "message": "Unrecognized workflow ID: ${MockWorkflowStoreActor.unknownId}"
-              |}""".stripMargin
-        ) {
-          responseAs[String]
-        }
-      }
-  }
-
-  it should "return 400 for abort of a malformed workflow id" in {
-    Post(s"/workflows/$version/foobar/abort") ~>
-      abortRoute ~>
-      check {
-        assertResult(StatusCodes.BadRequest) {
-          status
-        }
-        assertResult(
-          """{
-              |  "status": "fail",
-              |  "message": "Invalid workflow ID: 'foobar'."
+              |  "status": "error",
+              |  "message": "Workflow ID '${MockApiService.AbortedWorkflowId}' is in terminal state 'Aborted' and cannot be aborted."
               |}""".stripMargin
         ) {
           responseAs[String]
         }
       }
-  }
-
-  it should "return 403 for abort of a workflow in a terminal state" in {
-    publishStatusAndSubmission(MockWorkflowStoreActor.abortedWorkflowId, WorkflowAborted)
-
-    Post(s"/workflows/$version/${MockWorkflowStoreActor.abortedWorkflowId}/abort") ~>
-    abortRoute ~>
-    check {
-      assertResult(StatusCodes.Forbidden) {
-        status
-      }
-      assertResult(
-        s"""{
-            |  "status": "error",
-            |  "message": "Workflow ID '${MockWorkflowStoreActor.abortedWorkflowId}' is in terminal state 'Aborted' and cannot be aborted."
-            |}""".stripMargin
-      ) {
-        responseAs[String]
-      }
     }
-  }
-
-  it should "return 200 for abort of a known workflow id" in {
-    publishStatusAndSubmission(MockWorkflowStoreActor.runningWorkflowId, WorkflowRunning)
 
-    Post(s"/workflows/$version/${MockWorkflowStoreActor.runningWorkflowId}/abort") ~>
-      abortRoute ~>
-      check {
-        assertResult(
-          s"""{
-              |  "id": "${MockWorkflowStoreActor.runningWorkflowId.toString}",
-              |  "status": "Aborted"
-              |}"""
-            .stripMargin) {
-          responseAs[String]
-        }
-        assertResult(StatusCodes.OK) {
-          status
+    it should "return 200 for abort of a known workflow id" in {
+      Post(s"/workflows/$version/${MockApiService.ExistingWorkflowId}/abort") ~>
+        cromwellApiService.abortRoute ~>
+        check {
+          assertResult(
+            s"""{
+                |  "id": "${MockApiService.ExistingWorkflowId.toString}",
+                |  "status": "Aborted"
+                |}"""
+              .stripMargin) {
+            responseAs[String]
+          }
+          assertResult(StatusCodes.OK) {
+            status
+          }
         }
-      }
-  }
+    }
 
   behavior of "REST API submission endpoint"
   it should "return 201 for a successful workflow submission " in {
     val bodyParts: Map[String, BodyPart] = Map("wdlSource" -> BodyPart(HelloWorld.wdlSource()), "workflowInputs" -> BodyPart(HelloWorld.rawInputs.toJson.toString()))
     Post(s"/workflows/$version", MultipartFormData(bodyParts)) ~>
-      submitRoute ~>
+      cromwellApiService.submitRoute ~>
       check {
         assertResult(
           s"""{
-              |  "id": "${MockWorkflowStoreActor.submittedWorkflowId.toString}",
+              |  "id": "${MockApiService.ExistingWorkflowId.toString}",
               |  "status": "Submitted"
               |}""".stripMargin) {
           responseAs[String]
@@ -258,10 +151,11 @@ class CromwellApiServiceSpec extends FlatSpec with CromwellApiService with Scala
         }
       }
   }
+
   it should "return 400 for an unrecognized form data request parameter " in {
     val bodyParts: Map[String, BodyPart] = Map("incorrectParameter" -> BodyPart(HelloWorld.wdlSource()))
     Post(s"/workflows/$version", MultipartFormData(bodyParts)) ~>
-      submitRoute ~>
+      cromwellApiService.submitRoute ~>
       check {
         assertResult(
           s"""{
@@ -275,12 +169,13 @@ class CromwellApiServiceSpec extends FlatSpec with CromwellApiService with Scala
         }
       }
   }
+
   it should "succesfully merge and override multiple input files" in {
 
     val input1 = Map("wf.a1" -> "hello", "wf.a2" -> "world").toJson.toString
     val input2 = Map.empty[String, String].toJson.toString
     val overrideInput1 = Map("wf.a2" -> "universe").toJson.toString
-    val allInputs = mergeMaps(Seq(Option(input1), Option(input2), Option(overrideInput1)))
+    val allInputs = cromwellApiService.mergeMaps(Seq(Option(input1), Option(input2), Option(overrideInput1)))
 
     check {
       allInputs.fields.keys should contain allOf("wf.a1", "wf.a2")
@@ -294,14 +189,14 @@ class CromwellApiServiceSpec extends FlatSpec with CromwellApiService with Scala
     val bodyParts = Map("wdlSource" -> BodyPart(HelloWorld.wdlSource()), "workflowInputs" -> BodyPart(s"[$inputs, $inputs]"))
 
     Post(s"/workflows/$version/batch", MultipartFormData(bodyParts)) ~>
-      submitBatchRoute ~>
+      cromwellApiService.submitBatchRoute ~>
       check {
         assertResult(
           s"""[{
-              |  "id": "${MockWorkflowStoreActor.submittedWorkflowId.toString}",
+              |  "id": "${MockApiService.ExistingWorkflowId.toString}",
               |  "status": "Submitted"
               |}, {
-              |  "id": "${MockWorkflowStoreActor.submittedWorkflowId.toString}",
+              |  "id": "${MockApiService.ExistingWorkflowId.toString}",
               |  "status": "Submitted"
               |}]""".stripMargin) {
           responseAs[String]
@@ -316,7 +211,7 @@ class CromwellApiServiceSpec extends FlatSpec with CromwellApiService with Scala
     val bodyParts = Map("wdlSource" -> BodyPart(HelloWorld.wdlSource()))
 
     Post(s"/workflows/$version/batch", MultipartFormData(bodyParts)) ~>
-      submitBatchRoute ~>
+      cromwellApiService.submitBatchRoute ~>
       check {
         assertResult(
           s"""{
@@ -331,21 +226,10 @@ class CromwellApiServiceSpec extends FlatSpec with CromwellApiService with Scala
       }
   }
 
-  // TODO: Test tha batch submission returns expected workflow ids in order
-  // TODO: Also (assuming we still validate on submit) test a batch of mixed inputs that return submitted and failed
-
   behavior of "REST API /outputs endpoint"
-
   it should "return 200 with GET of outputs on successful execution of workflow" in {
-    val workflowId = WorkflowId.randomId()
-    val events = Seq(
-      MetadataEvent(MetadataKey(workflowId, None, s"${WorkflowMetadataKeys.Outputs}:myfirst"), MetadataValue("myOutput"))
-    )
-
-    publishMetadata(events)
-
-    Get(s"/workflows/$version/$workflowId/outputs") ~>
-      workflowOutputsRoute ~>
+    Get(s"/workflows/$version/${MockApiService.ExistingWorkflowId}/outputs") ~>
+      cromwellApiService.workflowOutputsRoute ~>
       check {
         status should be(StatusCodes.OK)
         val result = responseAs[JsObject]
@@ -354,8 +238,8 @@ class CromwellApiServiceSpec extends FlatSpec with CromwellApiService with Scala
   }
 
   it should "return 404 with outputs on unknown workflow" in {
-    Get(s"/workflows/$version/${MockWorkflowStoreActor.unknownId}/outputs") ~>
-    workflowOutputsRoute ~>
+    Get(s"/workflows/$version/${MockApiService.UnrecognizedWorkflowId}/outputs") ~>
+    cromwellApiService.workflowOutputsRoute ~>
     check {
       assertResult(StatusCodes.NotFound) {
         status
@@ -364,8 +248,8 @@ class CromwellApiServiceSpec extends FlatSpec with CromwellApiService with Scala
   }
 
   it should "return 405 with POST of outputs on successful execution of workflow" in {
-    Post(s"/workflows/$version/${MockWorkflowStoreActor.submittedWorkflowId.toString}/outputs") ~>
-      sealRoute(workflowOutputsRoute) ~>
+    Post(s"/workflows/$version/${MockApiService.UnrecognizedWorkflowId}/outputs") ~>
+      cromwellApiService.sealRoute(cromwellApiService.workflowOutputsRoute) ~>
       check {
         assertResult(StatusCodes.MethodNotAllowed) {
           status
@@ -374,21 +258,9 @@ class CromwellApiServiceSpec extends FlatSpec with CromwellApiService with Scala
   }
 
   behavior of "REST API /logs endpoint"
-
   it should "return 200 with paths to stdout/stderr/backend log" in {
-
-    val workflowId = WorkflowId.randomId()
-    val jobKey = Option(MetadataJobKey("mycall", None, 1))
-    val events = Seq(
-      MetadataEvent(MetadataKey(workflowId, jobKey, CallMetadataKeys.Stdout), MetadataValue("stdout.txt")),
-      MetadataEvent(MetadataKey(workflowId, jobKey, CallMetadataKeys.Stderr), MetadataValue("stderr.txt")),
-      MetadataEvent(MetadataKey(workflowId, jobKey, s"${CallMetadataKeys.BackendLogsPrefix}:log"), MetadataValue("backend.log"))
-    )
-
-    publishMetadata(events)
-
-    Get(s"/workflows/$version/$workflowId/logs") ~>
-      workflowLogsRoute ~>
+    Get(s"/workflows/$version/${MockApiService.ExistingWorkflowId}/logs") ~>
+      cromwellApiService.workflowLogsRoute ~>
       check {
         status should be(StatusCodes.OK)
         val result = responseAs[JsObject]
@@ -402,8 +274,8 @@ class CromwellApiServiceSpec extends FlatSpec with CromwellApiService with Scala
   }
 
   it should "return 404 with logs on unknown workflow" in {
-    Get(s"/workflows/$version/${MockWorkflowStoreActor.unknownId}/logs") ~>
-      workflowLogsRoute ~>
+    Get(s"/workflows/$version/${MockApiService.UnrecognizedWorkflowId}/logs") ~>
+      cromwellApiService.workflowLogsRoute ~>
       check {
         assertResult(StatusCodes.NotFound) {
           status
@@ -411,20 +283,10 @@ class CromwellApiServiceSpec extends FlatSpec with CromwellApiService with Scala
       }
   }
 
-
   behavior of "REST API /metadata endpoint"
-
   it should "return with full metadata from the metadata route" in {
-    val workflowId = WorkflowId.randomId()
-    val events = Seq(
-      MetadataEvent(MetadataKey(workflowId, None, "testKey1"), MetadataValue("myValue1")),
-      MetadataEvent(MetadataKey(workflowId, None, "testKey2"), MetadataValue("myValue2"))
-    )
-
-    publishMetadata(events)
-
-    Get(s"/workflows/$version/$workflowId/metadata") ~>
-      metadataRoute ~>
+    Get(s"/workflows/$version/${MockApiService.ExistingWorkflowId}/metadata") ~>
+      cromwellApiService.metadataRoute ~>
       check {
         status should be(StatusCodes.OK)
         val result = responseAs[JsObject]
@@ -436,19 +298,8 @@ class CromwellApiServiceSpec extends FlatSpec with CromwellApiService with Scala
   }
 
   it should "return with included metadata from the metadata route" in {
-    val workflowId = WorkflowId.randomId()
-
-    val events = Seq(
-      MetadataEvent(MetadataKey(workflowId, None, "testKey1a"), MetadataValue("myValue1a")),
-      MetadataEvent(MetadataKey(workflowId, None, "testKey1b"), MetadataValue("myValue1b")),
-      MetadataEvent(MetadataKey(workflowId, None, "testKey2a"), MetadataValue("myValue2a")),
-      MetadataEvent(MetadataKey(workflowId, None, "testKey2b"), MetadataValue("myValue2b"))
-    )
-
-    publishMetadata(events)
-
-    Get(s"/workflows/$version/$workflowId/metadata?includeKey=testKey1&includeKey=testKey2a") ~>
-      metadataRoute ~>
+    Get(s"/workflows/$version/${MockApiService.ExistingWorkflowId}/metadata?includeKey=testKey1&includeKey=testKey2a") ~>
+      cromwellApiService.metadataRoute ~>
       check {
         status should be(StatusCodes.OK)
         val result = responseAs[JsObject]
@@ -460,37 +311,23 @@ class CromwellApiServiceSpec extends FlatSpec with CromwellApiService with Scala
       }
   }
 
-  it should "return with excluded metadata from the metadata route" in {
-    val workflowId = WorkflowId.randomId()
-
-    val events = Seq(
-      MetadataEvent(MetadataKey(workflowId, None, "testKey1a"), MetadataValue("myValue1a")),
-      MetadataEvent(MetadataKey(workflowId, None, "testKey1b"), MetadataValue("myValue1b")),
-      MetadataEvent(MetadataKey(workflowId, None, "testKey2a"), MetadataValue("myValue2a")),
-      MetadataEvent(MetadataKey(workflowId, None, "testKey2b"), MetadataValue("myValue2b"))
-    )
-
-    publishMetadata(events)
-
-    Get(s"/workflows/$version/$workflowId/metadata?excludeKey=testKey2b&excludeKey=testKey3") ~>
-      metadataRoute ~>
-      check {
-        status should be(StatusCodes.OK)
-        val result = responseAs[JsObject]
-        result.fields.keys should contain allOf("testKey1a", "testKey1b", "testKey2a")
-        result.fields.keys should contain noneOf("testKey2b", "testKey3")
-        result.fields("testKey1a") should be(JsString("myValue1a"))
-        result.fields("testKey1b") should be(JsString("myValue1b"))
-        result.fields("testKey2a") should be(JsString("myValue2a"))
-      }
-  }
+    it should "return with excluded metadata from the metadata route" in {
+     Get(s"/workflows/$version/${MockApiService.ExistingWorkflowId}/metadata?excludeKey=testKey2b&excludeKey=testKey3") ~>
+        cromwellApiService.metadataRoute ~>
+        check {
+          status should be(StatusCodes.OK)
+          val result = responseAs[JsObject]
+          result.fields.keys should contain allOf("testKey1a", "testKey1b", "testKey2a")
+          result.fields.keys should contain noneOf("testKey2b", "testKey3")
+          result.fields("testKey1a") should be(JsString("myValue1a"))
+          result.fields("testKey1b") should be(JsString("myValue1b"))
+          result.fields("testKey2a") should be(JsString("myValue2a"))
+        }
+    }
 
   it should "return an error when included and excluded metadata requested from the metadata route" in {
-    val workflowId = WorkflowId.randomId()
-    publishStatusAndSubmission(workflowId, WorkflowSucceeded)
-
-    Get(s"/workflows/$version/$workflowId/metadata?includeKey=testKey1&excludeKey=testKey2") ~>
-      metadataRoute ~>
+    Get(s"/workflows/$version/${MockApiService.ExistingWorkflowId}/metadata?includeKey=testKey1&excludeKey=testKey2") ~>
+      cromwellApiService.metadataRoute ~>
       check {
         assertResult(StatusCodes.BadRequest) {
           status
@@ -507,12 +344,9 @@ class CromwellApiServiceSpec extends FlatSpec with CromwellApiService with Scala
   }
 
   behavior of "REST API /timing endpoint"
-
   it should "return 200 with an HTML document for the timings route" in {
-    publishStatusAndSubmission(MockWorkflowStoreActor.submittedWorkflowId, WorkflowSucceeded)
-
-    Get(s"/workflows/$version/${MockWorkflowStoreActor.submittedWorkflowId}/timing") ~>
-      timingRoute ~>
+    Get(s"/workflows/$version/${MockApiService.ExistingWorkflowId}/timing") ~>
+      cromwellApiService.timingRoute ~>
       check {
         assertResult(StatusCodes.OK) { status }
         assertResult("<html>") {
@@ -522,94 +356,22 @@ class CromwellApiServiceSpec extends FlatSpec with CromwellApiService with Scala
   }
 
   behavior of "REST API /query GET endpoint"
-
-  it should "return 400 for a bad query" in {
-    Get(s"/workflows/$version/query?BadKey=foo") ~>
-      queryRoute ~>
-      check {
-        assertResult(StatusCodes.BadRequest) {
-          status
-        }
-        assertResult(
-          s"""{
-              |  "status": "fail",
-              |  "message": "Unrecognized query keys: BadKey"
-              |}""".stripMargin
-        ) {
-          responseAs[String]
-        }
-      }
-  }
-
   it should "return good results for a good query" in {
-    val workflowId = WorkflowId.randomId()
-    val runningId = WorkflowId.randomId()
-    val events = Seq(
-      MetadataEvent(MetadataKey(workflowId, None, WorkflowMetadataKeys.Status), MetadataValue("Succeeded")),
-      MetadataEvent(MetadataKey(runningId, None, WorkflowMetadataKeys.Status), MetadataValue("Running"))
-    )
-
-    publishMetadata(events)
-    forceSummary()
-
-    Get(s"/workflows/$version/query?status=Succeeded&id=$workflowId") ~>
-      queryRoute ~>
+    Get(s"/workflows/$version/query?status=Succeeded&id=${MockApiService.ExistingWorkflowId}") ~>
+      cromwellApiService.queryRoute ~>
       check {
         status should be(StatusCodes.OK)
         val results = responseAs[JsObject].fields("results").convertTo[Seq[JsObject]]
 
-        results.head.fields("id") should be(JsString(workflowId.toString))
+        results.head.fields("id") should be(JsString(MockApiService.ExistingWorkflowId.toString))
         results.head.fields("status") should be(JsString("Succeeded"))
       }
   }
 
-  it should "return link headers for pagination when page and pagesize are set for a good query" in {
-    val workflowId = WorkflowId.randomId()
-    val events = Seq(
-      MetadataEvent(MetadataKey(workflowId, None, WorkflowMetadataKeys.Status), MetadataValue("Succeeded"))
-    )
-
-    publishMetadata(events)
-    forceSummary()
-
-    Get(s"/workflows/$version/query?status=Succeeded&id=$workflowId&page=1&pagesize=5") ~>
-      queryRoute ~>
-      check {
-        status should be(StatusCodes.OK)
-        val results = responseAs[JsObject].fields("results").convertTo[Seq[JsObject]]
-
-        results.head.fields("id") should be(JsString(workflowId.toString))
-        results.head.fields("status") should be(JsString("Succeeded"))
-
-        assertResult(4) {
-          headers count { header => header.is("link") }
-        }
-      }
-  }
-
   behavior of "REST API /query POST endpoint"
-
-  it should "return 400 for a bad query map body" in {
-    Post(s"/workflows/$version/query", HttpEntity(ContentTypes.`application/json`, """[{"BadKey":"foo"}]""")) ~>
-      queryPostRoute ~>
-      check {
-        assertResult(StatusCodes.BadRequest) {
-          status
-        }
-        assertResult(
-          s"""{
-              |  "status": "fail",
-              |  "message": "Unrecognized query keys: BadKey"
-              |}""".stripMargin
-        ) {
-          responseAs[String]
-        }
-      }
-  }
-
   it should "return good results for a good query map body" in {
     Post(s"/workflows/$version/query", HttpEntity(ContentTypes.`application/json`, """[{"status":"Succeeded"}]""")) ~>
-      queryPostRoute ~>
+      cromwellApiService.queryPostRoute ~>
       check {
         assertResult(StatusCodes.OK) {
           status
@@ -619,44 +381,107 @@ class CromwellApiServiceSpec extends FlatSpec with CromwellApiService with Scala
         }
       }
   }
+}
 
-  it should "return link headers for pagination when page and pagesize are set for a good query map body" in {
-    Post(s"/workflows/$version/query", HttpEntity(ContentTypes.`application/json`, """[{"status":"Succeeded"},  {"page": "1"}, {"pagesize": "5"}]""")) ~>
-      queryPostRoute ~>
-      check {
-        assertResult(StatusCodes.OK) {
-          status
-        }
-        assertResult(true) {
-          body.asString.contains("\"status\": \"Succeeded\"")
-          (headers count { header => header.is("link") }) == 4
-        }
+object CromwellApiServiceSpec {
+  class MockApiService()(implicit val system: ActorSystem) extends CromwellApiService {
+    import MockApiService._
+
+    override def actorRefFactory = system
+    override val workflowStoreActor = actorRefFactory.actorOf(Props(new MockWorkflowStoreActor()))
+    override val serviceRegistryActor = actorRefFactory.actorOf(Props.empty)
+    override val workflowManagerActor = actorRefFactory.actorOf(Props.empty)
+
+    override def handleMetadataRequest(message: AnyRef): Route = {
+      message match {
+        case GetStatus(w) => complete(submittedStatusResponse(w))
+        case WorkflowOutputs(w) => complete(outputResponse(w))
+        case GetLogs(w) => complete(logsResponse(w))
+        case GetSingleWorkflowMetadataAction(w, None, None, _) => complete(fullMetadataResponse(w))
+        case GetSingleWorkflowMetadataAction(w, Some(i), None, _) => complete(filteredMetadataResponse(w))
+        case GetSingleWorkflowMetadataAction(w, None, Some(_), _) => complete(filteredMetadataResponse(w))
+        case _ => throw new IllegalArgumentException("Woopsie!")
       }
-  }
+    }
 
+    override def handleQueryMetadataRequest(parameters: Seq[(String, String)]): Route = {
+      complete(QueryResponse)
+    }
 
-  it should "return good results for a multiple query map body" in {
-    Post(s"/workflows/$version/query", HttpEntity(ContentTypes.`application/json`,
-      """[{"status":"Succeeded"}, {"status":"Failed"}]""")) ~>
-      queryPostRoute ~>
-      check {
-        assertResult(StatusCodes.OK) {
-          status
+    override def withRecognizedWorkflowId(possibleWorkflowId: String)(recognizedWorkflowId: WorkflowId => Route): Route = {
+      requestContext =>
+        Try(WorkflowId.fromString(possibleWorkflowId)) match {
+          case Success(workflowId) =>
+              if (RecognizedWorkflowIds.contains(workflowId)) recognizedWorkflowId(workflowId)(requestContext)
+              else failBadRequest(new NoSuchElementException("A hollow voice says 'fool'"), StatusCodes.NotFound)(requestContext)
+          case Failure(e) =>
+            failBadRequest(new RuntimeException(s"Invalid workflow ID: '$possibleWorkflowId'."))(requestContext)
         }
-        assertResult(true) {
-          body.asString.contains("\"status\": \"Succeeded\"")
-        }
-      }
+    }
   }
 
-  it should "return 400 bad request for a bad query format body" in {
-    Post(s"/workflows/$version/query", HttpEntity(ContentTypes.`application/json`, """[{"status":["Succeeded"]}]""")) ~>
-      sealRoute(queryPostRoute) ~>
-      check {
-        assertResult(StatusCodes.BadRequest) {
-          status
-        }
-      }
+  object MockApiService {
+    val ExistingWorkflowId = WorkflowId.fromString("c4c6339c-8cc9-47fb-acc5-b5cb8d2809f5")
+    val AbortedWorkflowId = WorkflowId.fromString("0574111c-c7d3-4145-8190-7a7ed8e8324a")
+    val UnrecognizedWorkflowId = WorkflowId.fromString("2bdd06cc-e794-46c8-a897-4c86cedb6a06")
+    val RecognizedWorkflowIds = Set(ExistingWorkflowId, AbortedWorkflowId)
+
+    val QueryResponse = JsObject(Map(
+      "results" -> JsArray(JsObject(Map(
+        WorkflowMetadataKeys.Id -> JsString(ExistingWorkflowId.toString),
+        WorkflowMetadataKeys.Status -> JsString(WorkflowSucceeded.toString)
+      )))
+    ))
+
+    def fullMetadataResponse(workflowId: WorkflowId) = JsObject(Map(
+      "testKey1" -> JsString("myValue1"),
+      "testKey2" -> JsString("myValue2")
+    ))
+
+    def filteredMetadataResponse(workflowId: WorkflowId) = JsObject(Map(
+      "testKey1a" -> JsString("myValue1a"),
+      "testKey1b" -> JsString("myValue1b"),
+      "testKey2a" -> JsString("myValue2a")
+    ))
+
+    def submittedStatusResponse(workflowId: WorkflowId) = JsObject(Map(
+      WorkflowMetadataKeys.Status -> JsString(WorkflowSubmitted.toString),
+      WorkflowMetadataKeys.Id -> JsString(workflowId.toString)
+    ))
+
+    def outputResponse(workflowId: WorkflowId) = JsObject(Map(
+      WorkflowMetadataKeys.Id -> JsString(workflowId.toString),
+      WorkflowMetadataKeys.Outputs -> JsString("Some random stuff")
+    ))
+
+    def logsResponse(workflowId: WorkflowId) = JsObject(Map(
+      "calls" -> JsObject(Map(
+        "mycall" -> JsArray(JsObject(Map(
+          "stdout" -> JsString("stdout.txt"),
+          "stderr" -> JsString("stderr.txt"),
+          "backendLogs" -> JsObject(Map(
+            "log" -> JsString("backend.log")
+          ))
+        )))
+      ))
+    ))
+  }
+
+  class MockWorkflowStoreActor extends Actor {
+    override def receive = {
+      case SubmitWorkflow(source) => sender ! WorkflowSubmittedToStore(MockApiService.ExistingWorkflowId)
+      case BatchSubmitWorkflows(sources) =>
+        val response = WorkflowsBatchSubmittedToStore(sources map { _ => MockApiService.ExistingWorkflowId })
+        sender ! response
+      case AbortWorkflow(id, manager) =>
+        val message = id match {
+          case MockApiService.ExistingWorkflowId =>
+            WorkflowStoreEngineActor.WorkflowAborted(id)
+          case MockApiService.AbortedWorkflowId   =>
+            WorkflowAbortFailed(id, new IllegalStateException(s"Workflow ID '$id' is in terminal state 'Aborted' and cannot be aborted."))
+        }
+
+        sender ! message
+    }
   }
 }
-
diff --git a/project/Dependencies.scala b/project/Dependencies.scala
index 8711f0c08..eca62006a 100644
--- a/project/Dependencies.scala
+++ b/project/Dependencies.scala
@@ -12,7 +12,7 @@ object Dependencies {
   - http://doc.akka.io/docs/akka/2.4/scala/http/common/json-support.html#akka-http-spray-json
    */
   lazy val sprayJsonV = "1.3.2"
-  lazy val akkaV = "2.4.14"
+  lazy val akkaV = "2.4.16"
   lazy val slickV = "3.1.1"
   lazy val googleClientApiV = "1.22.0"
   lazy val googleGenomicsServicesApiV = "1.22.0"
diff --git a/services/src/main/scala/cromwell/services/metadata/impl/MetadataDatabaseAccess.scala b/services/src/main/scala/cromwell/services/metadata/impl/MetadataDatabaseAccess.scala
index 5250107b6..6736b221d 100644
--- a/services/src/main/scala/cromwell/services/metadata/impl/MetadataDatabaseAccess.scala
+++ b/services/src/main/scala/cromwell/services/metadata/impl/MetadataDatabaseAccess.scala
@@ -78,10 +78,8 @@ trait MetadataDatabaseAccess {
       val value = metadataEvent.value map { _.value }
       val valueType = metadataEvent.value map { _.valueType.typeName }
       val jobKey = key.jobKey map { jk => (jk.callFqn, jk.index, jk.attempt) }
-      MetadataEntry(
-        workflowUuid, jobKey.map(_._1), jobKey.flatMap(_._2), jobKey.map(_._3), key.key, value, valueType, timestamp)
+      MetadataEntry(workflowUuid, jobKey.map(_._1), jobKey.flatMap(_._2), jobKey.map(_._3), key.key, value, valueType, timestamp)
     }
-
     databaseInterface.addMetadataEntries(metadata)
   }
 
diff --git a/services/src/main/scala/cromwell/services/metadata/impl/MetadataServiceActor.scala b/services/src/main/scala/cromwell/services/metadata/impl/MetadataServiceActor.scala
index 1f66a5b52..34e6502f5 100644
--- a/services/src/main/scala/cromwell/services/metadata/impl/MetadataServiceActor.scala
+++ b/services/src/main/scala/cromwell/services/metadata/impl/MetadataServiceActor.scala
@@ -11,7 +11,9 @@ import cromwell.services.metadata.MetadataService.{PutMetadataAction, ReadAction
 import cromwell.services.metadata.impl.MetadataServiceActor._
 import cromwell.services.metadata.impl.MetadataSummaryRefreshActor.{MetadataSummaryFailure, MetadataSummarySuccess, SummarizeMetadata}
 import net.ceedubs.ficus.Ficus._
-import scala.concurrent.duration.{Duration, FiniteDuration}
+
+import scala.concurrent.duration._
+import scala.language.postfixOps
 import scala.util.{Failure, Success, Try}
 
 object MetadataServiceActor {
@@ -30,7 +32,10 @@ case class MetadataServiceActor(serviceConfig: Config, globalConfig: Config)
   private val summaryActor: Option[ActorRef] = buildSummaryActor
 
   val readActor = context.actorOf(ReadMetadataActor.props(), "read-metadata-actor")
-  val writeActor = context.actorOf(WriteMetadataActor.props(), "write-metadata-actor")
+
+  val dbFlushRate = serviceConfig.as[Option[FiniteDuration]]("services.MetadataService.db-flush-rate").getOrElse(5 seconds)
+  val dbBatchSize = serviceConfig.as[Option[Int]]("services.MetadataService.db-batch-size").getOrElse(1)
+  val writeActor = context.actorOf(WriteMetadataActor.props(dbBatchSize, dbFlushRate), "write-metadata-actor")
   implicit val ec = context.dispatcher
 
   summaryActor foreach { _ => self ! RefreshSummary }
diff --git a/services/src/main/scala/cromwell/services/metadata/impl/WriteMetadataActor.scala b/services/src/main/scala/cromwell/services/metadata/impl/WriteMetadataActor.scala
index dd45607fd..cb37d0bcb 100644
--- a/services/src/main/scala/cromwell/services/metadata/impl/WriteMetadataActor.scala
+++ b/services/src/main/scala/cromwell/services/metadata/impl/WriteMetadataActor.scala
@@ -1,29 +1,93 @@
 package cromwell.services.metadata.impl
 
-import akka.actor.{Actor, ActorLogging, Props}
+import akka.actor.{ActorLogging, LoggingFSM, Props}
+import cats.data.NonEmptyVector
 import cromwell.core.Dispatcher.ServiceDispatcher
 import cromwell.services.SingletonServicesStore
-import cromwell.services.metadata.MetadataService.{MetadataPutAcknowledgement, MetadataPutFailed, PutMetadataAction}
+import cromwell.services.metadata.MetadataEvent
+import cromwell.services.metadata.MetadataService.PutMetadataAction
+import cromwell.services.metadata.impl.WriteMetadataActor.{WriteMetadataActorData, WriteMetadataActorState}
 
+import scala.concurrent.duration._
 import scala.util.{Failure, Success}
 
-object WriteMetadataActor {
-  def props() = Props(new WriteMetadataActor()).withDispatcher(ServiceDispatcher)
+final case class WriteMetadataActor(batchRate: Int, flushRate: FiniteDuration)
+  extends LoggingFSM[WriteMetadataActorState, WriteMetadataActorData] with ActorLogging with
+  MetadataDatabaseAccess with SingletonServicesStore {
+  import WriteMetadataActor._
+
+  implicit val ec = context.dispatcher
+
+  override def preStart(): Unit = {
+    context.system.scheduler.schedule(0.seconds, flushRate, self, ScheduledFlushToDb)
+    super.preStart()
+  }
+
+  startWith(WaitingToWrite, NoEvents)
+
+  when(WaitingToWrite) {
+    case Event(PutMetadataAction(events), curData) =>
+      curData.addEvents(events) match {
+        case data@HasEvents(e) if e.length > batchRate => goto(WritingToDb) using data
+        case e => stay using e
+      }
+    case Event(ScheduledFlushToDb, curData) =>
+      log.debug("Initiating periodic metadata flush to DB")
+      goto(WritingToDb) using curData
+  }
+
+  when(WritingToDb) {
+    case Event(ScheduledFlushToDb, curData) => stay using curData
+    case Event(PutMetadataAction(events), curData) => stay using curData.addEvents(events)
+    case Event(FlushBatchToDb, NoEvents) =>
+      log.debug("Attempted metadata flush to DB but had nothing to write")
+      goto(WaitingToWrite) using NoEvents
+    case Event(FlushBatchToDb, HasEvents(e)) =>
+      log.debug("Flushing {} metadata events to the DB", e.length)
+      addMetadataEvents(e.toVector) onComplete {
+        case Success(_) => self ! DbWriteComplete
+        case Failure(regerts) =>
+          log.error("Failed to properly flush metadata to database", regerts)
+          self ! DbWriteComplete
+      }
+
+      stay using NoEvents
+    case Event(DbWriteComplete, curData) =>
+      log.debug("Flush of metadata events complete")
+      goto(WaitingToWrite) using curData
+  }
+
+  onTransition {
+    case WaitingToWrite -> WritingToDb => self ! FlushBatchToDb
+  }
 }
 
-class WriteMetadataActor extends Actor with ActorLogging with MetadataDatabaseAccess with SingletonServicesStore {
+object WriteMetadataActor {
+  def props(batchRate: Int, flushRate: FiniteDuration) = Props(new WriteMetadataActor(batchRate, flushRate)).withDispatcher(ServiceDispatcher)
 
-  implicit val ec = context.dispatcher
+  sealed trait WriteMetadataActorMessage
+  case object DbWriteComplete extends WriteMetadataActorMessage
+  case object FlushBatchToDb extends WriteMetadataActorMessage
+  case object ScheduledFlushToDb extends WriteMetadataActorMessage
 
-  def receive = {
-    case action@PutMetadataAction(events) =>
-      val sndr = sender()
-      addMetadataEvents(events) onComplete {
-        case Success(_) => sndr ! MetadataPutAcknowledgement(action)
-        case Failure(t) =>
-          val msg = MetadataPutFailed(action, t)
-          log.error(t, "Sending {} failure message {}", sndr, msg)
-          sndr ! msg
+  sealed trait WriteMetadataActorState
+  case object WaitingToWrite extends WriteMetadataActorState
+  case object WritingToDb extends WriteMetadataActorState
+
+  sealed trait WriteMetadataActorData {
+    def addEvents(newEvents: Iterable[MetadataEvent]): WriteMetadataActorData = {
+      NonEmptyVector.fromVector(newEvents.toVector) match {
+        case Some(v) =>
+          val newEvents = this match {
+            case NoEvents => v
+            case HasEvents(e) => e.concatNev(v)
+          }
+          HasEvents(newEvents)
+        case None => this
       }
+    }
   }
+
+  case object NoEvents extends WriteMetadataActorData
+  case class HasEvents(events: NonEmptyVector[MetadataEvent]) extends WriteMetadataActorData
 }
diff --git a/services/src/test/scala/cromwell/services/metadata/impl/MetadataServiceActorSpec.scala b/services/src/test/scala/cromwell/services/metadata/impl/MetadataServiceActorSpec.scala
index 12d201269..46829d71a 100644
--- a/services/src/test/scala/cromwell/services/metadata/impl/MetadataServiceActorSpec.scala
+++ b/services/src/test/scala/cromwell/services/metadata/impl/MetadataServiceActorSpec.scala
@@ -10,10 +10,9 @@ import cromwell.services.metadata.MetadataService._
 import cromwell.services.metadata._
 
 class MetadataServiceActorSpec extends ServicesSpec("Metadata") {
-  "MetadataServiceActor" should {
-
-    val config = ConfigFactory.empty()
-    val actor = system.actorOf(MetadataServiceActor.props(config, config))
+  import MetadataServiceActorSpec.Config
+  val config = ConfigFactory.parseString(Config)
+  val actor = system.actorOf(MetadataServiceActor.props(config, config))
 
     val workflowId = WorkflowId.randomId()
 
@@ -33,21 +32,15 @@ class MetadataServiceActorSpec extends ServicesSpec("Metadata") {
     val event3_1 = MetadataEvent(key3, Option(MetadataValue("value3")), moment)
     val event3_2 = MetadataEvent(key3, None, moment)
 
-    "Store values for different keys" in {
+  "MetadataServiceActor" should {
+    "Store values for different keys and then retrieve those values" in {
       val putAction1 = PutMetadataAction(event1_1)
       val putAction2 = PutMetadataAction(event1_2)
       val putAction3 = PutMetadataAction(event2_1, event3_1, event3_2)
-      (for {
-        response1 <- (actor ? putAction1).mapTo[MetadataServiceResponse]
-        response2 <- (actor ? putAction2).mapTo[MetadataServiceResponse]
-        response3 <- (actor ? putAction3).mapTo[MetadataServiceResponse]
-        _ = response1 shouldBe MetadataPutAcknowledgement(putAction1)
-        _ = response2 shouldBe MetadataPutAcknowledgement(putAction2)
-        _ = response3 shouldBe MetadataPutAcknowledgement(putAction3)
-      } yield ()).futureValue
-    }
+      actor ! putAction1
+      actor ! putAction2
+      actor ! putAction3
 
-    "Retrieve the correct values for different keys" in {
       val query1 = MetadataQuery.forKey(key1)
       val query2 = MetadataQuery.forKey(key2)
       val query3 = MetadataQuery.forKey(key3)
@@ -72,25 +65,13 @@ class MetadataServiceActorSpec extends ServicesSpec("Metadata") {
 
       } yield ()).futureValue
     }
-
-    "store and retrieve null values" in {
-      val metadataWorkflowId = WorkflowId.randomId()
-      val metadataKey = MetadataKey(metadataWorkflowId, None, "nullTestKey")
-      val metadataValue = MetadataValue(null)
-      val metadataEvent = MetadataEvent(metadataKey, Option(metadataValue), moment)
-
-      val putMetadataAction = PutMetadataAction(metadataEvent)
-
-      val metadataQuery = MetadataQuery.forKey(metadataKey)
-
-      (for {
-        putResponse <- (actor ? putMetadataAction).mapTo[MetadataServiceResponse]
-        _ = putResponse shouldBe MetadataPutAcknowledgement(putMetadataAction)
-
-        queryResponse <- (actor ? GetMetadataQueryAction(metadataQuery)).mapTo[MetadataServiceResponse]
-        _ = queryResponse shouldBe MetadataLookupResponse(metadataQuery, Seq(metadataEvent))
-      } yield ()).futureValue
-    }
-
   }
 }
+
+object MetadataServiceActorSpec {
+  val Config =
+    """
+      |services.MetadataService.db-batch-size = 3
+      |services.MetadataService.db-flush-rate = 100 millis
+    """.stripMargin
+}
diff --git a/services/src/test/scala/cromwell/services/metadata/impl/MetadataValueSpec.scala b/services/src/test/scala/cromwell/services/metadata/impl/MetadataValueSpec.scala
new file mode 100644
index 000000000..316ad5578
--- /dev/null
+++ b/services/src/test/scala/cromwell/services/metadata/impl/MetadataValueSpec.scala
@@ -0,0 +1,11 @@
+package cromwell.services.metadata.impl
+
+import cromwell.services.metadata.MetadataValue
+import org.scalatest.{FlatSpec, Matchers}
+
+class MetadataValueSpec extends FlatSpec with Matchers {
+  "MetadataValue" should "not NPE with a null value" in {
+    val metadataValue = MetadataValue(null)
+    metadataValue.value shouldBe ""
+  }
+}
diff --git a/services/src/test/scala/cromwell/services/metadata/impl/WriteMetadataActorSpec.scala b/services/src/test/scala/cromwell/services/metadata/impl/WriteMetadataActorSpec.scala
new file mode 100644
index 000000000..577013aef
--- /dev/null
+++ b/services/src/test/scala/cromwell/services/metadata/impl/WriteMetadataActorSpec.scala
@@ -0,0 +1,44 @@
+package cromwell.services.metadata.impl
+
+import java.time.OffsetDateTime
+
+import akka.testkit.TestFSMRef
+import cats.data.NonEmptyVector
+import cromwell.core.WorkflowId
+import cromwell.services.ServicesSpec
+import cromwell.services.metadata.MetadataService.PutMetadataAction
+import cromwell.services.metadata.{MetadataEvent, MetadataKey, MetadataValue}
+import cromwell.services.metadata.impl.WriteMetadataActor.{HasEvents, NoEvents, WaitingToWrite, WritingToDb}
+
+import scala.concurrent.duration._
+
+class WriteMetadataActorSpec extends ServicesSpec("Metadata") {
+  import WriteMetadataActorSpec.Action
+
+  val actor = TestFSMRef(WriteMetadataActor(10, 1.days)) // A WMA that won't (hopefully!) perform a time based flush during this test
+
+  "WriteMetadataActor" should {
+    "start with no events and waiting to write" in {
+      assert(actor.stateName == WaitingToWrite)
+      assert(actor.stateData == NoEvents)
+    }
+
+    "Have one event and be waiting after one event is sent" in {
+      actor ! Action
+      assert(actor.stateName == WaitingToWrite)
+      assert(actor.stateData == HasEvents(NonEmptyVector.fromVectorUnsafe(Action.events.toVector)))
+    }
+
+    "Have one event after batch size + 1 is reached" in {
+      1 to 10 foreach { _ => actor ! Action }
+      assert(actor.stateName == WritingToDb)
+      actor ! Action
+      assert(actor.stateData == HasEvents(NonEmptyVector.fromVectorUnsafe(Action.events.toVector)))
+    }
+  }
+}
+
+object WriteMetadataActorSpec {
+  val Event = MetadataEvent(MetadataKey(WorkflowId.randomId(), None, "key"), Option(MetadataValue("value")), OffsetDateTime.now)
+  val Action = PutMetadataAction(Event)
+}

From dc8f0f16b42f883ef8d06108eb62a00f65da96ab Mon Sep 17 00:00:00 2001
From: Jeff Gentry <geoffjentry@gmail.com>
Date: Mon, 23 Jan 2017 17:34:51 -0500
Subject: [PATCH 29/84] Remove the 'Some' from an error message (#1899)

---
 .../jes/JesAsyncBackendJobExecutionActor.scala     | 56 ++++++++++++----------
 1 file changed, 30 insertions(+), 26 deletions(-)

diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala
index a59259677..11982fa89 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala
@@ -405,11 +405,11 @@ class JesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn
     errorMessage.substring(0, errorMessage.indexOf(':')).toInt
   }
 
-  private def preempted(errorCode: Int, errorMessage: Option[String]): Boolean = {
+  private def preempted(errorCode: Int, errorMessage: String): Boolean = {
     def isPreemptionCode(code: Int) = code == 13 || code == 14
 
     try {
-      errorCode == 10 && errorMessage.exists(e => isPreemptionCode(extractErrorCodeFromErrorMessage(e))) && preemptible
+      errorCode == 10 && isPreemptionCode(extractErrorCodeFromErrorMessage(errorMessage)) && preemptible
     } catch {
       case _: NumberFormatException | _: StringIndexOutOfBoundsException =>
         jobLogger.warn(s"Unable to parse JES error code from error messages: [{}], assuming this was not a preempted VM.", errorMessage.mkString(", "))
@@ -420,39 +420,43 @@ class JesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn
   override def handleExecutionFailure(runStatus: RunStatus,
                                       handle: StandardAsyncPendingExecutionHandle,
                                       returnCode: Option[Int]): ExecutionHandle = {
+    import lenthall.numeric.IntegerUtil._
+
     val failed = runStatus match {
       case failedStatus: RunStatus.Failed => failedStatus
       case unknown => throw new RuntimeException(s"handleExecutionFailure not called with RunStatus.Failed. Instead got $unknown")
     }
+
+    // In the event that the error isn't caught by the special cases, this will be used
+    def nonRetryableException = {
+      val exception = failed.toFailure(jobPaths.jobKey.tag, Option(jobPaths.stderr))
+      FailedNonRetryableExecutionHandle(exception, returnCode)
+    }
     
     val errorCode = failed.errorCode
-    val errorMessage = failed.errorMessage
-
-    import lenthall.numeric.IntegerUtil._
-
     val taskName = s"${workflowDescriptor.id}:${call.unqualifiedName}"
     val attempt = jobDescriptor.key.attempt
 
-    if (errorMessage.exists(_.contains("Operation canceled at")))  {
-      AbortedExecutionHandle
-    } else if (preempted(errorCode, errorMessage)) {
-      val preemptedMsg = s"Task $taskName was preempted for the ${attempt.toOrdinal} time."
-
-      if (attempt < maxPreemption) {
-        val e = PreemptedException(
-          s"""$preemptedMsg The call will be restarted with another preemptible VM (max preemptible attempts number is $maxPreemption).
-             |Error code $errorCode. Message: $errorMessage""".stripMargin
-        )
-        FailedRetryableExecutionHandle(e, returnCode)
-      } else {
-        val e = PreemptedException(
-          s"""$preemptedMsg The maximum number of preemptible attempts ($maxPreemption) has been reached. The call will be restarted with a non-preemptible VM.
-             |Error code $errorCode. Message: $errorMessage)""".stripMargin)
-        FailedRetryableExecutionHandle(e, returnCode)
-      }
-    } else {
-      val exception = failed.toFailure(jobPaths.jobKey.tag, Option(jobPaths.stderr))
-      FailedNonRetryableExecutionHandle(exception, returnCode)
+    failed.errorMessage match {
+      case Some(errorMessage) =>
+        if (errorMessage.contains("Operation canceled at"))  {
+          AbortedExecutionHandle
+        } else if (preempted(errorCode, errorMessage)) {
+          val preemptedMsg = s"Task $taskName was preempted for the ${attempt.toOrdinal} time."
+          if (attempt < maxPreemption) {
+            val e = PreemptedException(
+              s"""$preemptedMsg The call will be restarted with another preemptible VM (max preemptible attempts number is $maxPreemption).
+                 |Error code $errorCode. Message: $errorMessage""".stripMargin
+            )
+            FailedRetryableExecutionHandle(e, returnCode)
+          } else {
+            val e = PreemptedException(
+              s"""$preemptedMsg The maximum number of preemptible attempts ($maxPreemption) has been reached. The call will be restarted with a non-preemptible VM.
+                 |Error code $errorCode. Message: $errorMessage)""".stripMargin)
+            FailedRetryableExecutionHandle(e, returnCode)
+          }
+        } else { nonRetryableException }
+      case None => nonRetryableException
     }
   }
 

From 885f8c98d9f24010dbdafbf4890124cf38124ea0 Mon Sep 17 00:00:00 2001
From: Khalid Shakir <kshakir@broadinstitute.org>
Date: Mon, 23 Jan 2017 23:16:42 -0500
Subject: [PATCH 30/84] Synced database schema and added more specs. PK, FK,
 UC, and IX names all use a standard naming pattern, and synced up
 clob/varchar lengths. Added specs that check for the above. Removed checks
 for unique index mismatches, as we have none. Added blob implicits to match
 clob implicits. Reimplemented database deadlock test. Added an eventually
 wrapper within MetadataServiceActorSpec.

---
 .../migration/src/main/resources/changelog.xml     |   1 +
 .../standardize_column_names_patches.xml           |  64 ++++
 .../migration/liquibase/DiffResultFilter.scala     |  56 ++--
 .../slick/tables/CallCachingEntryComponent.scala   |   7 +-
 .../tables/CallCachingHashEntryComponent.scala     |   4 +-
 .../CallCachingSimpletonEntryComponent.scala       |   4 +-
 .../slick/tables/JobStoreEntryComponent.scala      |   4 +-
 .../tables/JobStoreSimpletonEntryComponent.scala   |   6 +-
 .../slick/tables/MetadataEntryComponent.scala      |   6 +-
 .../tables/SubWorkflowStoreEntryComponent.scala    |   4 +-
 .../WorkflowMetadataSummaryEntryComponent.scala    |   6 +-
 .../slick/tables/WorkflowStoreEntryComponent.scala |   6 +-
 .../cromwell/database/sql/SqlConverters.scala      |  13 +-
 .../sql/tables/CallCachingDetritusEntry.scala      |   2 +-
 .../sql/tables/CallCachingSimpletonEntry.scala     |   4 +-
 .../database/sql/tables/JobStoreEntry.scala        |   4 +-
 .../sql/tables/JobStoreSimpletonEntry.scala        |   4 +-
 .../database/sql/tables/MetadataEntry.scala        |   4 +-
 .../sql/tables/SubWorkflowStoreEntry.scala         |   4 +-
 .../database/sql/tables/WorkflowStoreEntry.scala   |   2 +-
 engine/src/main/scala/cromwell/Simpletons.scala    |   5 +-
 .../execution/callcaching/CallCache.scala          |   4 +-
 .../workflow/workflowstore/SqlWorkflowStore.scala  |   7 +-
 .../main/scala/cromwell/jobstore/SqlJobStore.scala |   8 +-
 .../metadata/impl/MetadataDatabaseAccess.scala     |  15 +-
 .../cromwell/services/ServicesStoreSpec.scala      | 351 ++++++++++++++++-----
 .../metadata/impl/MetadataServiceActorSpec.scala   |  30 +-
 27 files changed, 460 insertions(+), 165 deletions(-)
 create mode 100644 database/migration/src/main/resources/changesets/standardize_column_names_patches.xml

diff --git a/database/migration/src/main/resources/changelog.xml b/database/migration/src/main/resources/changelog.xml
index cbbfe50f0..2fb2a8f47 100644
--- a/database/migration/src/main/resources/changelog.xml
+++ b/database/migration/src/main/resources/changelog.xml
@@ -56,6 +56,7 @@
     <include file="changesets/workflow_store_imports_file.xml" relativeToChangelogFile="true" />
     <include file="changesets/workflow_store_labels_file.xml" relativeToChangelogFile="true" />
     <include file="changesets/embiggen_detritus_value.xml" relativeToChangelogFile="true" />
+    <include file="changesets/standardize_column_names_patches.xml" relativeToChangelogFile="true" />
 </databaseChangeLog>
 <!--
 
diff --git a/database/migration/src/main/resources/changesets/standardize_column_names_patches.xml b/database/migration/src/main/resources/changesets/standardize_column_names_patches.xml
new file mode 100644
index 000000000..85eef3be7
--- /dev/null
+++ b/database/migration/src/main/resources/changesets/standardize_column_names_patches.xml
@@ -0,0 +1,64 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
+
+    <changeSet author="kshakir" id="standardize_column_names_patches">
+        <comment>
+            Columns now all match standardized naming.
+            FK_CALL_CACHING_HASH_ENTRY_CALL_CACHING_ENTRY_ID must be dropped and re-added so that
+            UC_CALL_CACHING_HASH_ENTRY_CCEI may be renamed.
+        </comment>
+        <dropForeignKeyConstraint baseTableName="CALL_CACHING_HASH_ENTRY"
+                                  constraintName="FK_CALL_CACHING_HASH_ENTRY_CALL_CACHING_ENTRY_ID"/>
+        <dropForeignKeyConstraint baseTableName="JOB_STORE_SIMPLETON_ENTRY"
+                                  constraintName="FK_JOB_STORE_RESULT_SIMPLETON_JOB_STORE_ENTRY_ID"/>
+        <dropForeignKeyConstraint baseTableName="SUB_WORKFLOW_STORE_ENTRY"
+                                  constraintName="FK_SUB_WORKFLOW_STORE_ROOT_WORKFLOW_ID_WORKFLOW_STORE_ENTRY_ID"/>
+
+        <dropUniqueConstraint tableName="CALL_CACHING_ENTRY"
+                              constraintName="UC_CALL_CACHING_ENTRY_WEU_CQFN_JI"/>
+        <dropUniqueConstraint tableName="CALL_CACHING_HASH_ENTRY"
+                              constraintName="UC_CALL_CACHING_HASH_ENTRY_CCEI"/>
+
+        <addUniqueConstraint
+                tableName="CALL_CACHING_ENTRY"
+                constraintName="UC_CALL_CACHING_ENTRY_WEU_CFQN_JI"
+                columnNames="WORKFLOW_EXECUTION_UUID, CALL_FULLY_QUALIFIED_NAME, JOB_INDEX"/>
+        <addUniqueConstraint
+                tableName="CALL_CACHING_HASH_ENTRY"
+                constraintName="UC_CALL_CACHING_HASH_ENTRY_CCEI_HK"
+                columnNames="CALL_CACHING_ENTRY_ID, HASH_KEY"/>
+
+        <addForeignKeyConstraint constraintName="FK_CALL_CACHING_HASH_ENTRY_CALL_CACHING_ENTRY_ID"
+                                 baseTableName="CALL_CACHING_HASH_ENTRY"
+                                 baseColumnNames="CALL_CACHING_ENTRY_ID"
+                                 referencedTableName="CALL_CACHING_ENTRY"
+                                 referencedColumnNames="CALL_CACHING_ENTRY_ID"/>
+        <addForeignKeyConstraint constraintName="FK_JOB_STORE_SIMPLETON_ENTRY_JOB_STORE_ENTRY_ID" onDelete="CASCADE"
+                                 baseTableName="JOB_STORE_SIMPLETON_ENTRY"
+                                 baseColumnNames="JOB_STORE_ENTRY_ID"
+                                 referencedTableName="JOB_STORE_ENTRY"
+                                 referencedColumnNames="JOB_STORE_ENTRY_ID"/>
+        <addForeignKeyConstraint constraintName="FK_SUB_WORKFLOW_STORE_ENTRY_ROOT_WORKFLOW_ID" onDelete="CASCADE"
+                                 baseTableName="SUB_WORKFLOW_STORE_ENTRY"
+                                 baseColumnNames="ROOT_WORKFLOW_ID"
+                                 referencedTableName="WORKFLOW_STORE_ENTRY"
+                                 referencedColumnNames="WORKFLOW_STORE_ENTRY_ID"/>
+    </changeSet>
+
+    <changeSet author="kshakir" id="hsqldb_longvarchar_to_longtext" dbms="hsqldb">
+        <comment>
+            HSQLDB can handle longtext just fine. Instead of using very wide varchars, use clob.
+        </comment>
+        <modifyDataType newDataType="LONGTEXT" tableName="CALL_CACHING_DETRITUS_ENTRY" columnName="DETRITUS_VALUE"/>
+        <modifyDataType newDataType="LONGTEXT" tableName="CALL_CACHING_SIMPLETON_ENTRY" columnName="SIMPLETON_VALUE"/>
+        <modifyDataType newDataType="LONGTEXT" tableName="JOB_STORE_ENTRY" columnName="EXCEPTION_MESSAGE"/>
+        <modifyDataType newDataType="LONGTEXT" tableName="JOB_STORE_SIMPLETON_ENTRY" columnName="SIMPLETON_VALUE"/>
+        <modifyDataType newDataType="LONGTEXT" tableName="WORKFLOW_STORE_ENTRY" columnName="CUSTOM_LABELS"/>
+        <modifyDataType newDataType="LONGTEXT" tableName="WORKFLOW_STORE_ENTRY" columnName="WORKFLOW_DEFINITION"/>
+        <modifyDataType newDataType="LONGTEXT" tableName="WORKFLOW_STORE_ENTRY" columnName="WORKFLOW_INPUTS"/>
+        <modifyDataType newDataType="LONGTEXT" tableName="WORKFLOW_STORE_ENTRY" columnName="WORKFLOW_OPTIONS"/>
+    </changeSet>
+
+</databaseChangeLog>
diff --git a/database/migration/src/main/scala/cromwell/database/migration/liquibase/DiffResultFilter.scala b/database/migration/src/main/scala/cromwell/database/migration/liquibase/DiffResultFilter.scala
index e341b824b..34bc11ee3 100644
--- a/database/migration/src/main/scala/cromwell/database/migration/liquibase/DiffResultFilter.scala
+++ b/database/migration/src/main/scala/cromwell/database/migration/liquibase/DiffResultFilter.scala
@@ -3,7 +3,7 @@ package cromwell.database.migration.liquibase
 import liquibase.database.Database
 import liquibase.diff.{DiffResult, Difference, ObjectDifferences}
 import liquibase.structure.DatabaseObject
-import liquibase.structure.core.{DataType, Index}
+import liquibase.structure.core.DataType
 
 import scala.collection.JavaConverters._
 
@@ -24,30 +24,7 @@ object DiffResultFilter {
   /**
     * The standard type filters to ignore, including interchangeable types.
     */
-  val StandardTypeFilters: Seq[DiffFilter] =
-    Seq(isReordered, isTypeSimilar("CLOB", "VARCHAR"), isTypeSimilar("TINYINT", "BOOLEAN"))
-
-  /**
-    * Removes all unique index differences.
-    *
-    * When searching for unique indexes, liquibase seems to get confused in locating the unique indexes within the
-    * database objects. This leads to false positive differences. The error messages look like:
-    *
-    * [info] - should have the same liquibase and slick schema *** FAILED ***
-    * [info]   {SYS_IDX_10267 on SGE_JOB(EXECUTION_ID)=liquibase.diff.ObjectDifferences@4ad0de69,
-    * SYS_IDX_UK_JJ_EXECUTION_UUID_10194 unique  on JES_JOB(EXECUTION_ID)=liquibase.diff.ObjectDifferences@56873b89,
-    * SYS_IDX_10259 on LOCAL_JOB(EXECUTION_ID)=liquibase.diff.ObjectDifferences@62e10bdf, SYS_IDX_10224 on
-    * WORKFLOW_EXECUTION_AUX(WORKFLOW_EXECUTION_ID)=liquibase.diff.ObjectDifferences@2a0b9461} was not empty
-    * (SlickDataAccessSpec.scala)
-    *
-    * When diving down into the `ObjectDifferences`, each is an instance of either:
-    *
-    * - unique changed from 'true' to 'false'
-    * - unique changed from 'false' to 'true'
-    */
-  val UniqueIndexFilter: DiffFilter = { (_, _, databaseObject, difference) =>
-    difference.getField == "unique" && databaseObject.isInstanceOf[Index]
-  }
+  val StandardTypeFilters: Seq[DiffFilter] = Seq(isReordered, isVarchar255, isTypeSimilar("TINYINT", "BOOLEAN"))
 
   /**
     * Filters diff results using an object filter and changed object filters. Filters that return false are removed.
@@ -104,6 +81,28 @@ object DiffResultFilter {
   }
 
   /**
+    * Checks if the difference is due to the schemas being created with our default varchar(255).
+    *
+    * @param referenceDatabase  The reference database.
+    * @param comparisonDatabase The comparison database.
+    * @param databaseObject     The database object.
+    * @param difference         The difference reported.
+    * @return True if the object is actually the same with slightly different column widths.
+    */
+  def isVarchar255(referenceDatabase: Database, comparisonDatabase: Database,
+                   databaseObject: DatabaseObject, difference: Difference): Boolean = {
+    val compared = difference.getComparedValue
+    val referenced = difference.getReferenceValue
+    compared.isInstanceOf[DataType] && referenced.isInstanceOf[DataType] && {
+      val comparedDataType = compared.asInstanceOf[DataType]
+      val referencedDataType = referenced.asInstanceOf[DataType]
+      comparedDataType.getTypeName == "VARCHAR" && referencedDataType.getTypeName == "VARCHAR" &&
+        // Our liquibase copypasta defaults VARCHAR to 255. Slick without a value defaults to 254
+        (comparedDataType.getColumnSize + referencedDataType.getColumnSize == 255 + 254)
+    }
+  }
+
+  /**
     * Checks if the difference is due to the schemas being created with different but still similar types.
     *
     * Returns true if the type names are both found in the list, or if the types are the same name.
@@ -121,10 +120,9 @@ object DiffResultFilter {
     val compared = difference.getComparedValue
     val referenced = difference.getReferenceValue
     compared.isInstanceOf[DataType] && referenced.isInstanceOf[DataType] && {
-      val comparedType = compared.asInstanceOf[DataType].getTypeName
-      val referencedType = referenced.asInstanceOf[DataType].getTypeName
-      comparedType == referencedType ||
-        (similarTypes.contains(comparedType.toUpperCase) && similarTypes.contains(referencedType.toUpperCase))
+      val comparedType = compared.asInstanceOf[DataType].getTypeName.toUpperCase
+      val referencedType = referenced.asInstanceOf[DataType].getTypeName.toUpperCase
+      similarTypes.contains(comparedType) && similarTypes.contains(referencedType.toUpperCase)
     }
   }
 
diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingEntryComponent.scala
index 189402814..5e37932e7 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingEntryComponent.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingEntryComponent.scala
@@ -1,7 +1,6 @@
 package cromwell.database.slick.tables
 
 import cromwell.database.sql.tables.CallCachingEntry
-import slick.profile.RelationalProfile.ColumnOption.Default
 
 trait CallCachingEntryComponent {
 
@@ -20,13 +19,13 @@ trait CallCachingEntryComponent {
 
     def returnCode = column[Option[Int]]("RETURN_CODE")
 
-    def allowResultReuse = column[Boolean]("ALLOW_RESULT_REUSE", Default(true))
+    def allowResultReuse = column[Boolean]("ALLOW_RESULT_REUSE", O.Default(true))
 
     override def * = (workflowExecutionUuid, callFullyQualifiedName, jobIndex, returnCode, allowResultReuse,
       callCachingEntryId.?) <> (CallCachingEntry.tupled, CallCachingEntry.unapply)
 
-    def ucCallCachingEntryWeuCqfnJi =
-      index("UC_CALL_CACHING_ENTRY_WEU_CQFN_JI", (workflowExecutionUuid, callFullyQualifiedName, jobIndex),
+    def ucCallCachingEntryWeuCfqnJi =
+      index("UC_CALL_CACHING_ENTRY_WEU_CFQN_JI", (workflowExecutionUuid, callFullyQualifiedName, jobIndex),
         unique = true)
   }
 
diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingHashEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingHashEntryComponent.scala
index b82ec957c..049f2a4f1 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingHashEntryComponent.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingHashEntryComponent.scala
@@ -24,8 +24,8 @@ trait CallCachingHashEntryComponent {
     def fkCallCachingHashEntryCallCachingEntryId = foreignKey("FK_CALL_CACHING_HASH_ENTRY_CALL_CACHING_ENTRY_ID",
       callCachingEntryId, callCachingEntries)(_.callCachingEntryId)
 
-    def ucCallCachingHashEntryCcei =
-      index("UC_CALL_CACHING_HASH_ENTRY_CCEI", (callCachingEntryId, hashKey), unique = true)
+    def ucCallCachingHashEntryCceiHk =
+      index("UC_CALL_CACHING_HASH_ENTRY_CCEI_HK", (callCachingEntryId, hashKey), unique = true)
   }
 
   protected val callCachingHashEntries = TableQuery[CallCachingHashEntries]
diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingSimpletonEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingSimpletonEntryComponent.scala
index 881e13500..a47f0df19 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingSimpletonEntryComponent.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingSimpletonEntryComponent.scala
@@ -1,5 +1,7 @@
 package cromwell.database.slick.tables
 
+import java.sql.Clob
+
 import cromwell.database.sql.tables.CallCachingSimpletonEntry
 
 trait CallCachingSimpletonEntryComponent {
@@ -14,7 +16,7 @@ trait CallCachingSimpletonEntryComponent {
 
     def simpletonKey = column[String]("SIMPLETON_KEY")
 
-    def simpletonValue = column[String]("SIMPLETON_VALUE")
+    def simpletonValue = column[Clob]("SIMPLETON_VALUE")
 
     def wdlType = column[String]("WDL_TYPE")
 
diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/JobStoreEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/JobStoreEntryComponent.scala
index 6026e2aec..58d457fdd 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/tables/JobStoreEntryComponent.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/tables/JobStoreEntryComponent.scala
@@ -1,5 +1,7 @@
 package cromwell.database.slick.tables
 
+import java.sql.Clob
+
 import cromwell.database.sql.tables.JobStoreEntry
 
 trait JobStoreEntryComponent {
@@ -24,7 +26,7 @@ trait JobStoreEntryComponent {
     def returnCode = column[Option[Int]]("RETURN_CODE")
 
     // Only set for failure:
-    def exceptionMessage = column[Option[String]]("EXCEPTION_MESSAGE")
+    def exceptionMessage = column[Option[Clob]]("EXCEPTION_MESSAGE")
 
     def retryableFailure = column[Option[Boolean]]("RETRYABLE_FAILURE")
 
diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/JobStoreSimpletonEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/JobStoreSimpletonEntryComponent.scala
index f0232886f..c9d51ca51 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/tables/JobStoreSimpletonEntryComponent.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/tables/JobStoreSimpletonEntryComponent.scala
@@ -1,5 +1,7 @@
 package cromwell.database.slick.tables
 
+import java.sql.Clob
+
 import cromwell.database.sql.tables.JobStoreSimpletonEntry
 import slick.model.ForeignKeyAction.Cascade
 
@@ -14,7 +16,7 @@ trait JobStoreSimpletonEntryComponent {
 
     def simpletonKey = column[String]("SIMPLETON_KEY")
 
-    def simpletonValue = column[String]("SIMPLETON_VALUE")
+    def simpletonValue = column[Clob]("SIMPLETON_VALUE")
 
     def wdlType = column[String]("WDL_TYPE")
 
@@ -23,7 +25,7 @@ trait JobStoreSimpletonEntryComponent {
     override def * = (simpletonKey, simpletonValue, wdlType, jobStoreEntryId.?, jobStoreSimpletonEntryId.?) <>
       (JobStoreSimpletonEntry.tupled, JobStoreSimpletonEntry.unapply)
 
-    def fkJobStoreResultSimpletonJobStoreEntryId = foreignKey("FK_JOB_STORE_RESULT_SIMPLETON_JOB_STORE_ENTRY_ID",
+    def fkJobStoreSimpletonEntryJobStoreEntryId = foreignKey("FK_JOB_STORE_SIMPLETON_ENTRY_JOB_STORE_ENTRY_ID",
       jobStoreEntryId, jobStoreEntries)(_.jobStoreEntryId, onDelete = Cascade)
 
     def ucJobStoreSimpletonEntryJseiSk =
diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/MetadataEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/MetadataEntryComponent.scala
index 4cc960213..745a1e593 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/tables/MetadataEntryComponent.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/tables/MetadataEntryComponent.scala
@@ -1,6 +1,6 @@
 package cromwell.database.slick.tables
 
-import java.sql.Timestamp
+import java.sql.{Clob, Timestamp}
 
 import cats.data.NonEmptyList
 import cromwell.database.sql.tables.MetadataEntry
@@ -36,9 +36,9 @@ trait MetadataEntryComponent {
 
     def metadataKey = column[String]("METADATA_KEY")
 
-    def metadataValue = column[Option[String]]("METADATA_VALUE")
+    def metadataValue = column[Option[Clob]]("METADATA_VALUE")
 
-    def metadataValueType = column[Option[String]]("METADATA_VALUE_TYPE")
+    def metadataValueType = column[Option[String]]("METADATA_VALUE_TYPE", O.Length(10))
 
     def metadataTimestamp = column[Timestamp]("METADATA_TIMESTAMP")
 
diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/SubWorkflowStoreEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/SubWorkflowStoreEntryComponent.scala
index 848c60c4b..11593f4a7 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/tables/SubWorkflowStoreEntryComponent.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/tables/SubWorkflowStoreEntryComponent.scala
@@ -26,10 +26,10 @@ trait SubWorkflowStoreEntryComponent {
 
     override def * = (rootWorkflowId.?, parentWorkflowExecutionUuid, callFullyQualifiedName, callIndex, callAttempt, subWorkflowExecutionUuid, subWorkflowStoreEntryId.?) <> (SubWorkflowStoreEntry.tupled, SubWorkflowStoreEntry.unapply)
 
-    def ucSubWorkflowStoreEntryPweuCfqnJiJa = index("UC_SUB_WORKFLOW_STORE_ENTRY_PWEU_CFQN_CI_CA",
+    def ucSubWorkflowStoreEntryPweuCfqnCiCa = index("UC_SUB_WORKFLOW_STORE_ENTRY_PWEU_CFQN_CI_CA",
       (parentWorkflowExecutionUuid, callFullyQualifiedName, callIndex, callAttempt), unique = true)
 
-    def fkSubWorkflowStoreRootWorkflowStoreEntryId = foreignKey("FK_SUB_WORKFLOW_STORE_ROOT_WORKFLOW_ID_WORKFLOW_STORE_ENTRY_ID",
+    def fkSubWorkflowStoreEntryRootWorkflowId = foreignKey("FK_SUB_WORKFLOW_STORE_ENTRY_ROOT_WORKFLOW_ID",
       rootWorkflowId, workflowStoreEntries)(_.workflowStoreEntryId, onDelete = Cascade)
 
     def ixSubWorkflowStoreEntryPweu = index("IX_SUB_WORKFLOW_STORE_ENTRY_PWEU", parentWorkflowExecutionUuid, unique = false)
diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowMetadataSummaryEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowMetadataSummaryEntryComponent.scala
index 65ea61530..2142c7188 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowMetadataSummaryEntryComponent.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowMetadataSummaryEntryComponent.scala
@@ -14,11 +14,11 @@ trait WorkflowMetadataSummaryEntryComponent {
     extends Table[WorkflowMetadataSummaryEntry](tag, "WORKFLOW_METADATA_SUMMARY_ENTRY") {
     def workflowMetadataSummaryEntryId = column[Long]("WORKFLOW_METADATA_SUMMARY_ENTRY_ID", O.PrimaryKey, O.AutoInc)
 
-    def workflowExecutionUuid = column[String]("WORKFLOW_EXECUTION_UUID")
+    def workflowExecutionUuid = column[String]("WORKFLOW_EXECUTION_UUID", O.Length(100))
 
-    def workflowName = column[Option[String]]("WORKFLOW_NAME")
+    def workflowName = column[Option[String]]("WORKFLOW_NAME", O.Length(100))
 
-    def workflowStatus = column[Option[String]]("WORKFLOW_STATUS")
+    def workflowStatus = column[Option[String]]("WORKFLOW_STATUS", O.Length(50))
 
     def startTimestamp = column[Option[Timestamp]]("START_TIMESTAMP")
 
diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowStoreEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowStoreEntryComponent.scala
index 5f4df4216..213e2cbe6 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowStoreEntryComponent.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowStoreEntryComponent.scala
@@ -23,14 +23,14 @@ trait WorkflowStoreEntryComponent {
 
     def customLabels = column[Clob]("CUSTOM_LABELS")
 
-    def workflowState = column[String]("WORKFLOW_STATE")
+    def workflowState = column[String]("WORKFLOW_STATE", O.Length(15))
 
     def submissionTime = column[Timestamp]("SUBMISSION_TIME")
 
-    def importsZipFile = column[Option[Blob]]("IMPORTS_ZIP")
+    def importsZip = column[Option[Blob]]("IMPORTS_ZIP")
 
     override def * = (workflowExecutionUuid, workflowDefinition, workflowInputs, workflowOptions, workflowState,
-      submissionTime, importsZipFile, customLabels, workflowStoreEntryId.?) <> (WorkflowStoreEntry.tupled, WorkflowStoreEntry.unapply)
+      submissionTime, importsZip, customLabels, workflowStoreEntryId.?) <> (WorkflowStoreEntry.tupled, WorkflowStoreEntry.unapply)
 
     def ucWorkflowStoreEntryWeu = index("UC_WORKFLOW_STORE_ENTRY_WEU", workflowExecutionUuid, unique = true)
 
diff --git a/database/sql/src/main/scala/cromwell/database/sql/SqlConverters.scala b/database/sql/src/main/scala/cromwell/database/sql/SqlConverters.scala
index e1756188b..3498e3da8 100644
--- a/database/sql/src/main/scala/cromwell/database/sql/SqlConverters.scala
+++ b/database/sql/src/main/scala/cromwell/database/sql/SqlConverters.scala
@@ -1,8 +1,8 @@
 package cromwell.database.sql
 
-import java.sql.{Clob, Timestamp}
+import java.sql.{Blob, Clob, Timestamp}
 import java.time.{OffsetDateTime, ZoneId}
-import javax.sql.rowset.serial.SerialClob
+import javax.sql.rowset.serial.{SerialBlob, SerialClob}
 
 object SqlConverters {
 
@@ -20,12 +20,19 @@ object SqlConverters {
 
   implicit class ClobToRawString(val clob: Clob) extends AnyVal {
     def toRawString: String = clob.getSubString(1, clob.length.toInt) // yes, it starts at 1
+
+    def parseSystemTimestamp: Timestamp = OffsetDateTime.parse(toRawString).toSystemTimestamp
   }
 
   implicit class StringToClob(val str: String) extends AnyVal {
     def toClob: Clob = new SerialClob(str.toCharArray)
+  }
 
-    def toNonEmptyClob: Option[Clob] = if (str.isEmpty) None else Option(new SerialClob(str.toCharArray))
+  implicit class BlobToBytes(val blob: Blob) extends AnyVal {
+    def toBytes: Array[Byte] = blob.getBytes(1, blob.length.toInt) // yes, it starts at 1
   }
 
+  implicit class StringToBlob(val bytes: Array[Byte]) extends AnyVal {
+    def toBlob: Blob = new SerialBlob(bytes)
+  }
 }
diff --git a/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingDetritusEntry.scala b/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingDetritusEntry.scala
index 9e325d5c1..acbaeb876 100644
--- a/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingDetritusEntry.scala
+++ b/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingDetritusEntry.scala
@@ -7,5 +7,5 @@ case class CallCachingDetritusEntry
   detritusKey: String,
   detritusValue: Clob,
   callCachingEntryId: Option[Int] = None,
-  callCachingDetritusId: Option[Int] = None
+  callCachingDetritusEntryId: Option[Int] = None
 )
diff --git a/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingSimpletonEntry.scala b/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingSimpletonEntry.scala
index ea398cad4..148f45f6a 100644
--- a/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingSimpletonEntry.scala
+++ b/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingSimpletonEntry.scala
@@ -1,9 +1,11 @@
 package cromwell.database.sql.tables
 
+import java.sql.Clob
+
 case class CallCachingSimpletonEntry
 (
   simpletonKey: String,
-  simpletonValue: String,
+  simpletonValue: Clob,
   wdlType: String,
   callCachingEntryId: Option[Int] = None,
   callCachingSimpletonEntryId: Option[Int] = None
diff --git a/database/sql/src/main/scala/cromwell/database/sql/tables/JobStoreEntry.scala b/database/sql/src/main/scala/cromwell/database/sql/tables/JobStoreEntry.scala
index ee8a76699..bf819e632 100644
--- a/database/sql/src/main/scala/cromwell/database/sql/tables/JobStoreEntry.scala
+++ b/database/sql/src/main/scala/cromwell/database/sql/tables/JobStoreEntry.scala
@@ -1,5 +1,7 @@
 package cromwell.database.sql.tables
 
+import java.sql.Clob
+
 case class JobStoreEntry
 (
   workflowExecutionUuid: String,
@@ -8,7 +10,7 @@ case class JobStoreEntry
   jobAttempt: Int,
   jobSuccessful: Boolean,
   returnCode: Option[Int],
-  exceptionMessage: Option[String],
+  exceptionMessage: Option[Clob],
   retryableFailure: Option[Boolean],
   jobStoreEntryId: Option[Int] = None
 )
diff --git a/database/sql/src/main/scala/cromwell/database/sql/tables/JobStoreSimpletonEntry.scala b/database/sql/src/main/scala/cromwell/database/sql/tables/JobStoreSimpletonEntry.scala
index 804a6ff6d..7989ed817 100644
--- a/database/sql/src/main/scala/cromwell/database/sql/tables/JobStoreSimpletonEntry.scala
+++ b/database/sql/src/main/scala/cromwell/database/sql/tables/JobStoreSimpletonEntry.scala
@@ -1,9 +1,11 @@
 package cromwell.database.sql.tables
 
+import java.sql.Clob
+
 case class JobStoreSimpletonEntry
 (
   simpletonKey: String,
-  simpletonValue: String,
+  simpletonValue: Clob,
   wdlType: String,
   jobStoreEntryId: Option[Int] = None,
   jobStoreSimpletonEntryId: Option[Int] = None
diff --git a/database/sql/src/main/scala/cromwell/database/sql/tables/MetadataEntry.scala b/database/sql/src/main/scala/cromwell/database/sql/tables/MetadataEntry.scala
index 7578ff18e..fcc4a4000 100644
--- a/database/sql/src/main/scala/cromwell/database/sql/tables/MetadataEntry.scala
+++ b/database/sql/src/main/scala/cromwell/database/sql/tables/MetadataEntry.scala
@@ -1,6 +1,6 @@
 package cromwell.database.sql.tables
 
-import java.sql.Timestamp
+import java.sql.{Clob, Timestamp}
 
 case class MetadataEntry
 (
@@ -9,7 +9,7 @@ case class MetadataEntry
   jobIndex: Option[Int],
   jobAttempt: Option[Int],
   metadataKey: String,
-  metadataValue: Option[String],
+  metadataValue: Option[Clob],
   metadataValueType: Option[String],
   metadataTimestamp: Timestamp,
   metadataEntryId: Option[Long] = None
diff --git a/database/sql/src/main/scala/cromwell/database/sql/tables/SubWorkflowStoreEntry.scala b/database/sql/src/main/scala/cromwell/database/sql/tables/SubWorkflowStoreEntry.scala
index 2e718179a..96ac39aa6 100644
--- a/database/sql/src/main/scala/cromwell/database/sql/tables/SubWorkflowStoreEntry.scala
+++ b/database/sql/src/main/scala/cromwell/database/sql/tables/SubWorkflowStoreEntry.scala
@@ -5,8 +5,8 @@ case class SubWorkflowStoreEntry
   rootWorkflowId: Option[Int],
   parentWorkflowExecutionUuid: String,
   callFullyQualifiedName: String,
-  jobIndex: Int,
-  jobAttempt: Int,
+  callIndex: Int,
+  callAttempt: Int,
   subWorkflowExecutionUuid: String,
   subWorkflowStoreEntryId: Option[Int] = None
 )
diff --git a/database/sql/src/main/scala/cromwell/database/sql/tables/WorkflowStoreEntry.scala b/database/sql/src/main/scala/cromwell/database/sql/tables/WorkflowStoreEntry.scala
index fe9387582..a6e2403bb 100644
--- a/database/sql/src/main/scala/cromwell/database/sql/tables/WorkflowStoreEntry.scala
+++ b/database/sql/src/main/scala/cromwell/database/sql/tables/WorkflowStoreEntry.scala
@@ -10,7 +10,7 @@ case class WorkflowStoreEntry
   workflowOptions: Clob,
   workflowState: String,
   submissionTime: Timestamp,
-  importsZipFile: Option[Blob],
+  importsZip: Option[Blob],
   customLabels: Clob,
   workflowStoreEntryId: Option[Int] = None
 )
diff --git a/engine/src/main/scala/cromwell/Simpletons.scala b/engine/src/main/scala/cromwell/Simpletons.scala
index 95983bb32..1d30e6c6b 100644
--- a/engine/src/main/scala/cromwell/Simpletons.scala
+++ b/engine/src/main/scala/cromwell/Simpletons.scala
@@ -1,6 +1,7 @@
 package cromwell
 
 import cromwell.core.simpleton.WdlValueSimpleton
+import cromwell.database.sql.SqlConverters._
 import cromwell.database.sql.tables.{CallCachingSimpletonEntry, JobStoreSimpletonEntry}
 import wdl4s.types.{WdlBooleanType, WdlFloatType, WdlIntegerType, WdlStringType}
 import wdl4s.values.{WdlPrimitive, WdlSingleFile, WdlValue}
@@ -12,11 +13,11 @@ import scala.util.Try
   */
 object Simpletons {
   def toSimpleton(entry: CallCachingSimpletonEntry): WdlValueSimpleton = {
-    toSimpleton(entry.wdlType, entry.simpletonKey, entry.simpletonValue)
+    toSimpleton(entry.wdlType, entry.simpletonKey, entry.simpletonValue.toRawString)
   }
 
   def toSimpleton(entry: JobStoreSimpletonEntry): WdlValueSimpleton = {
-    toSimpleton(entry.wdlType, entry.simpletonKey, entry.simpletonValue)
+    toSimpleton(entry.wdlType, entry.simpletonKey, entry.simpletonValue.toRawString)
   }
 
   private def toSimpleton(wdlType: String, simpletonKey: String, simpletonValue: String): WdlValueSimpleton = {
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala
index 9a149155e..95cf3cbb8 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala
@@ -7,8 +7,8 @@ import cromwell.backend.BackendJobExecutionActor.JobSucceededResponse
 import cromwell.core.ExecutionIndex.IndexEnhancedIndex
 import cromwell.core.WorkflowId
 import cromwell.core.callcaching.HashResult
-import cromwell.core.simpleton.WdlValueSimpleton
 import cromwell.core.path.PathImplicits._
+import cromwell.core.simpleton.WdlValueSimpleton
 import cromwell.database.sql.SqlConverters._
 import cromwell.database.sql._
 import cromwell.database.sql.joins.CallCachingJoin
@@ -49,7 +49,7 @@ class CallCache(database: CallCachingSqlDatabase) {
     val resultToInsert: Iterable[CallCachingSimpletonEntry] = {
       result map {
         case WdlValueSimpleton(simpletonKey, wdlPrimitive) =>
-          CallCachingSimpletonEntry(simpletonKey, wdlPrimitive.valueString, wdlPrimitive.wdlType.toWdlString)
+          CallCachingSimpletonEntry(simpletonKey, wdlPrimitive.valueString.toClob, wdlPrimitive.wdlType.toWdlString)
       }
     }
 
diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStore.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStore.scala
index d527378e3..afab96748 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStore.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStore.scala
@@ -1,16 +1,15 @@
 package cromwell.engine.workflow.workflowstore
 
 import java.time.OffsetDateTime
-import javax.sql.rowset.serial.SerialBlob
 
 import cats.data.NonEmptyList
 import com.typesafe.config.ConfigFactory
-import net.ceedubs.ficus.Ficus._
 import cromwell.core.{WorkflowId, WorkflowSourceFilesCollection}
 import cromwell.database.sql.SqlConverters._
 import cromwell.database.sql.WorkflowStoreSqlDatabase
 import cromwell.database.sql.tables.WorkflowStoreEntry
 import cromwell.engine.workflow.workflowstore.WorkflowStoreState.StartableState
+import net.ceedubs.ficus.Ficus._
 
 import scala.concurrent.{ExecutionContext, Future}
 
@@ -58,7 +57,7 @@ case class SqlWorkflowStore(sqlDatabase: WorkflowStoreSqlDatabase) extends Workf
       workflowStoreEntry.workflowInputs.toRawString,
       workflowStoreEntry.workflowOptions.toRawString,
       workflowStoreEntry.customLabels.toRawString,
-      workflowStoreEntry.importsZipFile.map(b => b.getBytes(1, b.length.asInstanceOf[Int]))
+      workflowStoreEntry.importsZip.map(_.toBytes)
     )
     WorkflowToStart(
       WorkflowId.fromString(workflowStoreEntry.workflowExecutionUuid),
@@ -75,7 +74,7 @@ case class SqlWorkflowStore(sqlDatabase: WorkflowStoreSqlDatabase) extends Workf
       customLabels = workflowSourceFiles.labelsJson.toClob,
       workflowState = WorkflowStoreState.Submitted.toString,
       submissionTime = OffsetDateTime.now.toSystemTimestamp,
-      importsZipFile = workflowSourceFiles.importsZipFileOption.map(new SerialBlob(_))
+      importsZip = workflowSourceFiles.importsZipFileOption.map(_.toBlob)
     )
   }
 
diff --git a/engine/src/main/scala/cromwell/jobstore/SqlJobStore.scala b/engine/src/main/scala/cromwell/jobstore/SqlJobStore.scala
index 085f30e06..d510c3873 100644
--- a/engine/src/main/scala/cromwell/jobstore/SqlJobStore.scala
+++ b/engine/src/main/scala/cromwell/jobstore/SqlJobStore.scala
@@ -12,6 +12,7 @@ import cromwell.database.sql.tables.{JobStoreEntry, JobStoreSimpletonEntry}
 import wdl4s.TaskOutput
 
 import scala.concurrent.{ExecutionContext, Future}
+import cromwell.database.sql.SqlConverters._
 
 class SqlJobStore(sqlDatabase: JobStoreSqlDatabase) extends JobStore {
   override def writeToDatabase(jobCompletions: Map[JobStoreKey, JobResult], workflowCompletions: List[WorkflowId])(implicit ec: ExecutionContext): Future[Unit] = {
@@ -36,7 +37,7 @@ class SqlJobStore(sqlDatabase: JobStoreSqlDatabase) extends JobStore {
         val jobStoreResultSimpletons =
           jobOutputs.mapValues(_.wdlValue).simplify.map {
             wdlValueSimpleton => JobStoreSimpletonEntry(
-              wdlValueSimpleton.simpletonKey, wdlValueSimpleton.simpletonValue.valueString,
+              wdlValueSimpleton.simpletonKey, wdlValueSimpleton.simpletonValue.valueString.toClob,
               wdlValueSimpleton.simpletonValue.wdlType.toWdlString)
           }
         JobStoreJoin(entry, jobStoreResultSimpletons.toSeq)
@@ -48,7 +49,7 @@ class SqlJobStore(sqlDatabase: JobStoreSqlDatabase) extends JobStore {
           key.attempt,
           jobSuccessful = false,
           returnCode,
-          Option(throwable.getMessage),
+          Option(throwable.getMessage.toClob),
           Option(retryable))
         JobStoreJoin(entry, Seq.empty)
     }
@@ -64,7 +65,8 @@ class SqlJobStore(sqlDatabase: JobStoreSqlDatabase) extends JobStore {
             val jobOutputs = WdlValueBuilder.toJobOutputs(taskOutputs, simpletons)
             JobResultSuccess(returnCode, jobOutputs)
           case JobStoreEntry(_, _, _, _, false, returnCode, Some(exceptionMessage), Some(retryable), _) =>
-            JobResultFailure(returnCode, JobAlreadyFailedInJobStore(jobStoreKey.tag, exceptionMessage), retryable)
+            JobResultFailure(returnCode, JobAlreadyFailedInJobStore(jobStoreKey.tag, exceptionMessage.toRawString),
+              retryable)
           case bad =>
             throw new Exception(s"Invalid contents of JobStore table: $bad")
         }
diff --git a/services/src/main/scala/cromwell/services/metadata/impl/MetadataDatabaseAccess.scala b/services/src/main/scala/cromwell/services/metadata/impl/MetadataDatabaseAccess.scala
index 6736b221d..18e8689de 100644
--- a/services/src/main/scala/cromwell/services/metadata/impl/MetadataDatabaseAccess.scala
+++ b/services/src/main/scala/cromwell/services/metadata/impl/MetadataDatabaseAccess.scala
@@ -1,7 +1,5 @@
 package cromwell.services.metadata.impl
 
-import java.time.OffsetDateTime
-
 import cats.Semigroup
 import cats.data.NonEmptyList
 import cats.syntax.semigroup._
@@ -43,12 +41,12 @@ object MetadataDatabaseAccess {
     def toSummary: WorkflowMetadataSummaryEntry = {
       val base = baseSummary(metadatum.workflowExecutionUuid)
       metadatum.metadataKey match {
-        case WorkflowMetadataKeys.Name => base.copy(workflowName = metadatum.metadataValue)
-        case WorkflowMetadataKeys.Status => base.copy(workflowStatus = metadatum.metadataValue)
+        case WorkflowMetadataKeys.Name => base.copy(workflowName = metadatum.metadataValue.map(_.toRawString))
+        case WorkflowMetadataKeys.Status => base.copy(workflowStatus = metadatum.metadataValue.map(_.toRawString))
         case WorkflowMetadataKeys.StartTime =>
-          base.copy(startTimestamp = metadatum.metadataValue map OffsetDateTime.parse map { _.toSystemTimestamp })
+          base.copy(startTimestamp = metadatum.metadataValue.map(_.parseSystemTimestamp))
         case WorkflowMetadataKeys.EndTime =>
-          base.copy(endTimestamp = metadatum.metadataValue map OffsetDateTime.parse map { _.toSystemTimestamp })
+          base.copy(endTimestamp = metadatum.metadataValue.map(_.parseSystemTimestamp))
       }
     }
   }
@@ -78,7 +76,8 @@ trait MetadataDatabaseAccess {
       val value = metadataEvent.value map { _.value }
       val valueType = metadataEvent.value map { _.valueType.typeName }
       val jobKey = key.jobKey map { jk => (jk.callFqn, jk.index, jk.attempt) }
-      MetadataEntry(workflowUuid, jobKey.map(_._1), jobKey.flatMap(_._2), jobKey.map(_._3), key.key, value, valueType, timestamp)
+      MetadataEntry(workflowUuid, jobKey.map(_._1), jobKey.flatMap(_._2), jobKey.map(_._3),
+        key.key, value.map(_.toClob), valueType, timestamp)
     }
     databaseInterface.addMetadataEntries(metadata)
   }
@@ -95,7 +94,7 @@ trait MetadataDatabaseAccess {
       val value =  for {
         mValue <- m.metadataValue
         mType <- m.metadataValueType
-      } yield MetadataValue(mValue, MetadataType.fromString(mType))
+      } yield MetadataValue(mValue.toRawString, MetadataType.fromString(mType))
 
       MetadataEvent(key, value, m.metadataTimestamp.toSystemOffsetDateTime)
     }
diff --git a/services/src/test/scala/cromwell/services/ServicesStoreSpec.scala b/services/src/test/scala/cromwell/services/ServicesStoreSpec.scala
index 7558a3988..5a44d3672 100644
--- a/services/src/test/scala/cromwell/services/ServicesStoreSpec.scala
+++ b/services/src/test/scala/cromwell/services/ServicesStoreSpec.scala
@@ -6,107 +6,214 @@ import java.sql.Connection
 import better.files._
 import com.typesafe.config.ConfigFactory
 import cromwell.core.Tags._
+import cromwell.core.WorkflowId
 import cromwell.database.migration.liquibase.LiquibaseUtils
 import cromwell.database.slick.SlickDatabase
+import cromwell.database.sql.joins.JobStoreJoin
+import cromwell.database.sql.tables.JobStoreEntry
 import liquibase.diff.DiffResult
 import liquibase.diff.output.DiffOutputControl
 import liquibase.diff.output.changelog.DiffToChangeLog
 import org.hsqldb.persist.HsqlDatabaseProperties
 import org.scalactic.StringNormalizations
+import org.scalatest.concurrent.PatienceConfiguration.Timeout
 import org.scalatest.concurrent.ScalaFutures
 import org.scalatest.time.{Millis, Seconds, Span}
 import org.scalatest.{FlatSpec, Matchers}
 import slick.driver.JdbcProfile
+import slick.jdbc.meta._
 
-import scala.concurrent.duration.Duration
+import scala.concurrent.duration._
 import scala.concurrent.{Await, ExecutionContext, Future}
+import scala.util.Try
 import scala.xml._
 
 class ServicesStoreSpec extends FlatSpec with Matchers with ScalaFutures with StringNormalizations {
 
-  behavior of "ServicesStore"
-
   import ServicesStoreSpec._
 
   implicit val ec = ExecutionContext.global
 
   implicit val defaultPatience = PatienceConfig(timeout = Span(5, Seconds), interval = Span(100, Millis))
 
-  it should "have the same liquibase and slick schema" in {
+  behavior of "ServicesStore"
+
+  it should "not deadlock" in {
+    // Test based on https://github.com/kwark/slick-deadlock/blob/82525fc/src/main/scala/SlickDeadlock.scala
+    val databaseConfig = ConfigFactory.parseString(
+      s"""|db.url = "jdbc:hsqldb:mem:$${uniqueSchema};shutdown=false;hsqldb.tx=mvcc"
+          |db.driver = "org.hsqldb.jdbcDriver"
+          |db.connectionTimeout = 3000
+          |db.numThreads = 2
+          |driver = "slick.driver.HsqldbDriver$$"
+          |""".stripMargin)
+    import ServicesStore.EnhancedSqlDatabase
     for {
-      liquibaseDatabase <- databaseForSchemaManager("liquibase").autoClosed
-      slickDatabase <- databaseForSchemaManager("slick").autoClosed
+      database <- new SlickDatabase(databaseConfig).initialized.autoClosed
     } {
-      compare(
-        liquibaseDatabase.dataAccess.driver, liquibaseDatabase.database,
-        slickDatabase.dataAccess.driver, slickDatabase.database) { diffResult =>
-
-        // TODO PBE get rid of this after the migration of #789 has run.
-        val oldeTables = Seq(
-          "EXECUTION",
-          "EXECUTION_INFO",
-          "EXECUTION_EVENT",
-          "FAILURE_EVENT",
-          "RUNTIME_ATTRIBUTES",
-          "SYMBOL",
-          "WORKFLOW_EXECUTION",
-          "WORKFLOW_EXECUTION_AUX"
-        )
-
-        import cromwell.database.migration.liquibase.DiffResultFilter._
-        val diffFilters = StandardTypeFilters :+ UniqueIndexFilter
-        val filteredDiffResult = diffResult
-          .filterLiquibaseObjects
-          .filterTableObjects(oldeTables)
-          .filterChangedObjects(diffFilters)
-
-        val totalChanged =
-          filteredDiffResult.getChangedObjects.size +
-            filteredDiffResult.getMissingObjects.size +
-            filteredDiffResult.getUnexpectedObjects.size
-
-        if (totalChanged > 0) {
-          val outputStream = new ByteArrayOutputStream
-          val printStream = new PrintStream(outputStream, true)
-          val diffOutputControl = new DiffOutputControl(false, false, false, Array.empty)
-          val diffToChangeLog = new DiffToChangeLog(filteredDiffResult, diffOutputControl)
-          diffToChangeLog.print(printStream)
-          val changeSetsScoped = XML.loadString(outputStream.toString) \ "changeSet" \ "_"
-          val changeSets = changeSetsScoped map stripNodeScope
-          fail(changeSets.mkString(
-            "The following changes are in liquibase but not in slick:\n  ",
-            "\n  ",
-            "\nEither add the changes to slick or remove them from liquibase."))
-        }
+      val futures = 1 to 20 map { _ =>
+        val workflowUuid = WorkflowId.randomId().toString
+        val callFqn = "call.fqn"
+        val jobIndex = 1
+        val jobAttempt = 1
+        val jobSuccessful = false
+        val jobStoreEntry = JobStoreEntry(workflowUuid, callFqn, jobIndex, jobAttempt, jobSuccessful, None, None, None)
+        val jobStoreJoins = Seq(JobStoreJoin(jobStoreEntry, Seq()))
+        // NOTE: This test just needs to repeatedly read/write from a table that acts as a PK for a FK.
+        for {
+          _ <- database.addJobStores(jobStoreJoins)
+          queried <- database.queryJobStores(workflowUuid, callFqn, jobIndex, jobAttempt)
+          _ = queried.get.jobStoreEntry.workflowExecutionUuid should be(workflowUuid)
+        } yield ()
       }
+      Future.sequence(futures).futureValue(Timeout(10.seconds))
     }
   }
 
-  it should "not deadlock" taggedAs PostMVP ignore {
-    //    // Test based on https://github.com/kwark/slick-deadlock/blob/82525fc/src/main/scala/SlickDeadlock.scala
-    //    val databaseConfig = ConfigFactory.parseString(
-    //      s"""
-    //         |db.url = "jdbc:hsqldb:mem:$${slick.uniqueSchema};shutdown=false;hsqldb.tx=mvcc"
-    //         |db.driver = "org.hsqldb.jdbcDriver"
-    //         |db.numThreads = 2
-    //         |driver = "slick.driver.HsqldbDriver$$"
-    //         |""".stripMargin)
-    //
-    //    for {
-    //      dataAccess <- (new SlickDatabase(databaseConfig) with DataAccess).autoClosed
-    //    } {
-    //      val futures = 1 to 20 map { _ =>
-    //        val workflowId = WorkflowId.randomId()
-    //        val workflowInfo = createMaterializedEngineWorkflowDescriptor(id = workflowId, workflowSources = test1Sources)
-    //        for {
-    //          _ <- dataAccess.createWorkflow(workflowInfo, test1Sources, Nil, Nil, localBackend)
-    //          _ <- dataAccess.getWorkflowExecutionAndAux(workflowInfo.id) map { result =>
-    //            result.execution.workflowExecutionUuid should be(workflowId.toString)
-    //          }
-    //        } yield ()
-    //      }
-    //      Future.sequence(futures).futureValue(Timeout(10.seconds))
-    //    }
+  "Slick" should behave like testSchemaManager("slick")
+
+  "Liquibase" should behave like testSchemaManager("liquibase")
+
+  def testSchemaManager(schemaManager: String): Unit = {
+    val otherSchemaManager = if (schemaManager == "slick") "liquibase" else "slick"
+
+    it should s"have the same schema as $otherSchemaManager" in {
+      for {
+        actualDatabase <- databaseForSchemaManager(schemaManager).autoClosed
+        expectedDatabase <- databaseForSchemaManager(otherSchemaManager).autoClosed
+      } {
+        compare(
+          actualDatabase.dataAccess.driver, actualDatabase.database,
+          expectedDatabase.dataAccess.driver, expectedDatabase.database) { diffResult =>
+
+          import cromwell.database.migration.liquibase.DiffResultFilter._
+
+          /*
+          NOTE: Unique indexes no longer need to be filtered, as WE SHOULD NOT BE USING THEM!
+          See notes at the bottom of changelog.xml
+           */
+          val diffFilters = StandardTypeFilters
+          val filteredDiffResult = diffResult
+            .filterLiquibaseObjects
+            .filterTableObjects(oldeTables)
+            .filterChangedObjects(diffFilters)
+
+          val totalChanged =
+            filteredDiffResult.getChangedObjects.size +
+              filteredDiffResult.getMissingObjects.size +
+              filteredDiffResult.getUnexpectedObjects.size
+
+          if (totalChanged > 0) {
+            val outputStream = new ByteArrayOutputStream
+            val printStream = new PrintStream(outputStream, true)
+            val diffOutputControl = new DiffOutputControl(false, false, false, Array.empty)
+            val diffToChangeLog = new DiffToChangeLog(filteredDiffResult, diffOutputControl)
+            diffToChangeLog.print(printStream)
+            val changeSetsScoped = XML.loadString(outputStream.toString) \ "changeSet" \ "_"
+            val changeSets = changeSetsScoped map stripNodeScope
+            fail(changeSets.mkString(
+              s"The following changes are in $schemaManager but not in $otherSchemaManager:\n  ",
+              "\n  ",
+              "\nEnsure that the columns/fields exist, with the same lengths in " +
+                s"$schemaManager and $otherSchemaManager and synchronize the two."))
+          }
+        }
+      }
+    }
+
+    it should "match expected generated names" in {
+      var schemaMetadata: SchemaMetadata = null
+
+      for {
+        slickDatabase <- databaseForSchemaManager(schemaManager).autoClosed
+      } {
+        import slickDatabase.dataAccess.driver.api._
+        val schemaMetadataFuture =
+          for {
+            tables <- slickDatabase.database.run(MTable.getTables(Option("PUBLIC"), Option("PUBLIC"), None, None))
+            workingTables = tables
+              .filterNot(_.name.name.startsWith("DATABASECHANGELOG"))
+              .filterNot(table => oldeTables.contains(table.name.name))
+              // NOTE: MetadataEntry column names are perma-busted due to the large size of the table.
+              .filterNot(_.name.name == "METADATA_ENTRY")
+            columns <- slickDatabase.database.run(DBIO.sequence(workingTables.map(_.getColumns)))
+            indexes <- slickDatabase.database.run(DBIO.sequence(workingTables.map(_.getIndexInfo())))
+            primaryKeys <- slickDatabase.database.run(DBIO.sequence(workingTables.map(_.getPrimaryKeys)))
+            foreignKeys <- slickDatabase.database.run(DBIO.sequence(workingTables.map(_.getExportedKeys)))
+          } yield SchemaMetadata(tables, columns.flatten, indexes.flatten.filterNot(isGenerated),
+            primaryKeys.flatten.filterNot(isGenerated), foreignKeys.flatten)
+
+        schemaMetadata = schemaMetadataFuture.futureValue
+      }
+
+      var misnamed = Seq.empty[String]
+
+      schemaMetadata.primaryKeyMetadata foreach { primaryKey =>
+        val actual = primaryKey.pkName.get
+        val expected = s"PK_${primaryKey.table.name}"
+        if (actual != expected) {
+          misnamed :+=
+            s"""|PrimaryKey: $actual
+                |Should Be:  $expected
+                |""".stripMargin
+        }
+      }
+
+      schemaMetadata.foreignKeyMetadata foreach { foreignKey =>
+        val actual = foreignKey.fkName.get
+        val expected = s"FK_${foreignKey.fkTable.name}_${foreignKey.fkColumn}"
+        if (actual != expected) {
+          misnamed :+=
+            s"""|ForeignKey: $actual
+                |Should Be:  $expected
+                |""".stripMargin
+        }
+      }
+
+      schemaMetadata.indexMetadata.groupBy(getIndexName) foreach {
+        case (indexName, indexColumns) =>
+          val index = indexColumns.head
+          val prefix = if (index.nonUnique) "IX" else "UC"
+          val tableName = index.table.name
+          val sortedColumns = indexColumns.sortBy(_.ordinalPosition)
+          val abbrColumns = sortedColumns.map(indexColumn => snakeAbbreviate(indexColumn.column.get))
+
+          val actual = indexName
+          val expected = abbrColumns.mkString(s"${prefix}_${tableName}_", "_", "")
+
+          if (actual != expected) {
+            misnamed :+=
+              s"""|Index:     $actual
+                  |Should Be: $expected
+                  |""".stripMargin
+          }
+      }
+
+      if (misnamed.nonEmpty) {
+        fail(misnamed.mkString(s"The following items are misnamed in $schemaManager:\n", "\n", ""))
+      }
+
+      var missing = Seq.empty[String]
+
+      schemaMetadata.columns foreach { column =>
+        if (!schemaMetadata.existsTableItem(column)) {
+          missing :+= s"${tableClassName(column.tableName)}.${column.itemName}"
+        }
+      }
+
+      schemaMetadata.slickItems foreach { databaseItem =>
+        if (!schemaMetadata.existsSlickMapping(databaseItem)) {
+          missing :+= s"${slickClassName(databaseItem.tableName)}.${databaseItem.itemName}"
+        }
+      }
+
+      if (missing.nonEmpty) {
+        fail(missing.mkString(
+          s"Based on the schema in $schemaManager, please ensure that the following tables/columns exist:\n  ",
+          "\n  ",
+          ""))
+      }
+    }
   }
 
   "SlickDatabase (hsqldb)" should behave like testWith("database")
@@ -147,6 +254,18 @@ class ServicesStoreSpec extends FlatSpec with Matchers with ScalaFutures with St
 }
 
 object ServicesStoreSpec {
+  // TODO PBE get rid of this after the migration of #789 has run.
+  private val oldeTables = Seq(
+    "EXECUTION",
+    "EXECUTION_EVENT",
+    "EXECUTION_INFO",
+    "FAILURE_EVENT",
+    "RUNTIME_ATTRIBUTES",
+    "SYMBOL",
+    "WORKFLOW_EXECUTION",
+    "WORKFLOW_EXECUTION_AUX"
+  )
+
   // strip the namespace from elems and their children
   private def stripNodeScope(node: Node): Node = {
     node match {
@@ -228,4 +347,92 @@ object ServicesStoreSpec {
       }
     }
   }
+
+  private val SnakeRegex = "_([a-z])".r
+
+  private def snakeToCamel(value: String): String = {
+    SnakeRegex.replaceAllIn(value.toLowerCase, _.group(1).toUpperCase)
+  }
+
+  private def snakeAbbreviate(value: String): String = {
+    SnakeRegex.findAllMatchIn("_" + value.toLowerCase).map(_.group(1)).mkString("").toUpperCase
+  }
+
+  private val SlickPrimaryKeyRegex = """SYS_PK_\d+""".r
+
+  private def isGenerated(primaryKey: MPrimaryKey): Boolean = {
+    primaryKey.pkName.get match {
+      case SlickPrimaryKeyRegex(_*) => true
+      case _ => false
+    }
+  }
+
+  private val LiquibasePrimaryKeyIndexRegex = """SYS_IDX_PK_[A-Z_]+_\d+""".r
+  private val SlickPrimaryKeyIndexRegex = """SYS_IDX_SYS_PK_\d+_\d+""".r
+  private val SlickForeignKeyIndexRegex = """SYS_IDX_\d+""".r
+
+  private def isGenerated(index: MIndexInfo): Boolean = {
+    index.indexName.get match {
+      case LiquibasePrimaryKeyIndexRegex(_*) => true
+      case SlickPrimaryKeyIndexRegex(_*) => true
+      case SlickForeignKeyIndexRegex(_*) => true
+      case _ => false
+    }
+  }
+
+  private def tableClassName(tableName: String) = s"cromwell.database.sql.tables.$tableName"
+
+  private def slickClassName(tableName: String) =
+    s"cromwell.database.slick.tables.${tableName}Component$$${tableName.replace("Entry", "Entries")}"
+
+  private def getIndexName(index: MIndexInfo) = index.indexName.get.replaceAll("(^SYS_IDX_|_\\d+$)", "")
+
+  case class TableClass(tableName: String) {
+    private def getClass(name: String): Try[Class[_]] = Try(Class.forName(name))
+
+    private lazy val tableColumns = getClass(tableClassName(tableName)).map(_.getDeclaredFields).getOrElse(Array.empty)
+    private lazy val slickMapping = getClass(slickClassName(tableName)).map(_.getDeclaredMethods).getOrElse(Array.empty)
+
+    def existsTableField(name: String): Boolean = tableColumns.exists(_.getName == name)
+
+    def existsSlickMapping(name: String): Boolean = slickMapping.exists(_.getName == name)
+  }
+
+  case class DatabaseItem(tableName: String, itemName: String)
+
+  case class SchemaMetadata(tableMetadata: Seq[MTable], columnMetadata: Seq[MColumn], indexMetadata: Seq[MIndexInfo],
+                            primaryKeyMetadata: Seq[MPrimaryKey], foreignKeyMetadata: Seq[MForeignKey]) {
+    lazy val tables: Seq[TableClass] = tableMetadata.map({ table =>
+      val tableName = snakeToCamel(table.name.name).capitalize
+      TableClass(tableName)
+    }).distinct
+
+    lazy val columns: Seq[DatabaseItem] = columnMetadata.map({ column =>
+      val tableName = snakeToCamel(column.table.name).capitalize
+      val columnName = snakeToCamel(column.name)
+      DatabaseItem(tableName, columnName)
+    }).distinct
+
+    lazy val indexes: Seq[DatabaseItem] = indexMetadata.map({ index =>
+      val tableName = snakeToCamel(index.table.name).capitalize
+      val indexName = snakeToCamel(getIndexName(index))
+      DatabaseItem(tableName, indexName)
+    }).distinct
+
+    lazy val foreignKeys: Seq[DatabaseItem] = foreignKeyMetadata.map({ foreignKey =>
+      val tableName = snakeToCamel(foreignKey.fkTable.name).capitalize
+      val indexName = snakeToCamel(foreignKey.fkName.get)
+      DatabaseItem(tableName, indexName)
+    }).distinct
+
+    lazy val slickItems: Seq[DatabaseItem] = columns ++ indexes ++ foreignKeys
+
+    def existsTableItem(tableItem: DatabaseItem): Boolean = {
+      tables.find(_.tableName == tableItem.tableName).exists(_.existsTableField(tableItem.itemName))
+    }
+
+    def existsSlickMapping(tableItem: DatabaseItem): Boolean = {
+      tables.find(_.tableName == tableItem.tableName).exists(_.existsSlickMapping(tableItem.itemName))
+    }
+  }
 }
diff --git a/services/src/test/scala/cromwell/services/metadata/impl/MetadataServiceActorSpec.scala b/services/src/test/scala/cromwell/services/metadata/impl/MetadataServiceActorSpec.scala
index 46829d71a..a08dfb9fa 100644
--- a/services/src/test/scala/cromwell/services/metadata/impl/MetadataServiceActorSpec.scala
+++ b/services/src/test/scala/cromwell/services/metadata/impl/MetadataServiceActorSpec.scala
@@ -8,6 +8,10 @@ import cromwell.core.WorkflowId
 import cromwell.services.ServicesSpec
 import cromwell.services.metadata.MetadataService._
 import cromwell.services.metadata._
+import org.scalatest.concurrent.Eventually._
+import org.scalatest.concurrent.PatienceConfiguration.{Interval, Timeout}
+
+import scala.concurrent.duration._
 
 class MetadataServiceActorSpec extends ServicesSpec("Metadata") {
   import MetadataServiceActorSpec.Config
@@ -47,23 +51,25 @@ class MetadataServiceActorSpec extends ServicesSpec("Metadata") {
       val query4 = MetadataQuery.forWorkflow(workflowId)
       val query5 = MetadataQuery.forJob(workflowId, supJob)
 
-      (for {
-        response1 <- (actor ? GetMetadataQueryAction(query1)).mapTo[MetadataServiceResponse]
-        _ = response1 shouldBe MetadataLookupResponse(query1, Seq(event1_1, event1_2))
+      eventually(Timeout(10.seconds), Interval(2.seconds)) {
+        (for {
+          response1 <- (actor ? GetMetadataQueryAction(query1)).mapTo[MetadataServiceResponse]
+          _ = response1 shouldBe MetadataLookupResponse(query1, Seq(event1_1, event1_2))
 
-        response2 <- (actor ? GetMetadataQueryAction(query2)).mapTo[MetadataServiceResponse]
-        _ = response2 shouldBe MetadataLookupResponse(query2, Seq(event2_1))
+          response2 <- (actor ? GetMetadataQueryAction(query2)).mapTo[MetadataServiceResponse]
+          _ = response2 shouldBe MetadataLookupResponse(query2, Seq(event2_1))
 
-        response3 <- (actor ? GetMetadataQueryAction(query3)).mapTo[MetadataServiceResponse]
-        _ = response3 shouldBe MetadataLookupResponse(query3, Seq(event3_1, event3_2))
+          response3 <- (actor ? GetMetadataQueryAction(query3)).mapTo[MetadataServiceResponse]
+          _ = response3 shouldBe MetadataLookupResponse(query3, Seq(event3_1, event3_2))
 
-        response4 <- (actor ? GetMetadataQueryAction(query4)).mapTo[MetadataServiceResponse]
-        _ = response4 shouldBe MetadataLookupResponse(query4, Seq(event1_1, event1_2, event2_1, event3_1, event3_2))
+          response4 <- (actor ? GetMetadataQueryAction(query4)).mapTo[MetadataServiceResponse]
+          _ = response4 shouldBe MetadataLookupResponse(query4, Seq(event1_1, event1_2, event2_1, event3_1, event3_2))
 
-        response5 <- (actor ? GetMetadataQueryAction(query5)).mapTo[MetadataServiceResponse]
-        _ = response5 shouldBe MetadataLookupResponse(query5, Seq(event3_1, event3_2))
+          response5 <- (actor ? GetMetadataQueryAction(query5)).mapTo[MetadataServiceResponse]
+          _ = response5 shouldBe MetadataLookupResponse(query5, Seq(event3_1, event3_2))
 
-      } yield ()).futureValue
+        } yield ()).futureValue
+      }
     }
   }
 }

From af7f8f10b9b5e9fc7b006e20a5666852f4ad9e30 Mon Sep 17 00:00:00 2001
From: Jeff Gentry <geoffjentry@gmail.com>
Date: Tue, 24 Jan 2017 11:22:26 -0500
Subject: [PATCH 31/84] Remove MetadataPutAcks & Failures. Closes #1811 (#1903)

---
 engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala  | 6 ------
 .../main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala  | 5 -----
 .../workflow/lifecycle/execution/SubWorkflowExecutionActor.scala    | 4 ----
 .../workflow/lifecycle/execution/WorkflowExecutionActor.scala       | 6 ------
 .../engine/workflow/workflowstore/WorkflowStoreEngineActor.scala    | 3 ---
 .../src/main/scala/cromwell/services/metadata/MetadataService.scala | 5 +----
 6 files changed, 1 insertion(+), 28 deletions(-)

diff --git a/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala b/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala
index 0fb49191d..eefa9ceeb 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala
@@ -18,7 +18,6 @@ import cromwell.engine.workflow.lifecycle.WorkflowInitializationActor.{StartInit
 import cromwell.engine.workflow.lifecycle._
 import cromwell.engine.workflow.lifecycle.execution.{WorkflowExecutionActor, WorkflowMetadataHelper}
 import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor._
-import cromwell.services.metadata.MetadataService._
 import cromwell.subworkflowstore.SubWorkflowStoreActor.WorkflowComplete
 import cromwell.webservice.EngineStatsActor
 import wdl4s.{LocallyQualifiedName => _}
@@ -259,11 +258,6 @@ class WorkflowActor(val workflowId: WorkflowId,
   when(WorkflowSucceededState) { FSM.NullFunction }
 
   whenUnhandled {
-    case Event(MetadataPutFailed(action, error), _) =>
-      // Do something useful here??
-      workflowLogger.warn(s"Put failed for Metadata action $action : ${error.getMessage}")
-      stay
-    case Event(MetadataPutAcknowledgement(_), _) => stay()
     case Event(AbortWorkflowCommand, WorkflowActorData(Some(actor), _, _, _)) =>
       actor ! EngineLifecycleActorAbortCommand
       goto(WorkflowAbortingState)
diff --git a/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala b/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala
index bc2019f40..9caf91409 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala
@@ -13,7 +13,6 @@ import cromwell.engine.workflow.WorkflowActor._
 import cromwell.engine.workflow.WorkflowManagerActor._
 import cromwell.engine.workflow.workflowstore.{WorkflowStoreActor, WorkflowStoreEngineActor, WorkflowStoreState}
 import cromwell.jobstore.JobStoreActor.{JobStoreWriteFailure, JobStoreWriteSuccess, RegisterWorkflowCompleted}
-import cromwell.services.metadata.MetadataService._
 import cromwell.webservice.EngineStatsActor
 import lenthall.exception.ThrowableAggregation
 import net.ceedubs.ficus.Ficus._
@@ -243,10 +242,6 @@ class WorkflowManagerActor(params: WorkflowManagerActorParams)
   when (Done) { FSM.NullFunction }
 
   whenUnhandled {
-    case Event(MetadataPutFailed(action, error), _) =>
-      log.warning(s"$tag Put failed for Metadata action $action : ${error.getMessage}")
-      stay()
-    case Event(MetadataPutAcknowledgement(_), _) => stay()
     // Uninteresting transition and current state notifications.
     case Event((Transition(_, _, _) | CurrentState(_, _)), _) => stay()
     case Event(JobStoreWriteSuccess(_), _) => stay() // Snoozefest
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/SubWorkflowExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/SubWorkflowExecutionActor.scala
index e3813dd61..1fcf670c8 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/SubWorkflowExecutionActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/SubWorkflowExecutionActor.scala
@@ -94,10 +94,6 @@ class SubWorkflowExecutionActor(key: SubWorkflowKey,
     case Event(SubWorkflowStoreFailure(command, reason), _) =>
       jobLogger.error(reason, s"SubWorkflowStore failure for command $command")
       stay()
-    case Event(MetadataPutFailed(action, error), _) =>
-      jobLogger.warn(s"Put failed for Metadata action $action", error)
-      stay()
-    case Event(MetadataPutAcknowledgement(_), _) => stay()
   }
 
   onTransition {
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala
index f4a4e4143..30504f2a1 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala
@@ -16,7 +16,6 @@ import cromwell.engine.backend.{BackendSingletonCollection, CromwellBackends}
 import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor.{apply => _, _}
 import cromwell.engine.workflow.lifecycle.{EngineLifecycleActorAbortCommand, EngineLifecycleActorAbortedResponse}
 import cromwell.engine.{ContinueWhilePossible, EngineWorkflowDescriptor}
-import cromwell.services.metadata.MetadataService.{MetadataPutAcknowledgement, MetadataPutFailed}
 import cromwell.util.StopAndLogSupervisor
 import cromwell.webservice.EngineStatsActor
 import lenthall.exception.ThrowableAggregation
@@ -191,11 +190,6 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor,
 
   whenUnhandled {
     case Event(Terminated(actorRef), stateData) => handleTerminated(actorRef) using stateData.removeEngineJobExecutionActor(actorRef)
-    case Event(MetadataPutFailed(action, error), _) =>
-      // Do something useful here??
-      workflowLogger.warn(s"$tag Put failed for Metadata action $action", error)
-      stay()
-    case Event(MetadataPutAcknowledgement(_), _) => stay()
     case Event(EngineLifecycleActorAbortCommand, stateData) =>
       if (stateData.hasRunningActors) {
         log.info(s"$tag: Abort received. " +
diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreEngineActor.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreEngineActor.scala
index 180138753..b35346581 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreEngineActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreEngineActor.scala
@@ -10,7 +10,6 @@ import cromwell.engine.workflow.workflowstore.WorkflowStoreState.StartableState
 import WorkflowStoreEngineActor._
 import cromwell.core.WorkflowId
 import cromwell.engine.workflow.workflowstore.WorkflowStoreActor._
-import cromwell.services.metadata.MetadataService.MetadataPutAcknowledgement
 import org.apache.commons.lang3.exception.ExceptionUtils
 
 import scala.concurrent.{ExecutionContext, Future}
@@ -54,8 +53,6 @@ final case class WorkflowStoreEngineActor(store: WorkflowStore, serviceRegistryA
   }
 
   whenUnhandled {
-    case Event(MetadataPutAcknowledgement(_), _) =>
-      stay // Ignored
     case Event(msg, _) =>
       log.warning("Unexpected message to WorkflowStoreActor in state {} with data {}: {}", stateName, stateData, msg)
       stay
diff --git a/services/src/main/scala/cromwell/services/metadata/MetadataService.scala b/services/src/main/scala/cromwell/services/metadata/MetadataService.scala
index cf589b2a2..332793d4b 100644
--- a/services/src/main/scala/cromwell/services/metadata/MetadataService.scala
+++ b/services/src/main/scala/cromwell/services/metadata/MetadataService.scala
@@ -2,7 +2,7 @@ package cromwell.services.metadata
 
 import java.time.OffsetDateTime
 
-import akka.actor.{ActorRef, DeadLetterSuppression}
+import akka.actor.ActorRef
 import cats.data.NonEmptyList
 import cromwell.core.{JobKey, WorkflowId, WorkflowState}
 import cromwell.services.ServiceRegistryActor.ServiceRegistryMessage
@@ -77,9 +77,6 @@ object MetadataService {
     def reason: Throwable
   }
 
-  case class MetadataPutAcknowledgement(putRequest: PutMetadataAction) extends MetadataServiceResponse with DeadLetterSuppression
-  case class MetadataPutFailed(putRequest: PutMetadataAction, reason: Throwable) extends MetadataServiceFailure
-
   case class MetadataLookupResponse(query: MetadataQuery, eventList: Seq[MetadataEvent]) extends MetadataServiceResponse
   case class MetadataServiceKeyLookupFailed(query: MetadataQuery, reason: Throwable) extends MetadataServiceFailure
 

From 8d8559c2e0562b7d2ec9e58e4dd12a46a8e0f3be Mon Sep 17 00:00:00 2001
From: Chris Llanwarne <cjllanwarne@gmail.com>
Date: Mon, 23 Jan 2017 17:40:07 -0500
Subject: [PATCH 32/84] Bringing in empty array fix in WDL4S

---
 CHANGELOG.md                                       |  1 +
 .../scala/cromwell/ArrayOfArrayCoercionSpec.scala  |  2 +-
 .../test/scala/cromwell/ArrayWorkflowSpec.scala    |  8 ++---
 .../scala/cromwell/CopyWorkflowOutputsSpec.scala   |  4 +--
 .../scala/cromwell/FilePassingWorkflowSpec.scala   | 12 +++----
 .../src/test/scala/cromwell/MapWorkflowSpec.scala  |  6 ++--
 .../MultipleFilesWithSameNameWorkflowSpec.scala    |  4 +--
 .../cromwell/PostfixQuantifierWorkflowSpec.scala   | 10 +++---
 .../test/scala/cromwell/ScatterWorkflowSpec.scala  | 38 +++++++++++-----------
 .../cromwell/WdlFunctionsAtWorkflowLevelSpec.scala |  4 +--
 .../test/scala/cromwell/WorkflowOutputsSpec.scala  | 16 ++++-----
 .../cromwell/engine/WorkflowManagerActorSpec.scala |  2 +-
 project/Dependencies.scala                         |  2 +-
 13 files changed, 55 insertions(+), 54 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 15f868eb7..6b5e59971 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,7 @@
 
 ## 25
 
+* Cromwell's WDL parser now recognizes empty array literals correctly, e.g. `Array[String] emptyArray = []`.
 * Moved the config value `services.MetadataService.metadata-summary-refresh-interval` to `services.MetadataService.config.metadata-summary-refresh-interval`
 * Cromwell now applies default labels automatically to JES pipeline runs.
 * Added ability to override the default zone(s) used by JES via the config structure by setting `genomics.default-zones` in the JES configuration
diff --git a/engine/src/test/scala/cromwell/ArrayOfArrayCoercionSpec.scala b/engine/src/test/scala/cromwell/ArrayOfArrayCoercionSpec.scala
index 537440935..e0315d92f 100644
--- a/engine/src/test/scala/cromwell/ArrayOfArrayCoercionSpec.scala
+++ b/engine/src/test/scala/cromwell/ArrayOfArrayCoercionSpec.scala
@@ -13,7 +13,7 @@ class ArrayOfArrayCoercionSpec extends CromwellTestKitSpec {
         sampleWdl = SampleWdl.ArrayOfArrays,
         eventFilter = EventFilter.info(pattern = "Workflow complete", occurrences = 1),
         expectedOutputs = Map(
-          "wf_subtask_concatenated" -> WdlArray(WdlArrayType(WdlStringType), Seq(
+          "wf.subtask.concatenated" -> WdlArray(WdlArrayType(WdlStringType), Seq(
             WdlString("foo\nbar\nbaz"),
             WdlString("third\nfourth")
           ))
diff --git a/engine/src/test/scala/cromwell/ArrayWorkflowSpec.scala b/engine/src/test/scala/cromwell/ArrayWorkflowSpec.scala
index 9c9708307..0b1759716 100644
--- a/engine/src/test/scala/cromwell/ArrayWorkflowSpec.scala
+++ b/engine/src/test/scala/cromwell/ArrayWorkflowSpec.scala
@@ -22,9 +22,9 @@ class ArrayWorkflowSpec extends CromwellTestKitSpec {
         sampleWdl = SampleWdl.ArrayIO,
         eventFilter = EventFilter.info(pattern = "Workflow complete", occurrences = 1),
         expectedOutputs = Map(
-          "wf_count_lines_count" -> WdlInteger(3),
-          "wf_count_lines_array_count" -> WdlInteger(3),
-          "wf_serialize_contents" -> WdlString("str1\nstr2\nstr3")
+          "wf.count_lines.count" -> WdlInteger(3),
+          "wf.count_lines_array.count" -> WdlInteger(3),
+          "wf.serialize.contents" -> WdlString("str1\nstr2\nstr3")
         )
       )
     }
@@ -54,7 +54,7 @@ class ArrayWorkflowSpec extends CromwellTestKitSpec {
     }
     "Coerce Array[String] to Array[File] when running the workflow" in {
       val outputs = Map(
-        "wf_cat_lines" -> WdlArray(WdlArrayType(WdlStringType), Seq(
+        "wf.cat.lines" -> WdlArray(WdlArrayType(WdlStringType), Seq(
             WdlString("line1"),
             WdlString("line2"),
             WdlString("line3"),
diff --git a/engine/src/test/scala/cromwell/CopyWorkflowOutputsSpec.scala b/engine/src/test/scala/cromwell/CopyWorkflowOutputsSpec.scala
index b6465a8ac..d8d910913 100644
--- a/engine/src/test/scala/cromwell/CopyWorkflowOutputsSpec.scala
+++ b/engine/src/test/scala/cromwell/CopyWorkflowOutputsSpec.scala
@@ -31,7 +31,7 @@ class CopyWorkflowOutputsSpec extends CromwellTestKitSpec {
           pattern = "transition from FinalizingWorkflowState to WorkflowSucceededState", occurrences = 1),
         runtime = "",
         workflowOptions = s""" { "final_workflow_outputs_dir": "$tmpDir" } """,
-        expectedOutputs = Seq("A_out", "A_out2", "B_outs") map { o => ("wfoutputs_" + o) -> CromwellTestKitSpec.AnyValueIsFine } toMap,
+        expectedOutputs = Seq("A.out", "A.out2", "B.outs") map { o => ("wfoutputs." + o) -> CromwellTestKitSpec.AnyValueIsFine } toMap,
         allowOtherOutputs = false
       )
 
@@ -64,7 +64,7 @@ class CopyWorkflowOutputsSpec extends CromwellTestKitSpec {
           pattern = "transition from FinalizingWorkflowState to WorkflowSucceededState", occurrences = 1),
         runtime = "",
         workflowOptions = s""" { "final_workflow_outputs_dir": "$tmpDir" } """,
-        expectedOutputs = Map("wfoutputs_A_outs" -> CromwellTestKitSpec.AnyValueIsFine),
+        expectedOutputs = Map("wfoutputs.A.outs" -> CromwellTestKitSpec.AnyValueIsFine),
         allowOtherOutputs = false
       )
 
diff --git a/engine/src/test/scala/cromwell/FilePassingWorkflowSpec.scala b/engine/src/test/scala/cromwell/FilePassingWorkflowSpec.scala
index 40eb03624..d6d059467 100644
--- a/engine/src/test/scala/cromwell/FilePassingWorkflowSpec.scala
+++ b/engine/src/test/scala/cromwell/FilePassingWorkflowSpec.scala
@@ -13,12 +13,12 @@ class FilePassingWorkflowSpec extends CromwellTestKitSpec {
         sampleWdl = SampleWdl.FilePassingWorkflow,
         EventFilter.info(pattern = "Workflow complete", occurrences = 1),
         expectedOutputs = Map(
-          "file_passing_a_out" -> WdlFile("out"),
-          "file_passing_a_out_interpolation" -> WdlFile("out"),
-          "file_passing_a_contents" -> WdlString("foo bar baz"),
-          "file_passing_b_out" -> WdlFile("out"),
-          "file_passing_b_out_interpolation" -> WdlFile("out"),
-          "file_passing_b_contents" -> WdlString("foo bar baz")
+          "file_passing.a.out" -> WdlFile("out"),
+          "file_passing.a.out_interpolation" -> WdlFile("out"),
+          "file_passing.a.contents" -> WdlString("foo bar baz"),
+          "file_passing.b.out" -> WdlFile("out"),
+          "file_passing.b.out_interpolation" -> WdlFile("out"),
+          "file_passing.b.contents" -> WdlString("foo bar baz")
         ),
         patienceConfig = PatienceConfig(2.minutes.dilated)
       )
diff --git a/engine/src/test/scala/cromwell/MapWorkflowSpec.scala b/engine/src/test/scala/cromwell/MapWorkflowSpec.scala
index f13b022c5..ccf6ace5a 100644
--- a/engine/src/test/scala/cromwell/MapWorkflowSpec.scala
+++ b/engine/src/test/scala/cromwell/MapWorkflowSpec.scala
@@ -28,12 +28,12 @@ class MapWorkflowSpec extends CromwellTestKitSpec {
         sampleWdl = sampleWdl,
         EventFilter.info(pattern = "Starting calls: wf.read_map:NA:1, wf.write_map:NA:1", occurrences = 1),
         expectedOutputs = Map(
-          "wf_read_map_out_map" -> WdlMap(WdlMapType(WdlStringType, WdlIntegerType), Map(
+          "wf.read_map.out_map" -> WdlMap(WdlMapType(WdlStringType, WdlIntegerType), Map(
             WdlString("x") -> WdlInteger(500),
             WdlString("y") -> WdlInteger(600),
             WdlString("z") -> WdlInteger(700)
           )),
-          "wf_write_map_contents" -> WdlString("f1\talice\nf2\tbob\nf3\tchuck")
+          "wf.write_map.contents" -> WdlString("f1\talice\nf2\tbob\nf3\tchuck")
         )
       )
       sampleWdl.cleanup()
@@ -75,7 +75,7 @@ class MapWorkflowSpec extends CromwellTestKitSpec {
         sampleWdl,
         eventFilter = EventFilter.info(pattern = "Starting calls: wf.read_map:NA:1, wf.write_map:NA:1", occurrences = 1),
         expectedOutputs = Map(
-          "wf_read_map_out_map" -> WdlMap(WdlMapType(WdlStringType, WdlIntegerType), Map(
+          "wf.read_map.out_map" -> WdlMap(WdlMapType(WdlStringType, WdlIntegerType), Map(
             WdlString("x") -> WdlInteger(500),
             WdlString("y") -> WdlInteger(600),
             WdlString("z") -> WdlInteger(700)
diff --git a/engine/src/test/scala/cromwell/MultipleFilesWithSameNameWorkflowSpec.scala b/engine/src/test/scala/cromwell/MultipleFilesWithSameNameWorkflowSpec.scala
index 4fc24c56e..aa05c08f6 100644
--- a/engine/src/test/scala/cromwell/MultipleFilesWithSameNameWorkflowSpec.scala
+++ b/engine/src/test/scala/cromwell/MultipleFilesWithSameNameWorkflowSpec.scala
@@ -12,8 +12,8 @@ class MultipleFilesWithSameNameWorkflowSpec extends CromwellTestKitSpec {
         sampleWdl = SampleWdl.FileClobber,
         EventFilter.info(pattern = "Starting calls: two.x:NA:1, two.y:NA:1", occurrences = 1),
         expectedOutputs = Map(
-          "two_x_out" -> WdlString("first file.txt"),
-          "two_y_out" -> WdlString("second file.txt")
+          "two.x.out" -> WdlString("first file.txt"),
+          "two.y.out" -> WdlString("second file.txt")
         )
       )
     }
diff --git a/engine/src/test/scala/cromwell/PostfixQuantifierWorkflowSpec.scala b/engine/src/test/scala/cromwell/PostfixQuantifierWorkflowSpec.scala
index c72d98758..a436a6287 100644
--- a/engine/src/test/scala/cromwell/PostfixQuantifierWorkflowSpec.scala
+++ b/engine/src/test/scala/cromwell/PostfixQuantifierWorkflowSpec.scala
@@ -11,21 +11,21 @@ class PostfixQuantifierWorkflowSpec extends CromwellTestKitSpec {
       runWdlAndAssertOutputs(
         sampleWdl = SampleWdl.ZeroOrMorePostfixQuantifierWorkflowWithArrayInput,
         EventFilter.info(pattern = "Starting calls: postfix.hello", occurrences = 1),
-        expectedOutputs = Map("postfix_hello_greeting" -> WdlString("hello alice,bob,charles"))
+        expectedOutputs = Map("postfix.hello.greeting" -> WdlString("hello alice,bob,charles"))
       )
     }
     "accept an array of size 1" in {
       runWdlAndAssertOutputs(
         sampleWdl = SampleWdl.ZeroOrMorePostfixQuantifierWorkflowWithOneElementArrayInput,
         EventFilter.info(pattern = "Starting calls: postfix.hello", occurrences = 1),
-        expectedOutputs = Map("postfix_hello_greeting" -> WdlString("hello alice"))
+        expectedOutputs = Map("postfix.hello.greeting" -> WdlString("hello alice"))
       )
     }
     "accept an array of size 0" in {
       runWdlAndAssertOutputs(
         sampleWdl = SampleWdl.ZeroOrMorePostfixQuantifierWorkflowWithZeroElementArrayInput,
         EventFilter.info(pattern = "Starting calls: postfix.hello", occurrences = 1),
-        expectedOutputs = Map("postfix_hello_greeting" -> WdlString("hello"))
+        expectedOutputs = Map("postfix.hello.greeting" -> WdlString("hello"))
       )
     }
   }
@@ -35,14 +35,14 @@ class PostfixQuantifierWorkflowSpec extends CromwellTestKitSpec {
       runWdlAndAssertOutputs(
         sampleWdl = SampleWdl.OneOrMorePostfixQuantifierWorkflowWithArrayInput,
         EventFilter.info(pattern = "Starting calls: postfix.hello", occurrences = 1),
-        expectedOutputs = Map("postfix_hello_greeting" -> WdlString("hello alice,bob,charles"))
+        expectedOutputs = Map("postfix.hello.greeting" -> WdlString("hello alice,bob,charles"))
       )
     }
     "accept a scalar for the value" in {
       runWdlAndAssertOutputs(
         sampleWdl = SampleWdl.OneOrMorePostfixQuantifierWorkflowWithScalarInput,
         EventFilter.info(pattern = "Starting calls: postfix.hello", occurrences = 1),
-        expectedOutputs = Map("postfix_hello_greeting" -> WdlString("hello alice"))
+        expectedOutputs = Map("postfix.hello.greeting" -> WdlString("hello alice"))
       )
     }
   }
diff --git a/engine/src/test/scala/cromwell/ScatterWorkflowSpec.scala b/engine/src/test/scala/cromwell/ScatterWorkflowSpec.scala
index c1ca1ad9e..306d5b3bd 100644
--- a/engine/src/test/scala/cromwell/ScatterWorkflowSpec.scala
+++ b/engine/src/test/scala/cromwell/ScatterWorkflowSpec.scala
@@ -13,8 +13,8 @@ class ScatterWorkflowSpec extends CromwellTestKitSpec {
         sampleWdl = SampleWdl.SimpleScatterWdl,
         eventFilter = EventFilter.info(pattern = "Workflow complete", occurrences = 1),
         expectedOutputs = Map(
-          "scatter0_outside_scatter_out" -> WdlInteger(8000),
-          "scatter0_inside_scatter_out" -> WdlArray(WdlArrayType(WdlIntegerType), Seq(1, 2, 3, 4, 5).map(WdlInteger(_)))
+          "scatter0.outside_scatter.out" -> WdlInteger(8000),
+          "scatter0.inside_scatter.out" -> WdlArray(WdlArrayType(WdlIntegerType), Seq(1, 2, 3, 4, 5).map(WdlInteger(_)))
         )
       )
     }
@@ -25,11 +25,11 @@ class ScatterWorkflowSpec extends CromwellTestKitSpec {
         sampleWdl = new SampleWdl.ScatterWdl,
         eventFilter = EventFilter.info(pattern = "Workflow complete", occurrences = 1),
         expectedOutputs = Map(
-          "w_E_E_out" -> WdlArray(WdlArrayType(WdlIntegerType), Seq(9, 9, 9, 9, 9, 9).map(WdlInteger(_))),
-          "w_C_C_out" -> WdlArray(WdlArrayType(WdlIntegerType), Seq(400, 500, 600, 800, 600, 500).map(WdlInteger(_))),
-          "w_A_A_out" -> WdlArray(WdlArrayType(WdlStringType), Seq("jeff", "chris", "miguel", "thibault", "khalid", "ruchi").map(WdlString)),
-          "w_D_D_out" -> WdlInteger(34),
-          "w_B_B_out" -> WdlArray(WdlArrayType(WdlIntegerType), Seq(4, 5, 6, 8, 6, 5).map(WdlInteger(_)))
+          "w.E.E_out" -> WdlArray(WdlArrayType(WdlIntegerType), Seq(9, 9, 9, 9, 9, 9).map(WdlInteger(_))),
+          "w.C.C_out" -> WdlArray(WdlArrayType(WdlIntegerType), Seq(400, 500, 600, 800, 600, 500).map(WdlInteger(_))),
+          "w.A.A_out" -> WdlArray(WdlArrayType(WdlStringType), Seq("jeff", "chris", "miguel", "thibault", "khalid", "ruchi").map(WdlString)),
+          "w.D.D_out" -> WdlInteger(34),
+          "w.B.B_out" -> WdlArray(WdlArrayType(WdlIntegerType), Seq(4, 5, 6, 8, 6, 5).map(WdlInteger(_)))
         )
       )
     }
@@ -40,12 +40,12 @@ class ScatterWorkflowSpec extends CromwellTestKitSpec {
         sampleWdl = SampleWdl.SiblingsScatterWdl,
         eventFilter = EventFilter.info(pattern = "Workflow complete", occurrences = 1),
         expectedOutputs = Map(
-          "w_E_E_out" -> WdlArray(WdlArrayType(WdlIntegerType), Seq(9, 9, 9, 9, 9, 9).map(WdlInteger(_))),
-          "w_F_B_out" -> WdlArray(WdlArrayType(WdlIntegerType), Seq(4, 5, 6, 8, 6, 5).map(WdlInteger(_))),
-          "w_C_C_out" -> WdlArray(WdlArrayType(WdlIntegerType), Seq(400, 500, 600, 800, 600, 500).map(WdlInteger(_))),
-          "w_A_A_out" -> WdlArray(WdlArrayType(WdlStringType), Seq("jeff", "chris", "miguel", "thibault", "khalid", "ruchi").map(WdlString)),
-          "w_D_D_out" -> WdlInteger(34),
-          "w_B_B_out" -> WdlArray(WdlArrayType(WdlIntegerType), Seq(4, 5, 6, 8, 6, 5).map(WdlInteger(_)))
+          "w.E.E_out" -> WdlArray(WdlArrayType(WdlIntegerType), Seq(9, 9, 9, 9, 9, 9).map(WdlInteger(_))),
+          "w.F.B_out" -> WdlArray(WdlArrayType(WdlIntegerType), Seq(4, 5, 6, 8, 6, 5).map(WdlInteger(_))),
+          "w.C.C_out" -> WdlArray(WdlArrayType(WdlIntegerType), Seq(400, 500, 600, 800, 600, 500).map(WdlInteger(_))),
+          "w.A.A_out" -> WdlArray(WdlArrayType(WdlStringType), Seq("jeff", "chris", "miguel", "thibault", "khalid", "ruchi").map(WdlString)),
+          "w.D.D_out" -> WdlInteger(34),
+          "w.B.B_out" -> WdlArray(WdlArrayType(WdlIntegerType), Seq(4, 5, 6, 8, 6, 5).map(WdlInteger(_)))
         )
       )
     }
@@ -57,9 +57,9 @@ class ScatterWorkflowSpec extends CromwellTestKitSpec {
         sampleWdl = SampleWdl.PrepareScatterGatherWdl(),
         eventFilter = EventFilter.info(pattern = "Workflow complete", occurrences = 1),
         expectedOutputs = Map(
-          "sc_test_do_gather_sum" -> WdlInteger(11),
-          "sc_test_do_prepare_split_files" -> WdlArray(WdlArrayType(WdlFileType), Seq("temp_aa", "temp_ab", "temp_ac", "temp_ad").map(WdlFile(_))),
-          "sc_test_do_scatter_count_file" -> WdlArray(WdlArrayType(WdlFileType), (1 to 4).map(_ => WdlFile("output.txt")))
+          "sc_test.do_gather.sum" -> WdlInteger(11),
+          "sc_test.do_prepare.split_files" -> WdlArray(WdlArrayType(WdlFileType), Seq("temp_aa", "temp_ab", "temp_ac", "temp_ad").map(WdlFile(_))),
+          "sc_test.do_scatter.count_file" -> WdlArray(WdlArrayType(WdlFileType), (1 to 4).map(_ => WdlFile("output.txt")))
         )
       )
     }
@@ -74,9 +74,9 @@ class ScatterWorkflowSpec extends CromwellTestKitSpec {
                   |}
                   """.stripMargin,
         expectedOutputs = Map(
-          "sc_test_do_gather_sum" -> WdlInteger(11),
-          "sc_test_do_prepare_split_files" -> WdlArray(WdlArrayType(WdlFileType), Seq("temp_aa", "temp_ab", "temp_ac", "temp_ad").map(WdlFile(_))),
-          "sc_test_do_scatter_count_file" -> WdlArray(WdlArrayType(WdlFileType), (1 to 4).map(_ => WdlFile("output.txt")))
+          "sc_test.do_gather.sum" -> WdlInteger(11),
+          "sc_test.do_prepare.split_files" -> WdlArray(WdlArrayType(WdlFileType), Seq("temp_aa", "temp_ab", "temp_ac", "temp_ad").map(WdlFile(_))),
+          "sc_test.do_scatter.count_file" -> WdlArray(WdlArrayType(WdlFileType), (1 to 4).map(_ => WdlFile("output.txt")))
         )
       )
     }
diff --git a/engine/src/test/scala/cromwell/WdlFunctionsAtWorkflowLevelSpec.scala b/engine/src/test/scala/cromwell/WdlFunctionsAtWorkflowLevelSpec.scala
index 3052e3370..186bd0714 100644
--- a/engine/src/test/scala/cromwell/WdlFunctionsAtWorkflowLevelSpec.scala
+++ b/engine/src/test/scala/cromwell/WdlFunctionsAtWorkflowLevelSpec.scala
@@ -19,8 +19,8 @@ class WdlFunctionsAtWorkflowLevelSpec extends CromwellTestKitSpec {
         sampleWdl = SampleWdl.WdlFunctionsAtWorkflowLevel,
         eventFilter = EventFilter.info(pattern = "Starting calls: w.a", occurrences = 1),
         expectedOutputs = Map(
-          "w_a_x" -> WdlString("one two three four five"),
-          "w_a_y" -> outputMap
+          "w.a.x" -> WdlString("one two three four five"),
+          "w.a.y" -> outputMap
         )
       )
     }
diff --git a/engine/src/test/scala/cromwell/WorkflowOutputsSpec.scala b/engine/src/test/scala/cromwell/WorkflowOutputsSpec.scala
index 0f210fd1e..b821dd742 100644
--- a/engine/src/test/scala/cromwell/WorkflowOutputsSpec.scala
+++ b/engine/src/test/scala/cromwell/WorkflowOutputsSpec.scala
@@ -13,9 +13,9 @@ class WorkflowOutputsSpec extends CromwellTestKitSpec {
         eventFilter = EventFilter.info(pattern = s"is in a terminal state: WorkflowSucceededState", occurrences = 1),
         runtime = "",
         expectedOutputs = Map(
-          "three_step_ps_procs" -> AnyValueIsFine,
-          "three_step_cgrep_count" -> AnyValueIsFine,
-          "three_step_wc_count" -> AnyValueIsFine
+          "three_step.ps.procs" -> AnyValueIsFine,
+          "three_step.cgrep.count" -> AnyValueIsFine,
+          "three_step.wc.count" -> AnyValueIsFine
         ),
         allowOtherOutputs = false
       )
@@ -27,8 +27,8 @@ class WorkflowOutputsSpec extends CromwellTestKitSpec {
         eventFilter = EventFilter.info(pattern = s"is in a terminal state: WorkflowSucceededState", occurrences = 1),
         runtime = "",
         expectedOutputs = Map(
-          "three_step_cgrep_count" -> AnyValueIsFine,
-          "three_step_wc_count" -> AnyValueIsFine
+          "three_step.cgrep.count" -> AnyValueIsFine,
+          "three_step.wc.count" -> AnyValueIsFine
         ),
         allowOtherOutputs = false
       )
@@ -40,8 +40,8 @@ class WorkflowOutputsSpec extends CromwellTestKitSpec {
         eventFilter = EventFilter.info(pattern = s"is in a terminal state: WorkflowSucceededState", occurrences = 1),
         runtime = "",
         expectedOutputs = Map(
-          "scatter0_outside_scatter_out" -> AnyValueIsFine,
-          "scatter0_inside_scatter_out" -> AnyValueIsFine
+          "scatter0.outside_scatter.out" -> AnyValueIsFine,
+          "scatter0.inside_scatter.out" -> AnyValueIsFine
         ),
         allowOtherOutputs = false
       )
@@ -53,7 +53,7 @@ class WorkflowOutputsSpec extends CromwellTestKitSpec {
         eventFilter = EventFilter.info(pattern = s"is in a terminal state: WorkflowSucceededState", occurrences = 1),
         runtime = "",
         expectedOutputs = Map(
-          "scatter0_inside_scatter_out" -> AnyValueIsFine
+          "scatter0.inside_scatter.out" -> AnyValueIsFine
         ),
         allowOtherOutputs = false
       )
diff --git a/engine/src/test/scala/cromwell/engine/WorkflowManagerActorSpec.scala b/engine/src/test/scala/cromwell/engine/WorkflowManagerActorSpec.scala
index 1d7d313b6..b22ebc13f 100644
--- a/engine/src/test/scala/cromwell/engine/WorkflowManagerActorSpec.scala
+++ b/engine/src/test/scala/cromwell/engine/WorkflowManagerActorSpec.scala
@@ -13,7 +13,7 @@ class WorkflowManagerActorSpec extends CromwellTestKitSpec with WorkflowDescript
     "run workflows in the correct directory" in {
       val outputs = runWdl(sampleWdl = SampleWdl.CurrentDirectory)
 
-      val outputName = "wf_whereami_whereami_pwd"
+      val outputName = "wf_whereami.whereami.pwd"
       val salutation = outputs(outputName)
       val actualOutput = salutation.valueString.trim
       actualOutput should endWith("/call-whereami/execution")
diff --git a/project/Dependencies.scala b/project/Dependencies.scala
index 8711f0c08..7f4723910 100644
--- a/project/Dependencies.scala
+++ b/project/Dependencies.scala
@@ -2,7 +2,7 @@ import sbt._
 
 object Dependencies {
   lazy val lenthallV = "0.20"
-  lazy val wdl4sV = "0.9-9e4b343-SNAP"
+  lazy val wdl4sV = "0.9-a728f84-SNAP"
   lazy val sprayV = "1.3.3"
   /*
   spray-json is an independent project from the "spray suite"

From bf77c34d72bfe1bfd69a78f6e72d328ab3858076 Mon Sep 17 00:00:00 2001
From: Thib <Horneth@users.noreply.github.com>
Date: Fri, 27 Jan 2017 10:26:43 -0500
Subject: [PATCH 33/84] Bring changes from 24_hotfix to develop (#1916)

* update to wdl4s 0.9 and bring changes from 24_hotfix

* custom centaur branch while PR pending

* remove unused import

* back to centaur develop
---
 .../scala/cromwell/backend/wdl/OutputEvaluator.scala     |  4 +++-
 .../workflow/lifecycle/CopyWorkflowLogsActor.scala       |  2 +-
 .../lifecycle/execution/WorkflowExecutionActor.scala     | 16 +++++++++++++---
 .../scala/cromwell/filesystems/gcs/GcsPathBuilder.scala  | 16 ++++++++++++----
 .../cromwell/filesystems/gcs/GcsPathBuilderSpec.scala    | 13 +++++++++++++
 project/Dependencies.scala                               |  2 +-
 .../impl/jes/JesAsyncBackendJobExecutionActorSpec.scala  |  6 +-----
 7 files changed, 44 insertions(+), 15 deletions(-)

diff --git a/backend/src/main/scala/cromwell/backend/wdl/OutputEvaluator.scala b/backend/src/main/scala/cromwell/backend/wdl/OutputEvaluator.scala
index ab6c66b0d..d41d25b19 100644
--- a/backend/src/main/scala/cromwell/backend/wdl/OutputEvaluator.scala
+++ b/backend/src/main/scala/cromwell/backend/wdl/OutputEvaluator.scala
@@ -13,7 +13,9 @@ object OutputEvaluator {
                       wdlFunctions: WdlStandardLibraryFunctions,
                       postMapper: WdlValue => Try[WdlValue] = v => Success(v)): Try[Map[LocallyQualifiedName, JobOutput]] = {
     jobDescriptor.call.task.evaluateOutputs(jobDescriptor.inputDeclarations, wdlFunctions, postMapper) map { outputs =>
-      outputs mapValues JobOutput
+      outputs map {
+        case (output, value) => output.unqualifiedName -> JobOutput(value)
+      } 
     }
   }
 }
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/CopyWorkflowLogsActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/CopyWorkflowLogsActor.scala
index bb1a7a460..920caf77a 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/CopyWorkflowLogsActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/CopyWorkflowLogsActor.scala
@@ -45,7 +45,7 @@ class CopyWorkflowLogsActor(serviceRegistryActor: ActorRef)
 
       workflowLogger.workflowLogPath foreach { src =>
         if (File(src).exists) {
-          val destPath = destinationDir.resolve(src.getFileName)
+          val destPath = destinationDir.resolve(src.getFileName.toString)
           workflowLogger.info(s"Copying workflow logs from $src to $destPath")
 
           copyAndClean(src, destPath)
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala
index 30504f2a1..8a76b20a7 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala
@@ -263,12 +263,22 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor,
       data.expressionLanguageFunctions,
       data.outputStore.fetchNodeOutputEntries
     ) map { workflowOutputs =>
+       // For logging and metadata
+       val workflowScopeOutputs = workflowOutputs map {
+         case (output, value) => output.locallyQualifiedName(workflowDescriptor.workflow) -> value
+       }
        workflowLogger.info(
          s"""Workflow ${workflowDescriptor.workflow.unqualifiedName} complete. Final Outputs:
-             |${workflowOutputs.stripLarge.toJson.prettyPrint}""".stripMargin
+             |${workflowScopeOutputs.stripLarge.toJson.prettyPrint}""".stripMargin
        )
-       pushWorkflowOutputMetadata(workflowOutputs)
-       (WorkflowExecutionSucceededResponse(data.jobExecutionMap, workflowOutputs mapValues JobOutput.apply), WorkflowExecutionSuccessfulState)
+       pushWorkflowOutputMetadata(workflowScopeOutputs)
+       
+       // For cromwell internal storage of outputs
+       val unqualifiedWorkflowOutputs = workflowOutputs map {
+         // JobOutput is poorly named here - a WorkflowOutput type would be better
+         case (output, value) => output.unqualifiedName -> JobOutput(value)
+       }
+       (WorkflowExecutionSucceededResponse(data.jobExecutionMap, unqualifiedWorkflowOutputs), WorkflowExecutionSuccessfulState)
     } recover {
        case ex =>
          (WorkflowExecutionFailedResponse(data.jobExecutionMap, ex), WorkflowExecutionFailedState)
diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsPathBuilder.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsPathBuilder.scala
index ec2f2da47..376a032cc 100644
--- a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsPathBuilder.scala
+++ b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsPathBuilder.scala
@@ -53,10 +53,16 @@ class GcsPathBuilder(authMode: GoogleAuthMode,
                      options: WorkflowOptions) extends PathBuilder {
   authMode.validate(options)
 
-  protected val storageOptions = StorageOptions.builder()
-    .authCredentials(authMode.authCredentials(options))
-    .retryParams(retryParams)
-    .build()
+  protected val storageOptionsBuilder = StorageOptions.builder()
+                                  .authCredentials(authMode.authCredentials(options))
+                                  .retryParams(retryParams)
+
+  // Grab the google project from Workflow Options if specified and set
+  // that to be the project used by the StorageOptions Builder
+  options.get("google_project") map storageOptionsBuilder.projectId
+
+
+  protected val storageOptions = storageOptionsBuilder.build()
 
   // The CloudStorageFileSystemProvider constructor is not public. Currently the only way to obtain one is through a CloudStorageFileSystem
   // Moreover at this point we can use the same provider for all operations as we have usable credentials
@@ -80,6 +86,8 @@ class GcsPathBuilder(authMode: GoogleAuthMode,
     }
   }
 
+  def getProjectId = storageOptions.projectId()
+
   def build(string: String): Try[Path] = {
     Try {
       val uri = getUri(string)
diff --git a/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GcsPathBuilderSpec.scala b/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GcsPathBuilderSpec.scala
index 3ce135e2f..6f7b00dbb 100644
--- a/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GcsPathBuilderSpec.scala
+++ b/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GcsPathBuilderSpec.scala
@@ -28,4 +28,17 @@ class GcsPathBuilderSpec extends TestKitSuite with FlatSpecLike with Matchers {
     path.get.getFileSystem.provider() shouldBe a[RetryableFileSystemProviderProxy[_]]
   }
 
+  it should "use google project credentials when provided in the workflow options" in {
+
+    val wfOptionsWithProject = WorkflowOptions.fromMap(Map("google_project" -> "my_project")).get
+
+    val gcsPathBuilderWithProjectInfo = new GcsPathBuilder(
+      GoogleAuthMode.NoAuthMode,
+      RetryParams.defaultInstance(),
+      CloudStorageConfiguration.DEFAULT,
+      wfOptionsWithProject
+    )
+
+    gcsPathBuilderWithProjectInfo.getProjectId shouldBe "my_project"
+  }
 }
diff --git a/project/Dependencies.scala b/project/Dependencies.scala
index 81d449aaf..0c4877da6 100644
--- a/project/Dependencies.scala
+++ b/project/Dependencies.scala
@@ -2,7 +2,7 @@ import sbt._
 
 object Dependencies {
   lazy val lenthallV = "0.20"
-  lazy val wdl4sV = "0.9-a728f84-SNAP"
+  lazy val wdl4sV = "0.9"
   lazy val sprayV = "1.3.3"
   /*
   spray-json is an independent project from the "spray suite"
diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala
index 56ccc1ced..9d33c195b 100644
--- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala
+++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala
@@ -20,7 +20,6 @@ import cromwell.core.labels.Labels
 import cromwell.core.logging.JobLogger
 import cromwell.core.path.PathImplicits._
 import cromwell.filesystems.gcs.auth.GoogleAuthMode.NoAuthMode
-import cromwell.filesystems.gcs.auth.GoogleAuthModeSpec
 import cromwell.filesystems.gcs.{GcsPathBuilder, GcsPathBuilderFactory}
 import cromwell.util.SampleWdl
 import org.scalatest._
@@ -39,10 +38,7 @@ import scala.util.{Success, Try}
 class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackendJobExecutionActorSpec")
   with FlatSpecLike with Matchers with ImplicitSender with Mockito with BackendSpec {
 
-  lazy val mockPathBuilder: GcsPathBuilder = {
-    GoogleAuthModeSpec.assumeHasApplicationDefaultCredentials()
-    GcsPathBuilderFactory(NoAuthMode).withOptions(mock[WorkflowOptions])
-  }
+  val mockPathBuilder: GcsPathBuilder = GcsPathBuilderFactory(NoAuthMode).withOptions(WorkflowOptions.empty)
 
   import JesTestConfig._
 

From 48f98489495552d0eb5d5a46b8f598ce4481f58a Mon Sep 17 00:00:00 2001
From: Chris Llanwarne <cjllanwarne@gmail.com>
Date: Tue, 17 Jan 2017 17:16:34 -0500
Subject: [PATCH 34/84] Bringing in WDL fix for 1830

---
 .../async/AsyncBackendJobExecutionActor.scala      | 11 ++++++++-
 .../test/scala/cromwell/backend/BackendSpec.scala  |  2 +-
 .../lifecycle/execution/JobPreparationActor.scala  | 27 +++++++++++-----------
 .../scala/cromwell/OptionalParamWorkflowSpec.scala |  6 ++---
 project/Dependencies.scala                         |  2 +-
 5 files changed, 27 insertions(+), 21 deletions(-)

diff --git a/backend/src/main/scala/cromwell/backend/async/AsyncBackendJobExecutionActor.scala b/backend/src/main/scala/cromwell/backend/async/AsyncBackendJobExecutionActor.scala
index 3aaadfd55..fdb860945 100644
--- a/backend/src/main/scala/cromwell/backend/async/AsyncBackendJobExecutionActor.scala
+++ b/backend/src/main/scala/cromwell/backend/async/AsyncBackendJobExecutionActor.scala
@@ -1,5 +1,6 @@
 package cromwell.backend.async
 
+
 import akka.actor.{Actor, ActorLogging, ActorRef}
 import cromwell.backend.BackendJobDescriptor
 import cromwell.backend.BackendJobExecutionActor._
@@ -35,7 +36,15 @@ trait AsyncBackendJobExecutionActor { this: Actor with ActorLogging =>
 
   def retryable: Boolean
 
-  def isFatal(throwable: Throwable): Boolean = throwable.isInstanceOf[CromwellFatalExceptionMarker]
+  // The scala package object (scala/package.scala) contains a neat list of runtime errors that are always going to be fatal.
+  // We also consider any Error as fatal, and include the CromwellFatalExceptionMarker so we can mark our own fatal exceptions.
+  def isFatal(throwable: Throwable): Boolean = throwable match {
+    case _: Error => true
+    case _: RuntimeException => true
+    case _: InterruptedException => true
+    case _: CromwellFatalExceptionMarker => true
+    case _ => false
+  }
 
   def isTransient(throwable: Throwable): Boolean = !isFatal(throwable)
 
diff --git a/backend/src/test/scala/cromwell/backend/BackendSpec.scala b/backend/src/test/scala/cromwell/backend/BackendSpec.scala
index aec7c8ade..810726822 100644
--- a/backend/src/test/scala/cromwell/backend/BackendSpec.scala
+++ b/backend/src/test/scala/cromwell/backend/BackendSpec.scala
@@ -52,7 +52,7 @@ trait BackendSpec extends ScalaFutures with Matchers with Mockito {
                                           runtimeAttributeDefinitions: Set[RuntimeAttributeDefinition]): BackendJobDescriptor = {
     val call = workflowDescriptor.workflow.taskCalls.head
     val jobKey = BackendJobDescriptorKey(call, None, 1)
-    val inputDeclarations = call.evaluateTaskInputs(inputs, NoFunctions)
+    val inputDeclarations = call.evaluateTaskInputs(inputs, NoFunctions).get // .get is ok because this is a test
     val evaluatedAttributes = RuntimeAttributeDefinition.evaluateRuntimeAttributes(call.task.runtimeAttributes, NoFunctions, inputDeclarations).get // .get is OK here because this is a test
     val runtimeAttributes = RuntimeAttributeDefinition.addDefaultsToAttributes(runtimeAttributeDefinitions, options)(evaluatedAttributes)
     BackendJobDescriptor(workflowDescriptor, jobKey, runtimeAttributes, inputDeclarations)
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/JobPreparationActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/JobPreparationActor.scala
index a636a8d10..4bb6c2ab9 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/JobPreparationActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/JobPreparationActor.scala
@@ -32,20 +32,19 @@ abstract class CallPreparationActor(val workflowDescriptor: EngineWorkflowDescri
   }
 
   def resolveAndEvaluateInputs(): Try[Map[Declaration, WdlValue]] = {
-    Try {
-      val call = callKey.scope
-      val scatterMap = callKey.index flatMap { i =>
-        // Will need update for nested scatters
-        call.ancestry collectFirst { case s: Scatter => Map(s -> i) }
-      } getOrElse Map.empty[Scatter, Int]
-
-      call.evaluateTaskInputs(
-        workflowDescriptor.backendDescriptor.knownValues,
-        expressionLanguageFunctions,
-        outputStore.fetchNodeOutputEntries,
-        scatterMap
-      )
-    } recoverWith {
+
+    val call = callKey.scope
+    val scatterMap = callKey.index flatMap { i =>
+      // Will need update for nested scatters
+      call.ancestry collectFirst { case s: Scatter => Map(s -> i) }
+    } getOrElse Map.empty[Scatter, Int]
+
+    call.evaluateTaskInputs(
+      workflowDescriptor.backendDescriptor.knownValues,
+      expressionLanguageFunctions,
+      outputStore.fetchNodeOutputEntries,
+      scatterMap
+    ).recoverWith {
       case t: Throwable => Failure(new VariableLookupException(s"Couldn't resolve all inputs for ${callKey.scope.fullyQualifiedName} at index ${callKey.index}.", List(t)))
     }
   }
diff --git a/engine/src/test/scala/cromwell/OptionalParamWorkflowSpec.scala b/engine/src/test/scala/cromwell/OptionalParamWorkflowSpec.scala
index 73347b794..29035c00b 100644
--- a/engine/src/test/scala/cromwell/OptionalParamWorkflowSpec.scala
+++ b/engine/src/test/scala/cromwell/OptionalParamWorkflowSpec.scala
@@ -27,10 +27,8 @@ class OptionalParamWorkflowSpec extends Matchers with WordSpecLike {
         fail("Expected to find task 'find'")
       }
 
-      val instantiateWithoutValue = findTask.instantiateCommand(findTask.inputsFromMap(Map("find.root" -> WdlFile("src"))), NoFunctions) getOrElse {
-        fail("Expected instantiation to work")
-      }
-      instantiateWithoutValue shouldEqual "find src"
+      val instantiateWithoutValue = findTask.instantiateCommand(findTask.inputsFromMap(Map("find.root" -> WdlFile("src"))), NoFunctions)
+      instantiateWithoutValue.get shouldEqual "find src"
 
       val instantiateWithValue = findTask.instantiateCommand(findTask.inputsFromMap(Map(
         "find.root" -> WdlFile("src"),
diff --git a/project/Dependencies.scala b/project/Dependencies.scala
index 0c4877da6..bd730569d 100644
--- a/project/Dependencies.scala
+++ b/project/Dependencies.scala
@@ -2,7 +2,7 @@ import sbt._
 
 object Dependencies {
   lazy val lenthallV = "0.20"
-  lazy val wdl4sV = "0.9"
+  lazy val wdl4sV = "0.10-3ad63bf-SNAP"
   lazy val sprayV = "1.3.3"
   /*
   spray-json is an independent project from the "spray suite"

From 5dc89479653744933a89ed030a6655565f6ddf54 Mon Sep 17 00:00:00 2001
From: Khalid Shakir <kshakir@broadinstitute.org>
Date: Fri, 27 Jan 2017 23:14:51 -0500
Subject: [PATCH 35/84] Fixed issues around empty lobs in MySQL. Closes #1919
 Only store null instead of "" for a clob. Only store null instead of
 Array.empty[Byte] for a blob. All lob columns are now nullable. Added test to
 ensure that all lob columns are nullable. Updated SqlConverters for
 converting to/from Option and empty lobs.

---
 .../migration/src/main/resources/changelog.xml     |   1 +
 .../main/resources/changesets/nullable_lobs.xml    |  21 ++++
 .../tables/CallCachingDetritusEntryComponent.scala |   2 +-
 .../CallCachingSimpletonEntryComponent.scala       |   2 +-
 .../tables/JobStoreSimpletonEntryComponent.scala   |   2 +-
 .../slick/tables/WorkflowStoreEntryComponent.scala |   8 +-
 .../cromwell/database/sql/SqlConverters.scala      |  34 ++++--
 .../sql/tables/CallCachingDetritusEntry.scala      |   2 +-
 .../sql/tables/CallCachingSimpletonEntry.scala     |   2 +-
 .../sql/tables/JobStoreSimpletonEntry.scala        |   2 +-
 .../database/sql/tables/WorkflowStoreEntry.scala   |   8 +-
 .../workflow/workflowstore/SqlWorkflowStore.scala  |   4 +-
 .../main/scala/cromwell/jobstore/SqlJobStore.scala |   7 +-
 .../metadata/impl/MetadataDatabaseAccess.scala     |  17 ++-
 .../cromwell/services/ServicesStoreSpec.scala      | 130 ++++++++++++++++++---
 15 files changed, 186 insertions(+), 56 deletions(-)
 create mode 100644 database/migration/src/main/resources/changesets/nullable_lobs.xml

diff --git a/database/migration/src/main/resources/changelog.xml b/database/migration/src/main/resources/changelog.xml
index 2fb2a8f47..851589da4 100644
--- a/database/migration/src/main/resources/changelog.xml
+++ b/database/migration/src/main/resources/changelog.xml
@@ -57,6 +57,7 @@
     <include file="changesets/workflow_store_labels_file.xml" relativeToChangelogFile="true" />
     <include file="changesets/embiggen_detritus_value.xml" relativeToChangelogFile="true" />
     <include file="changesets/standardize_column_names_patches.xml" relativeToChangelogFile="true" />
+    <include file="changesets/nullable_lobs.xml" relativeToChangelogFile="true" />
 </databaseChangeLog>
 <!--
 
diff --git a/database/migration/src/main/resources/changesets/nullable_lobs.xml b/database/migration/src/main/resources/changesets/nullable_lobs.xml
new file mode 100644
index 000000000..13783585c
--- /dev/null
+++ b/database/migration/src/main/resources/changesets/nullable_lobs.xml
@@ -0,0 +1,21 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
+    <changeSet author="kshakir" id="nullable_lobs">
+        <dropNotNullConstraint columnDataType="LONGTEXT"
+                               tableName="CALL_CACHING_DETRITUS_ENTRY" columnName="DETRITUS_VALUE"/>
+        <dropNotNullConstraint columnDataType="LONGTEXT"
+                               tableName="CALL_CACHING_SIMPLETON_ENTRY" columnName="SIMPLETON_VALUE"/>
+        <dropNotNullConstraint columnDataType="LONGTEXT"
+                               tableName="JOB_STORE_SIMPLETON_ENTRY" columnName="SIMPLETON_VALUE"/>
+        <dropNotNullConstraint columnDataType="LONGTEXT"
+                               tableName="WORKFLOW_STORE_ENTRY" columnName="WORKFLOW_DEFINITION"/>
+        <dropNotNullConstraint columnDataType="LONGTEXT"
+                               tableName="WORKFLOW_STORE_ENTRY" columnName="WORKFLOW_INPUTS"/>
+        <dropNotNullConstraint columnDataType="LONGTEXT"
+                               tableName="WORKFLOW_STORE_ENTRY" columnName="WORKFLOW_OPTIONS"/>
+        <dropNotNullConstraint columnDataType="LONGTEXT"
+                               tableName="WORKFLOW_STORE_ENTRY" columnName="CUSTOM_LABELS"/>
+    </changeSet>
+</databaseChangeLog>
diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingDetritusEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingDetritusEntryComponent.scala
index eae065b1c..040d61f64 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingDetritusEntryComponent.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingDetritusEntryComponent.scala
@@ -16,7 +16,7 @@ trait CallCachingDetritusEntryComponent {
 
     def detritusKey = column[String]("DETRITUS_KEY")
 
-    def detritusValue = column[Clob]("DETRITUS_VALUE")
+    def detritusValue = column[Option[Clob]]("DETRITUS_VALUE")
 
     def callCachingEntryId = column[Int]("CALL_CACHING_ENTRY_ID")
 
diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingSimpletonEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingSimpletonEntryComponent.scala
index a47f0df19..e2077e027 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingSimpletonEntryComponent.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingSimpletonEntryComponent.scala
@@ -16,7 +16,7 @@ trait CallCachingSimpletonEntryComponent {
 
     def simpletonKey = column[String]("SIMPLETON_KEY")
 
-    def simpletonValue = column[Clob]("SIMPLETON_VALUE")
+    def simpletonValue = column[Option[Clob]]("SIMPLETON_VALUE")
 
     def wdlType = column[String]("WDL_TYPE")
 
diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/JobStoreSimpletonEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/JobStoreSimpletonEntryComponent.scala
index c9d51ca51..4364170ac 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/tables/JobStoreSimpletonEntryComponent.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/tables/JobStoreSimpletonEntryComponent.scala
@@ -16,7 +16,7 @@ trait JobStoreSimpletonEntryComponent {
 
     def simpletonKey = column[String]("SIMPLETON_KEY")
 
-    def simpletonValue = column[Clob]("SIMPLETON_VALUE")
+    def simpletonValue = column[Option[Clob]]("SIMPLETON_VALUE")
 
     def wdlType = column[String]("WDL_TYPE")
 
diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowStoreEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowStoreEntryComponent.scala
index 213e2cbe6..733fc7848 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowStoreEntryComponent.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowStoreEntryComponent.scala
@@ -15,13 +15,13 @@ trait WorkflowStoreEntryComponent {
 
     def workflowExecutionUuid = column[String]("WORKFLOW_EXECUTION_UUID")
 
-    def workflowDefinition = column[Clob]("WORKFLOW_DEFINITION")
+    def workflowDefinition = column[Option[Clob]]("WORKFLOW_DEFINITION")
 
-    def workflowInputs = column[Clob]("WORKFLOW_INPUTS")
+    def workflowInputs = column[Option[Clob]]("WORKFLOW_INPUTS")
 
-    def workflowOptions = column[Clob]("WORKFLOW_OPTIONS")
+    def workflowOptions = column[Option[Clob]]("WORKFLOW_OPTIONS")
 
-    def customLabels = column[Clob]("CUSTOM_LABELS")
+    def customLabels = column[Option[Clob]]("CUSTOM_LABELS")
 
     def workflowState = column[String]("WORKFLOW_STATE", O.Length(15))
 
diff --git a/database/sql/src/main/scala/cromwell/database/sql/SqlConverters.scala b/database/sql/src/main/scala/cromwell/database/sql/SqlConverters.scala
index 3498e3da8..3c6abd3ed 100644
--- a/database/sql/src/main/scala/cromwell/database/sql/SqlConverters.scala
+++ b/database/sql/src/main/scala/cromwell/database/sql/SqlConverters.scala
@@ -18,21 +18,37 @@ object SqlConverters {
     def toSystemTimestamp = Timestamp.valueOf(offsetDateTime.atZoneSameInstant(ZoneId.systemDefault).toLocalDateTime)
   }
 
-  implicit class ClobToRawString(val clob: Clob) extends AnyVal {
-    def toRawString: String = clob.getSubString(1, clob.length.toInt) // yes, it starts at 1
+  implicit class ClobOptionToRawString(val clobOption: Option[Clob]) extends AnyVal {
+    // yes, it starts at 1
+    def toRawStringOption: Option[String] = clobOption.map(clob => clob.getSubString(1, clob.length.toInt))
 
-    def parseSystemTimestamp: Timestamp = OffsetDateTime.parse(toRawString).toSystemTimestamp
+    def toRawString: String = toRawStringOption.getOrElse("")
+
+    def parseSystemTimestampOption: Option[Timestamp] = toRawStringOption map { rawString =>
+      OffsetDateTime.parse(rawString).toSystemTimestamp
+    }
+  }
+
+  implicit class StringOptionToClobOption(val strOption: Option[String]) extends AnyVal {
+    def toClob: Option[Clob] = strOption.flatMap(_.toClob)
   }
 
-  implicit class StringToClob(val str: String) extends AnyVal {
-    def toClob: Clob = new SerialClob(str.toCharArray)
+  implicit class StringToClobOption(val str: String) extends AnyVal {
+    def toClob: Option[Clob] = if (str.isEmpty) None else Option(new SerialClob(str.toCharArray))
+  }
+
+  implicit class BlobToBytes(val blobOption: Option[Blob]) extends AnyVal {
+    // yes, it starts at 1
+    def toBytesOption: Option[Array[Byte]] = blobOption.map(blob => blob.getBytes(1, blob.length.toInt))
+
+    def toBytes: Array[Byte] = toBytesOption.getOrElse(Array.empty)
   }
 
-  implicit class BlobToBytes(val blob: Blob) extends AnyVal {
-    def toBytes: Array[Byte] = blob.getBytes(1, blob.length.toInt) // yes, it starts at 1
+  implicit class BytesOptionToBlob(val bytesOption: Option[Array[Byte]]) extends AnyVal {
+    def toBlob: Option[Blob] = bytesOption.flatMap(_.toBlob)
   }
 
-  implicit class StringToBlob(val bytes: Array[Byte]) extends AnyVal {
-    def toBlob: Blob = new SerialBlob(bytes)
+  implicit class BytesToBlob(val bytes: Array[Byte]) extends AnyVal {
+    def toBlob: Option[Blob] = if (bytes.isEmpty) None else Option(new SerialBlob(bytes))
   }
 }
diff --git a/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingDetritusEntry.scala b/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingDetritusEntry.scala
index acbaeb876..0fd6d87cb 100644
--- a/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingDetritusEntry.scala
+++ b/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingDetritusEntry.scala
@@ -5,7 +5,7 @@ import java.sql.Clob
 case class CallCachingDetritusEntry
 (
   detritusKey: String,
-  detritusValue: Clob,
+  detritusValue: Option[Clob],
   callCachingEntryId: Option[Int] = None,
   callCachingDetritusEntryId: Option[Int] = None
 )
diff --git a/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingSimpletonEntry.scala b/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingSimpletonEntry.scala
index 148f45f6a..fb2627ba3 100644
--- a/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingSimpletonEntry.scala
+++ b/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingSimpletonEntry.scala
@@ -5,7 +5,7 @@ import java.sql.Clob
 case class CallCachingSimpletonEntry
 (
   simpletonKey: String,
-  simpletonValue: Clob,
+  simpletonValue: Option[Clob],
   wdlType: String,
   callCachingEntryId: Option[Int] = None,
   callCachingSimpletonEntryId: Option[Int] = None
diff --git a/database/sql/src/main/scala/cromwell/database/sql/tables/JobStoreSimpletonEntry.scala b/database/sql/src/main/scala/cromwell/database/sql/tables/JobStoreSimpletonEntry.scala
index 7989ed817..909dd17fe 100644
--- a/database/sql/src/main/scala/cromwell/database/sql/tables/JobStoreSimpletonEntry.scala
+++ b/database/sql/src/main/scala/cromwell/database/sql/tables/JobStoreSimpletonEntry.scala
@@ -5,7 +5,7 @@ import java.sql.Clob
 case class JobStoreSimpletonEntry
 (
   simpletonKey: String,
-  simpletonValue: Clob,
+  simpletonValue: Option[Clob],
   wdlType: String,
   jobStoreEntryId: Option[Int] = None,
   jobStoreSimpletonEntryId: Option[Int] = None
diff --git a/database/sql/src/main/scala/cromwell/database/sql/tables/WorkflowStoreEntry.scala b/database/sql/src/main/scala/cromwell/database/sql/tables/WorkflowStoreEntry.scala
index a6e2403bb..467c6589c 100644
--- a/database/sql/src/main/scala/cromwell/database/sql/tables/WorkflowStoreEntry.scala
+++ b/database/sql/src/main/scala/cromwell/database/sql/tables/WorkflowStoreEntry.scala
@@ -5,12 +5,12 @@ import java.sql.{Blob, Clob, Timestamp}
 case class WorkflowStoreEntry
 (
   workflowExecutionUuid: String,
-  workflowDefinition: Clob,
-  workflowInputs: Clob,
-  workflowOptions: Clob,
+  workflowDefinition: Option[Clob],
+  workflowInputs: Option[Clob],
+  workflowOptions: Option[Clob],
   workflowState: String,
   submissionTime: Timestamp,
   importsZip: Option[Blob],
-  customLabels: Clob,
+  customLabels: Option[Clob],
   workflowStoreEntryId: Option[Int] = None
 )
diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStore.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStore.scala
index afab96748..be17de643 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStore.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStore.scala
@@ -57,7 +57,7 @@ case class SqlWorkflowStore(sqlDatabase: WorkflowStoreSqlDatabase) extends Workf
       workflowStoreEntry.workflowInputs.toRawString,
       workflowStoreEntry.workflowOptions.toRawString,
       workflowStoreEntry.customLabels.toRawString,
-      workflowStoreEntry.importsZip.map(_.toBytes)
+      workflowStoreEntry.importsZip.toBytesOption
     )
     WorkflowToStart(
       WorkflowId.fromString(workflowStoreEntry.workflowExecutionUuid),
@@ -74,7 +74,7 @@ case class SqlWorkflowStore(sqlDatabase: WorkflowStoreSqlDatabase) extends Workf
       customLabels = workflowSourceFiles.labelsJson.toClob,
       workflowState = WorkflowStoreState.Submitted.toString,
       submissionTime = OffsetDateTime.now.toSystemTimestamp,
-      importsZip = workflowSourceFiles.importsZipFileOption.map(_.toBlob)
+      importsZip = workflowSourceFiles.importsZipFileOption.toBlob
     )
   }
 
diff --git a/engine/src/main/scala/cromwell/jobstore/SqlJobStore.scala b/engine/src/main/scala/cromwell/jobstore/SqlJobStore.scala
index d510c3873..de7eef88a 100644
--- a/engine/src/main/scala/cromwell/jobstore/SqlJobStore.scala
+++ b/engine/src/main/scala/cromwell/jobstore/SqlJobStore.scala
@@ -49,7 +49,7 @@ class SqlJobStore(sqlDatabase: JobStoreSqlDatabase) extends JobStore {
           key.attempt,
           jobSuccessful = false,
           returnCode,
-          Option(throwable.getMessage.toClob),
+          throwable.getMessage.toClob,
           Option(retryable))
         JobStoreJoin(entry, Seq.empty)
     }
@@ -64,8 +64,9 @@ class SqlJobStore(sqlDatabase: JobStoreSqlDatabase) extends JobStore {
             val simpletons = simpletonEntries map toSimpleton
             val jobOutputs = WdlValueBuilder.toJobOutputs(taskOutputs, simpletons)
             JobResultSuccess(returnCode, jobOutputs)
-          case JobStoreEntry(_, _, _, _, false, returnCode, Some(exceptionMessage), Some(retryable), _) =>
-            JobResultFailure(returnCode, JobAlreadyFailedInJobStore(jobStoreKey.tag, exceptionMessage.toRawString),
+          case JobStoreEntry(_, _, _, _, false, returnCode, Some(_), Some(retryable), _) =>
+            JobResultFailure(returnCode,
+              JobAlreadyFailedInJobStore(jobStoreKey.tag, entry.exceptionMessage.toRawString),
               retryable)
           case bad =>
             throw new Exception(s"Invalid contents of JobStore table: $bad")
diff --git a/services/src/main/scala/cromwell/services/metadata/impl/MetadataDatabaseAccess.scala b/services/src/main/scala/cromwell/services/metadata/impl/MetadataDatabaseAccess.scala
index 18e8689de..066c5eeab 100644
--- a/services/src/main/scala/cromwell/services/metadata/impl/MetadataDatabaseAccess.scala
+++ b/services/src/main/scala/cromwell/services/metadata/impl/MetadataDatabaseAccess.scala
@@ -41,12 +41,12 @@ object MetadataDatabaseAccess {
     def toSummary: WorkflowMetadataSummaryEntry = {
       val base = baseSummary(metadatum.workflowExecutionUuid)
       metadatum.metadataKey match {
-        case WorkflowMetadataKeys.Name => base.copy(workflowName = metadatum.metadataValue.map(_.toRawString))
-        case WorkflowMetadataKeys.Status => base.copy(workflowStatus = metadatum.metadataValue.map(_.toRawString))
+        case WorkflowMetadataKeys.Name => base.copy(workflowName = metadatum.metadataValue.toRawStringOption)
+        case WorkflowMetadataKeys.Status => base.copy(workflowStatus = metadatum.metadataValue.toRawStringOption)
         case WorkflowMetadataKeys.StartTime =>
-          base.copy(startTimestamp = metadatum.metadataValue.map(_.parseSystemTimestamp))
+          base.copy(startTimestamp = metadatum.metadataValue.parseSystemTimestampOption)
         case WorkflowMetadataKeys.EndTime =>
-          base.copy(endTimestamp = metadatum.metadataValue.map(_.parseSystemTimestamp))
+          base.copy(endTimestamp = metadatum.metadataValue.parseSystemTimestampOption)
       }
     }
   }
@@ -77,7 +77,7 @@ trait MetadataDatabaseAccess {
       val valueType = metadataEvent.value map { _.valueType.typeName }
       val jobKey = key.jobKey map { jk => (jk.callFqn, jk.index, jk.attempt) }
       MetadataEntry(workflowUuid, jobKey.map(_._1), jobKey.flatMap(_._2), jobKey.map(_._3),
-        key.key, value.map(_.toClob), valueType, timestamp)
+        key.key, value.toClob, valueType, timestamp)
     }
     databaseInterface.addMetadataEntries(metadata)
   }
@@ -91,10 +91,9 @@ trait MetadataDatabaseAccess {
       } yield MetadataJobKey(callFqn, m.jobIndex, attempt)
 
       val key = MetadataKey(workflowId, metadataJobKey, m.metadataKey)
-      val value =  for {
-        mValue <- m.metadataValue
-        mType <- m.metadataValueType
-      } yield MetadataValue(mValue.toRawString, MetadataType.fromString(mType))
+      val value = m.metadataValueType.map(mType =>
+        MetadataValue(m.metadataValue.toRawString, MetadataType.fromString(mType))
+      )
 
       MetadataEvent(key, value, m.metadataTimestamp.toSystemOffsetDateTime)
     }
diff --git a/services/src/test/scala/cromwell/services/ServicesStoreSpec.scala b/services/src/test/scala/cromwell/services/ServicesStoreSpec.scala
index 5a44d3672..a23637974 100644
--- a/services/src/test/scala/cromwell/services/ServicesStoreSpec.scala
+++ b/services/src/test/scala/cromwell/services/ServicesStoreSpec.scala
@@ -1,7 +1,8 @@
 package cromwell.services
 
 import java.io.{ByteArrayOutputStream, PrintStream}
-import java.sql.Connection
+import java.sql.{Connection, JDBCType}
+import java.time.OffsetDateTime
 
 import better.files._
 import com.typesafe.config.ConfigFactory
@@ -9,8 +10,9 @@ import cromwell.core.Tags._
 import cromwell.core.WorkflowId
 import cromwell.database.migration.liquibase.LiquibaseUtils
 import cromwell.database.slick.SlickDatabase
+import cromwell.database.sql.SqlConverters._
 import cromwell.database.sql.joins.JobStoreJoin
-import cromwell.database.sql.tables.JobStoreEntry
+import cromwell.database.sql.tables.{JobStoreEntry, JobStoreSimpletonEntry, WorkflowStoreEntry}
 import liquibase.diff.DiffResult
 import liquibase.diff.output.DiffOutputControl
 import liquibase.diff.output.changelog.DiffToChangeLog
@@ -153,8 +155,8 @@ class ServicesStoreSpec extends FlatSpec with Matchers with ScalaFutures with St
         val expected = s"PK_${primaryKey.table.name}"
         if (actual != expected) {
           misnamed :+=
-            s"""|PrimaryKey: $actual
-                |Should Be:  $expected
+            s"""|  PrimaryKey: $actual
+                |  Should be:  $expected
                 |""".stripMargin
         }
       }
@@ -164,8 +166,8 @@ class ServicesStoreSpec extends FlatSpec with Matchers with ScalaFutures with St
         val expected = s"FK_${foreignKey.fkTable.name}_${foreignKey.fkColumn}"
         if (actual != expected) {
           misnamed :+=
-            s"""|ForeignKey: $actual
-                |Should Be:  $expected
+            s"""|  ForeignKey: $actual
+                |  Should be:  $expected
                 |""".stripMargin
         }
       }
@@ -183,35 +185,56 @@ class ServicesStoreSpec extends FlatSpec with Matchers with ScalaFutures with St
 
           if (actual != expected) {
             misnamed :+=
-              s"""|Index:     $actual
-                  |Should Be: $expected
+              s"""|  Index:     $actual
+                  |  Should be: $expected
                   |""".stripMargin
           }
       }
 
-      if (misnamed.nonEmpty) {
-        fail(misnamed.mkString(s"The following items are misnamed in $schemaManager:\n", "\n", ""))
-      }
-
       var missing = Seq.empty[String]
 
       schemaMetadata.columns foreach { column =>
         if (!schemaMetadata.existsTableItem(column)) {
-          missing :+= s"${tableClassName(column.tableName)}.${column.itemName}"
+          missing :+= s"  ${tableClassName(column.tableName)}.${column.itemName}"
         }
       }
 
       schemaMetadata.slickItems foreach { databaseItem =>
         if (!schemaMetadata.existsSlickMapping(databaseItem)) {
-          missing :+= s"${slickClassName(databaseItem.tableName)}.${databaseItem.itemName}"
+          missing :+= s"  ${slickClassName(databaseItem.tableName)}.${databaseItem.itemName}"
+        }
+      }
+
+      var nullTyped = Seq.empty[String]
+
+      schemaMetadata.columnMetadata foreach { column =>
+        if (column.isNullable == Option(false)) {
+          val jdbcType = JDBCType.valueOf(column.sqlType)
+          if (jdbcType == JDBCType.BLOB || jdbcType == JDBCType.CLOB) {
+            nullTyped :+= s"  $jdbcType column ${column.table.name}.${column.name}"
+          }
         }
       }
 
-      if (missing.nonEmpty) {
-        fail(missing.mkString(
-          s"Based on the schema in $schemaManager, please ensure that the following tables/columns exist:\n  ",
-          "\n  ",
-          ""))
+      if (missing.nonEmpty || misnamed.nonEmpty || nullTyped.nonEmpty) {
+        var failMessage = ""
+
+        if (misnamed.nonEmpty) {
+          failMessage += misnamed.mkString(s"The following items are misnamed in $schemaManager:\n", "\n", "\n")
+        }
+
+        if (missing.nonEmpty) {
+          failMessage += missing.mkString(
+            s"Based on the schema in $schemaManager, please ensure that the following tables/columns exist:\n",
+            "\n", "\n")
+        }
+
+        if (nullTyped.nonEmpty) {
+          failMessage += nullTyped.mkString(
+            s"The following columns should not be nullable due to incompatibilities with MySQL:\n", "\n", "\n")
+        }
+
+        fail(failMessage)
       }
     }
   }
@@ -247,6 +270,75 @@ class ServicesStoreSpec extends FlatSpec with Matchers with ScalaFutures with St
       } yield ()).futureValue
     }
 
+    it should "store and retrieve empty clobs" taggedAs DbmsTest in {
+      val workflowUuid = WorkflowId.randomId().toString
+      val callFqn = "call.fqn"
+      val jobIndex = 1
+      val jobAttempt = 1
+      val jobSuccessful = false
+      val jobStoreEntry = JobStoreEntry(workflowUuid, callFqn, jobIndex, jobAttempt, jobSuccessful, None, None, None)
+      val jobStoreSimpletonEntries = Seq(
+        JobStoreSimpletonEntry("empty", "".toClob, "WdlString"),
+        JobStoreSimpletonEntry("aEntry", "a".toClob, "WdlString")
+      )
+      val jobStoreJoins = Seq(JobStoreJoin(jobStoreEntry, jobStoreSimpletonEntries))
+
+      val future = for {
+        _ <- dataAccess.addJobStores(jobStoreJoins)
+        queried <- dataAccess.queryJobStores(workflowUuid, callFqn, jobIndex, jobAttempt)
+        _ = {
+          val jobStoreJoin = queried.get
+          jobStoreJoin.jobStoreEntry.workflowExecutionUuid should be(workflowUuid)
+
+          val emptyEntry = jobStoreJoin.jobStoreSimpletonEntries.find(_.simpletonKey == "empty").get
+          emptyEntry.simpletonValue.toRawString should be("")
+
+          val aEntry = jobStoreJoin.jobStoreSimpletonEntries.find(_.simpletonKey == "aEntry").get
+          aEntry.simpletonValue.toRawString should be("a")
+        }
+        _ <- dataAccess.removeJobStores(Seq(workflowUuid))
+      } yield ()
+      future.futureValue
+    }
+
+    it should "store and retrieve empty blobs" taggedAs DbmsTest in {
+      val testWorkflowState = "Testing"
+
+      val emptyWorkflowUuid = WorkflowId.randomId().toString
+      val emptyWorkflowStoreEntry = WorkflowStoreEntry(emptyWorkflowUuid, "{}".toClob, "{}".toClob, "{}".toClob,
+        testWorkflowState, OffsetDateTime.now.toSystemTimestamp, Option(Array.empty[Byte]).toBlob, "{}".toClob)
+
+      val noneWorkflowUuid = WorkflowId.randomId().toString
+      val noneWorkflowStoreEntry = WorkflowStoreEntry(noneWorkflowUuid, "{}".toClob, "{}".toClob, "{}".toClob,
+        testWorkflowState, OffsetDateTime.now.toSystemTimestamp, None, "{}".toClob)
+
+      val aByte = 'a'.toByte
+      val aByteWorkflowUuid = WorkflowId.randomId().toString
+      val aByteWorkflowStoreEntry = WorkflowStoreEntry(aByteWorkflowUuid, "{}".toClob, "{}".toClob, "{}".toClob,
+        testWorkflowState, OffsetDateTime.now.toSystemTimestamp, Option(Array(aByte)).toBlob, "{}".toClob)
+
+      val workflowStoreEntries = Seq(emptyWorkflowStoreEntry, noneWorkflowStoreEntry, aByteWorkflowStoreEntry)
+
+      val future = for {
+        _ <- dataAccess.addWorkflowStoreEntries(workflowStoreEntries)
+        queried <- dataAccess.queryWorkflowStoreEntries(Int.MaxValue, testWorkflowState, testWorkflowState)
+        _ = {
+          val emptyEntry = queried.find(_.workflowExecutionUuid == emptyWorkflowUuid).get
+          emptyEntry.importsZip.toBytesOption should be(None)
+
+          val noneEntry = queried.find(_.workflowExecutionUuid == noneWorkflowUuid).get
+          noneEntry.importsZip.toBytesOption should be(None)
+
+          val aByteEntry = queried.find(_.workflowExecutionUuid == aByteWorkflowUuid).get
+          aByteEntry.importsZip.toBytesOption.get.toSeq should be(Seq(aByte))
+        }
+        _ <- dataAccess.removeWorkflowStoreEntry(emptyWorkflowUuid)
+        _ <- dataAccess.removeWorkflowStoreEntry(noneWorkflowUuid)
+        _ <- dataAccess.removeWorkflowStoreEntry(aByteWorkflowUuid)
+      } yield ()
+      future.futureValue
+    }
+
     it should "close the database" taggedAs DbmsTest in {
       dataAccess.close()
     }

From f05c6ff96675648a98e5533801b6b9096891f811 Mon Sep 17 00:00:00 2001
From: Jeff Gentry <geoffjentry@gmail.com>
Date: Wed, 1 Feb 2017 12:20:43 -0500
Subject: [PATCH 36/84] unhardcode insert batch size (#1904)

---
 CHANGELOG.md                                                           | 1 +
 README.md                                                              | 2 ++
 core/src/main/resources/reference.conf                                 | 3 +++
 .../src/main/scala/cromwell/database/slick/MetadataSlickDatabase.scala | 2 +-
 .../sql/src/main/scala/cromwell/database/slick/SlickDatabase.scala     | 2 ++
 5 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7d8d52a7b..7ec14e5cf 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,7 @@
 
 ## 25
 
+* Added a field `insert-batch-size` to the `database` stanza which defines how many values from a batch insert will be processed at a time. This value defaults to 2000. 
 * Cromwell's WDL parser now recognizes empty array literals correctly, e.g. `Array[String] emptyArray = []`.
 * Moved the config value `services.MetadataService.metadata-summary-refresh-interval` to `services.MetadataService.config.metadata-summary-refresh-interval`
 * Cromwell now applies default labels automatically to JES pipeline runs.
diff --git a/README.md b/README.md
index 554dcbb3e..465d72cb8 100644
--- a/README.md
+++ b/README.md
@@ -477,6 +477,8 @@ database {
 }
 ```
 
+By default batch inserts will be processed in blocks of 2000. To modify this value add the field `insert-batch-size` to the `database` stanza.
+
 ## SIGINT abort handler
 
 For backends that support aborting task invocations, Cromwell can be configured to automatically try to abort all currently running calls (and set their status to `Aborted`) when a SIGINT is sent to the Cromwell process.  To turn this feature on, set the configuration option
diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf
index bf98ab431..d4e278413 100644
--- a/core/src/main/resources/reference.conf
+++ b/core/src/main/resources/reference.conf
@@ -434,6 +434,9 @@ database {
   #  connectionTimeout = 5000
   #}
 
+  # For batch inserts the number of inserts to send to the DB at a time
+  # insert-batch-size = 2000
+
   migration {
     # For databases with a very large number of symbols, selecting all the rows at once can generate a variety of
     # problems. In order to avoid any issue, the selection is paginated. This value sets how many rows should be
diff --git a/database/sql/src/main/scala/cromwell/database/slick/MetadataSlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/MetadataSlickDatabase.scala
index 521581011..fd22db710 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/MetadataSlickDatabase.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/MetadataSlickDatabase.scala
@@ -15,7 +15,7 @@ trait MetadataSlickDatabase extends MetadataSqlDatabase {
 
   override def addMetadataEntries(metadataEntries: Iterable[MetadataEntry])
                                  (implicit ec: ExecutionContext): Future[Unit] = {
-    val action = DBIO.seq(metadataEntries.grouped(2000).map(dataAccess.metadataEntries ++= _).toSeq:_*)
+    val action = DBIO.seq(metadataEntries.grouped(insertBatchSize).map(dataAccess.metadataEntries ++= _).toSeq:_*)
     runTransaction(action).map(_ => ())
   }
 
diff --git a/database/sql/src/main/scala/cromwell/database/slick/SlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/SlickDatabase.scala
index 173de0e89..860ad65b3 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/SlickDatabase.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/SlickDatabase.scala
@@ -100,6 +100,8 @@ class SlickDatabase(override val originalDatabaseConfig: Config) extends SqlData
     actionThreadPool, database.executor.executionContext.reportFailure
   )
 
+  protected[this] lazy val insertBatchSize = databaseConfig.as[Option[Int]]("insert-batch-size").getOrElse(2000)
+
   protected[this] lazy val useSlickUpserts =
     dataAccess.driver.capabilities.contains(JdbcProfile.capabilities.insertOrUpdate)
 

From 1cdac8e8a9af0e2df51b30d2531eeecc8b27fd94 Mon Sep 17 00:00:00 2001
From: Chris Llanwarne <cjllanwarne@gmail.com>
Date: Mon, 30 Jan 2017 15:00:25 -0500
Subject: [PATCH 37/84] Fixup the invalidate_bad_cache test

---
 .../engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala
index e68ea38a2..f25fc42a0 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala
@@ -129,7 +129,7 @@ class EngineJobExecutionActor(replyTo: ActorRef,
       log.debug("Cache miss for job {}", jobTag)
       runJob(data)
     case Event(hit: CacheHit, data: ResponsePendingData) =>
-      fetchCachedResults(jobDescriptorKey.call.task.outputs, hit.cacheResultIds.head, data)
+      fetchCachedResults(jobDescriptorKey.call.task.outputs, hit.cacheResultIds.head, data.copy(cacheHit = Option(hit)))
     case Event(HashError(t), data: ResponsePendingData) =>
       writeToMetadata(Map(callCachingReadResultMetadataKey -> s"Hashing Error: ${t.getMessage}"))
       disableCallCaching(t)

From 7c52320b3844fb83959a784a16c613c62b8bec1c Mon Sep 17 00:00:00 2001
From: Kate Voss <kvoss@broadinstitute.org>
Date: Thu, 2 Feb 2017 11:18:09 -0500
Subject: [PATCH 38/84] Added concurrent_job_limit (#1941) Closes #1841

* kv_doc_con-job: Added concurrent_job_limit to local and SGE

* Fixing things so Khalid is happy
---
 core/src/main/resources/reference.conf | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf
index d4e278413..d2aad5422 100644
--- a/core/src/main/resources/reference.conf
+++ b/core/src/main/resources/reference.conf
@@ -162,6 +162,10 @@ backend {
     Local {
       actor-factory = "cromwell.backend.impl.sfs.config.ConfigBackendLifecycleActorFactory"
       config {
+
+        # Limits the number of concurrent jobs
+        #concurrent-job-limit = 5
+
         run-in-background = true
         runtime-attributes = """
         String? docker
@@ -205,6 +209,10 @@ backend {
     #SGE {
     #  actor-factory = "cromwell.backend.impl.sfs.config.ConfigBackendLifecycleActorFactory"
     #  config {
+    #
+    #    # Limits the number of concurrent jobs
+    #    concurrent-job-limit = 5
+    #
     #    runtime-attributes = """
     #    Int cpu = 1
     #    Float? memory_gb

From f47997dbd7dd46fb590c7d69e0d8aebf3c141cf8 Mon Sep 17 00:00:00 2001
From: Chris Llanwarne <cjllanwarne@gmail.com>
Date: Wed, 1 Feb 2017 18:12:36 -0500
Subject: [PATCH 39/84] De-dupe EJEA events if call cache writing is enabled

---
 .../engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala
index f25fc42a0..796def7b2 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala
@@ -218,6 +218,9 @@ class EngineJobExecutionActor(replyTo: ActorRef,
       eventList ++= response.executionEvents
       log.debug(s"Got job result for {}, awaiting hashes", jobTag)
       stay using data.withSuccessResponse(response)
+    case Event(response: JobSucceededResponse, data: ResponsePendingData) =>
+      eventList ++= response.executionEvents
+      saveJobCompletionToJobStore(data.withSuccessResponse(response))
     case Event(response: BackendJobExecutionResponse, data: ResponsePendingData) =>
       saveJobCompletionToJobStore(data.withResponse(response))
   }
@@ -409,7 +412,6 @@ class EngineJobExecutionActor(replyTo: ActorRef,
   private def saveJobCompletionToJobStore(updatedData: ResponseData) = {
     updatedData.response match {
       case JobSucceededResponse(jobKey: BackendJobDescriptorKey, returnCode: Option[Int], jobOutputs: CallOutputs, _, executionEvents) =>
-        eventList ++= executionEvents
         saveSuccessfulJobResults(jobKey, returnCode, jobOutputs)
       case AbortedResponse(jobKey: BackendJobDescriptorKey) =>
         log.debug("{}: Won't save aborted job response to JobStore", jobTag)

From 5d06f1255fd4a9fba7632c7eeb613b81b17baee5 Mon Sep 17 00:00:00 2001
From: Ruchi <rmunshi@broadinstitute.org>
Date: Fri, 3 Feb 2017 12:40:35 -0500
Subject: [PATCH 40/84] Fix Travis Encryptions (#1953)

* try new travis run

* new credentials

* update json path

* updated jes.conf

* remove added newline
---
 src/bin/travis/resources/jesConf.tar.enc | Bin 20496 -> 16400 bytes
 src/bin/travis/testCentaurJes.sh         |   4 ++--
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/bin/travis/resources/jesConf.tar.enc b/src/bin/travis/resources/jesConf.tar.enc
index 3aa3b4af0c5aa66751cd9b8e32fb0e0018acf4ee..a84a42c63fd0d3e15218cf77c2f27cbce9b621d8 100644
GIT binary patch
literal 16400
zcmV+rK<~eit>Gi(NW3ZyS$P(Ixa`hpK<xgE3k6e<SwAG1nINM;y_fdL6M_ZpwBg#5
zi?_k`hs0>wI@@8DZlWgzhA6@IU$ozwfe`nZp0S8-?G`8-FL&GP2L=B5O=$g4eQ#!^
z5AY<M0M@qg?l8aDLu<L3*>zVFc**Ml=_PKQjLjlz5|lRPl*l=bc(k`tzQM%ifBB~W
z2;IY#^1=@UR0eHrmH;Sccx^ges{WnBc$&f1Eshw3h`uXiSmu%w<nAGTomz94lmD)L
z?Y0gspT}Blc>6rL^rmAEOt%BQHhdaay5u)SDo7AU8fl!>)Oud(AS}LHn(33e;40Y5
z9(7sv3&)25+Wk;@q3sB$lt+r1b_UD8K_vYMq@I7m+|akRG*g;(8HMT}W_r>W6f_-|
z9rfj8i$}e2(r4w@E9PvCKy|oA=j+^SND$k1QbX<r5La?n$p2)w?N*Vi!uKXBL6(!|
zVLGa#{eni#vYZ%<5KlNC!RW6HNsIBO=PF+B?B4y&HOXF(0NiN_v+pv<GXPjs{N^fS
zqm-u06a_}v(cV=<Rp``*-;q@9c_&U&iR0vor<F+ICS1yACtpQz);gerE{%Rgi1#X7
z$+0%wb-2{h7qV5~mQVG+k$<&gSwSLkBPP9Gh{&F-ONFN{!i`yjf!iX(;lCB8^Rxfg
zNn1_?xZ7XVy2g^m9uxrr8UWk&&S4Q~7}DOm^udvY>vW9p)?wEmZB~d%gQL*NCAQs|
zw`};ncv|=Usy1p$-<pwF>3Ua9bS1~go79*90u{F~kO$RyXPE9u)BQkV+y7Meh+7Z+
z0Ww_{l-ar7ZTMzK-Y@QjG%PEk`!1*L7LFc0T_59#jR$$hi5Gk+y(>!G2vrLLDzjCZ
zuTuBo#kl}{M*_Utj&`B*-sJ8K$bAJ7Df2fp6_8-xXj)d;#Sej=lR?6K1YGkyaWm`D
z&qlF6<8Nb8W_&!XK5q~rA4}L&p}Y#MNq-JyZ4i;as2QsvVMc5zfOH5EJ_70CLTuZ)
zw~Qenq?LP(C!04Z#b(ye0mbQhl_Mx;(}Y&D26cL7d2AeRu}K!vC`glAKLtj@>iLb&
z5Fl<!xZ8oGORLD6J=y)`>!*mpJA{L5dCfvyT~mC4_gRGk;VliGo`8-H1|*=g@=NIr
z1cuMEJkN;+=g+ye1+jCM5WR;pJ-={K!)MBOCazkXBsesXse7X6*7ZPMWhTlQIYULA
zy!PPY{0ll7W743nbQCkvNDTgR(veP+7xvW)?3=G|tR@I!Xz~ISP>&h4xA8#>*Mq-$
zYa4`uw2AZ}=YCWDIx_>`-WIokFwUZXj;pu8I1JF#AgjaT*j)YNn{Za)7)$92ywl#N
zpaCo>pS^)W41hi`5;6lvU>Ckmt!OX!DfF~uMBBTs;hy+c<5oqyHK{bN=^zPe3-C(1
z>*2=D*h$(R^2g#_bBl)bs6Q<gH-H~s7T+is62xAS%V*}eG6wsM#fBqCp9>Zur2FGq
z`H9(4k(eT4&%@}X3o0=V>@Xldy-uOE$d!u%5LO<q2q1sxRjsBX66Xg;g0?8optH>j
zUgLYIR+(9G#3$`yV=n-Kz+M58V|z<Jr9PYzoaCLZ$sy^@>{y*Ke!7ejGkfD9U3~kS
zJ$R9Fl}=#tup(ejBfjfM@Od%mp|<_MM&Z^IK~8l@oIiE5?{^n;3wE=D+v<siOsJ)t
zw;}u?g)qRj50L;QwlV?@)crugix$EWY9tH3a_}=2>QVgJQbzy%?IaZkc%avjV{e+1
z1j~j>NPiYwk}jOd>+E+=pL=*jL*^tQo;8gJ5G5V&zlbKlqQvjnN@b0l0%c`i2Q-Ch
z3;p?ko8+m_FK4&wm7I;FNyS?;^xC@2@#+V`y}f@FtPeyZ8HZ66p0*#&Kg=A2C;aZN
zb-o54-RRR+U@_=IVY)RGWoR*ysKLmNM^_z|s2eqGz@VP?H9-YFq`(Pt%m%W9-gfr!
z0MSB9YAUS2N^qv&uN^|(uhWn&$rQ1U0ni0ut$oHII+HmRKh;QceCgM7kiFtso?&+@
z3<|-Wd<g~*;Vz@-cR$kP{P40yo$iMFm24wlj+(>~VSHk3eTa1<_AoVD!6g6}*>RM3
z7F*SZra6nqesj7ZxTm@4M`%}PGYK^UMaN}Y<Zdv!sSsRX*R}x{jp2-x5Kkx%an0ox
zkduZTuFjZ&c41ZlV!^5T8+@14oih$0Z9PGSXz`a#f704iD@!_*I}2X_3deumJB%<N
zF*2LdxQP$e;_c+_gNx8+tuSjwe1QXa!977@U;;HSlK%Pqr8~gja$tAyIh*}7o&6KL
zzPL-@)_Z(3Ja8`B@}ZEBNV@r%WD{2`U36H4gXB3=fyG`ce2ZhSo#RzrW2Yf%dMu_U
z@V^yrc13Hye4x%LFG0M}#LlK0^Z1*hJkDKZV1?HfS?gF_+kB7Bt@Cv9VB&Aji4%PY
z($rCJbDn&ddrJ@8=iX1tt28(Ws6f)iv8B%k*z}h^(9|iB#dF3YXaaL+vGz9*Q^5d~
zjaZO8F|t11&n>FWgs*d70O=cl4}D|ATR*dForB9GKIjO3z4bfRWk4T(R7r6m0^LD^
zRoiGwhW`>xrDU;VM^KtK3Ka7wE#Ca`MU_F~qd#1PmxG&*fNOrxJOwvxt%q{iW|NMO
zxTD-!FAeMfZT>yygNoV4GL1a^8B8Y-gTedBfU=88`j0NM&QQ5uTlQ(xOUiN9oR{s8
z%X!61R-&vixjs8T-IM&bRF~=6o9PPQR561vrv9wCn|e}sP-*b6q6c!f<nr4xHnc=e
z$uI}9vp{0fK<Y47<0<?u<UxxDK$+vzE@~_n`QtDjO|0tQ+1B*GBQlQn9+f;oQa7VU
z{W-v&dzVc3q4J^>&OYQL@YPN~t8+wdPux_%VTCaj#WEgBh=ZH7vU&=RnmV#1D7SA0
zRN=N%ttQfE@GFs~D_zeL*v8dZc|)0g(6qY8CFGVPaIdD1KxlS(70oF&kY!TheI{jY
zx(v5!9|-5sqeb25&#;J(R~Qmg7euHP>hSk<3lR4U*cKt=Z@6UxUWF|xCXeGJY#AHe
zym;7n(G8Ai1YtkY(Nl*?r*LsEtU8JC{gYb6os`uIm&&TiPn!=;d16b~WLxnQF+Vav
zC(h00GI;0FeyOs-ZZ`OXpNWDF`)s3liYCL!;;Xdn#CP6&r$8;|xq7MPDLZHnWOaS$
zcNn1jJOl5;QMs(r&Z_V+nXrm^DZUYtq<#r6@1IMa@v6988J?3uYB4&0Ehun#r<qOK
ztzsmwJ?C;-;=^?7El&WU!jZhIfTDx_=AloQoqHf=HN+NmN{Q)8l&-^HE+^+O(7*|^
zVr0!0Kn}t3R1!cEQUG=Zu~C+<rI>}}2Pz9qCK&5QZ!=nlVr52z%|D=<%njG&ZM<-Z
zkVNnJu=nVDCA2he5mK<K{(q_Ne*({!HcET)Tt<guMAGvcDu)%;rfm7g1*E}hFM6F_
zTUheE7WVU0LUCFUkVE0gswYmBjs>#CO;0F{hx~JW3w!Zj&tDW7cIBX8fWo&|V9dzd
z#3YLPTjO<xV$Xnl91yw(xa2uQ>djNz!z97Q?_e;v3@0wG<`ZZ%d$iY9DR5IhSkBI3
zNq@{0I~<*_P>_lfv-mEki%p@W2|?i`av`Y$0+9s%4(TLx;Q14gqDb{0Xq!I`{~UW?
z1s-QZ9=;<Gec-$`&jvTqt~uJn?=9`SM278iU}?>4_l8#jGdC#{5aiO1>AWuZAqv`Z
z6rI*arFG$WHr$WZ8}?E^Se`30KR-{Iw-sdf!-}B<X}v0_GMgm{=+Z$CZuZ6mA8R|l
zTXDI^q-4<Iv+IU@CLgK2hK^2%HF^mq7J~w4nfl~8i#V;n*0dIPyb^X9TRXbRf@a4J
z#;a(oGb;CxfaOtyv6FeLLhnxj0JR+lP4p_w;Y4>9%z#d)p&EK`>6;RTJW0<r>WBTH
z;Qm=$^huCxntZ#VSsVry^5slSbRIqj*W`!+a*FBD`{=*mCuH0~03XYNIFKHn-kr?R
zxhoyemc<db%FQ@?wgzGJ0aN<eTLHA#tAG!vECin(QD23z>i{b7HM0mV#2Jkg1uH&U
zWX=$cbT^i@1C%#lXv{F-4ek2ancz_Zf{93h@yPpW*Ml16fZ{5ISg2G&^!N30@5Bb3
z;R5?^E5lYPQ#;U2POoCFRV0bI^orm=@X4A^gGAwIb{jog)v&`G>q$~Wb8Z4<kC9PV
zU%gxD4C76Suq~kTT`9GQp;Z{!&*>%OG}ecljSW7-dpOCuIRy#J#o49*>)?{GVJJyG
zrE1B>tv?++b^~nH#ByjZ-Yw~nuw7-rLZjKRQbNF9lLWI+H?0&4lhGY!2qc+a3^ahX
z_D1_KwVt6Dj1Z*QJ`T3zeE4o4eo48&(%I+G=v==+AlJoDZz^k46-hM#pIlKun;oAm
zfvJN@(5cOt%-p(#K!Y$tLbG$=|F{d|<<BGuhaB-SI`Rzq2T_?GyHaJYI;1KA!H;n$
zyk@GgG)kz@<X@g60P83oguMsp0_IwBBRM5<;h1Y4I|xQw9}y(29fEBHYZ){kcOnjl
zZJQrbcI-1}Qlr1Wc?*<%DR8KERFCs>A-l7GWUUg0j_mGXc{7IHD-KNhO_BT+eDS3Q
z#&t7Lr9QaMpGgX*aUJpZ(j-tpLLA2<mxRdfke~}pJ_?D`^U~@UnU+4gnVL7us(!1K
zj~!!<F+McSbbzJ^@x9kqaSC0D0mE#cbM;ut+o`-8)}sH<K4U5W?JE>h6@F>^u%l6(
z3}{Zx;jzunEZ;!bv?R^l1^mS1OD$G8={=Yj(OrCPeT~b(Uk4nkseu0WduZ75%Y3HB
zl0B)^{+jq>YJhrR^>7)ML<S?s8(84wo|O|e&Yledz#2-322G7MG8q2$RZC_zbese9
z#8|~7V(<cdlv7OutBZdDYKb}V(5=COFm$E_NqZ5@n4A{J&Swno=5P2dvOU>ow(N$H
zsru?uSg@aPJMn6p<>c~Q{~g1xGvM_D)bOrUx=ksDuCi6p4@{OHO}NWM%!4M7f-s$)
z@BV{Z#oGa@nKEwW{qawa+6@yaueI)b^WFES(&RmsaQnCMw{_XLERv&gj3K@{3xQli
z(s;Ov^?Bxzzf2z?rtP^Iaa^<i2M|1=5TYI<8Y}bjHT;2+JZS;F!HLbZqxY9;{|va;
zup{^?!?M6LMKA5vSQ^BNGn`q$p%Cro=jQW~l_0_;F<9*QxT`M1Hxo&60;l@iZPSMd
z@jVYLct_b&;Q2=pA=>}`iuSUbe9@2AoCG9`D7_Oy!08s@YM3>ujB5jcS%ORlO|_bp
zVG3_2vuTci2W;K?E+cV}Ca%l2#T{D$?#Xz~0w3ZRX8p4Gx~bBWhc~gMNtmx0{sM60
zZf<(V0VJluz(5ygI;<Ikh)(8DOm<lj)d#g2dlz*iCz*eSi|B_sePybW8pR4l4hUNT
zF%F86_arI`k|W)0Fr7heebHS8*2qd0(R9|^%NeIRWpTUtEUnYh(^k(`U}Q0obJPzj
z7S4d;CqETZe*XzSvre9)y?PG*C3odUo?9s^$HHg6iz0<95{=cos6RD3Xc~mYnsA9}
zA(S*@>pcdGc$TT-R;KPfETGT$Y`n2?14LJR?emVwqB!7w2b9Q2zR;xOA%B;@kV8hn
zr_zmJY9S}Nm|K_V!YBb@TM<~_52^lR0|>n|$x3^U+&QyU%w`HU^H9|dwSD7B08j2$
z^<y`hwRWB$7J^kHyNM%mL}T8J#~f8pI@_<y-84FJqwg(+oSKdJQ3t~;W+2|4q+y3F
zQiDBz7%9v_VX@OfzDfF<^I!dDcrqJbw>(_L3P)5_Rq6O={$mG$sou}o%fomc&g^WW
z&<QcUMFLU+hZuxVN-L@fNBqRrB}I*9DB|(syNs{s9jOsKAIfXq`g;V~?rfI_MQ^p|
z;RYr84ESjC(VH$v#3vi6K&49wo+i$vFF_Sn!w{3%Z#Qz<_^ydeX~-rNolq{U4>tv{
zh1oV{OIcSs&hD0^8;;kLHu~&P4s<{jq39JgUX3AK17@xcFFXLa%4{uZrS~B^6-%p3
z+6*t2UB3l1pReJi<I?%hah7y2cltei=22vKZ8ltNk}CobP+vbf9&++W`CS$ZSok*&
zAnW(|dDS0S84cZ*@<%-;-*)km;@~MrY$Qz@r&e(AF`YClxqt&NM6_}dgibY3rf^!E
z+Zcy=a%f$3r7&Csj+?Sj%fT+UeHT~iY(F0YmPTi9ht9~-j;PX!k$v9?FL^(-910Jv
zZetPFy+rH9o<18$enI7W^YPxbPa^Vq!Oyy@&#vtkhTQy#)dFrMf6{XEf1HlL-$MMp
ziWv&DY@Wzd0IbAs+J&ee&rC5kJzK6)Cy(8TRspPxJay2l6xXYcgil@@&bawGm8cZn
zNg%_tL7(=`RT;9SYIX#2S=nZ!0S$8H2`6h7ZC-%X^CmH5RIDgH9FnoP+v8+lSoh-x
zPgwn)Hn53}Y7yMt&KV>5pH(<Qarn|n8+xBJ#=vjHIfrFQoOGeDH%;<C*klDJL5OUf
z>J{!EsnXA7`IE@Z-AlbFv$In67~B}QG;}A{e>(fsikcNILfI0APG2CqHu6adMoP4i
zh*wHqnVhMrTP#BgB^B@dk)1uTs1$AZRkCPjDOI2sVAJ1I7Y%KvVU1d_Y`2Cwm38%%
zDFJIS7~|OwZv+LTes^TEqNDp9-dBiWDh|R_ttji7ZQHSi#+qQKRVBl&|6vzI5C4?{
z$+g%s{F`>ESPZyh@DKLpNATeN9*RDh>V_h4?y1MH=n1CX#;18zy^lBESr6E<op^Q)
zw^2Kr(NuYWn_DrY1Ho^0$){=k{8H;I#pJ#mU_wW`3J*>4fvgIkITduA#cN=QCWk+l
zSkA%DC%|t=f+o1N!7-7!c2W*51~Z7Zt(*+t%d%QDTD$~{JPIxud$nW+zh>p+4^p`N
zf&n}+F2x1u|EWy~chdm@h^x+3;cQEh0<x-PftQea7uV+uc50L%!#vq%#0N<X&f!EB
zNY>t73>OH}zFz~z5iAQ9w|$?&e`Ip9!sFgBUuY%&wL5ISx;}-Qrpj`bWvIYds?>9h
zXW8N3sg^cn(v7#(p_{JbFQTn=PAPtGut@<En#z|R-mPDAclhu8?qKE9zPtoVI*rZh
zNvqmPQm9E5%7q@~wDtzGxH9YI<x(<G(kVcd(S!XgJ`rV~+>jFmHse9z<(`ewrM#u3
z(vyLyTJ%6I{#tT)h=VRdJ{|GOeSs_~0ikfWlZ@-%s3x?e3t-6fC+aqz0!co03Z5Fx
zkUF!KG6jfIMVdXm6wnhqFQ6Dn#)x#lc8v{7q@zfiv8^n~VD}o#sqwi>KaZR7x^<1H
za3&2*!yMs-AEeLK@Cd+K?-89f`6YOICu`d<=Pf9ad0+6V3o|a|lJYKiBGX@lpUZPJ
zz-40fgT>z$NPha%P+LP$@MPK~R5V_1rgPtc1yR^iS=(oilcY`>%toqvZC9H{aN@nR
znS_uR>TYuyb*p;yy3KkQ;>#D=A3-C0x#*}D>f_a+S2)90EofYE)+X#&CEoR^qL(x)
z(xQAo<NKnyy7oos>Jw;wpt1D`uSB=<F-ZER1YAe>H6SuzF0z5FFWw0+bL?A!Tdr~5
zyug&C(UvV$kQTqxVos=3Jy^J7>}|GzynPiC%pal0jCqux=CojCwoXai0=Az)q=9b9
z^|GfgMIq_1C1G?4Jc%DtDg1Y9p--o6`!Q<a60Z{7|M!(?0FmyCSwx>q?d(6T73e#p
z(859<hE9&Zy<g?or(M3$!j+X6989!kX3y7kP6<8%5Cx+*<t#)%i}G9BOhTHGWBlWA
zb2x`~Kam0V8>{b8`%Vf0hcLf_*l2&zq_p`-wD_+*`75L7lJsLscN7MGF%G61Y3X#|
zak5<GV<8*K_)Q3;Fr?RJR)^(;#d{hbg<B(RjFw#Ds43Kj#B=gn5w839z}y$QoKR_*
z2CoWwZ+-$xNihK0w{+U3Zi*cudPeuqScR4FwElJ?B1XG<r1ZFZ=Mp<ATr9<@+_FX;
z2bJY10H`_^1CR>8#@Kq#cfz~0HWgnAVP=3*5IbN;tYEW@**FUWtbKf-qC|t9F>TqB
zb}zBbtmnHo&&N-%(WP|)zs*XUAqKedr)#LuJ#W?sAbAKF^JCdo#I@46>uKNEZz|Wx
z@R?XOJDd%g*En?NYxHYSk<%u`liO^)yl-CeNiWn-c9`(_E@Lexl<^bC#wtP*#V&7X
zG|El#;|jYdqI+(AqGk%r@m5e*om3kHJ7wy`{R!z9P}KQ>DFLSN_)&TCus{0ra*aBO
z8wiL+7^u+NkSX?N!Lw+lHUwxI<N}|duxA;BxJ#xcO%Z_gGfh5mu;%n3MhBi(afe2x
zQ?rbeykgX;PXif-<fUslyLG-KqyFsfxm0|B&r_C?PRZ7Hy4N}=`d(i1RaHy6xqa5&
z&JOXWaOt1DvJs)@MKr(e1%O@2>3}^=`U3Q=CbFmG<DjuOs<VSFivirUiahDr>pVZT
zijpUg&_l>$O)F7Fo%j$E)Ko169^MCDW-gZhmssB1oLafA-M2>rp!H-qb{Ecd2-xu+
z=i~T&Sy-$u`yd^Co@g_|3(T2JjqCQFVcnDK)ag&|S-jyMaH#C!p!d9{yv-Qg9@Ack
zp2vsSCpCPA*EW1^`G*_#g08Zenq0~z4fgpw#Kp$pYQR;#^n3)K>oIhAt`y&{%!-#U
zZ=y$mG_C%*+xiWRwt<gP5`uSRg0-wm%91=>A4m=~mU0erjk^N3(#aGcjvLHBCfjwD
zY+P7m2H7IB7eh!+4o83XXA%H~ieAN`q)@<g+6a{$EZMBlET&{zTsSLU1MSt?%!#{E
zFIvb`e;$2&Z-1}RSE_4uX!)MrJhmjs-cS<Px8QgoQOKhg+S@kA4c0`Nz-11>uC0pN
z205+B4Nsh<6AUG=EVf;GAtO{3>ly`gWk=7wmWPvMda*_N@PkqHV41w5@;B9o%ulA`
zjXTlD*D39oumV<3Ch&N~sfl#QtP*cgU4G|D$4o6loj5vL=ld(g$yHFjE=!IUG0-l^
zbO@GC|2^VXE+?@Fj?@_2x6QBvq$aYQ<e0jmIhpO<qfg>P5{_+<Em`N<YtE>nZ%Lce
zBQuM)Qr;<eEn+CkSI|UBPnvG+az&Uk^^?x6DHh+x5?H3sn-*5IaPk}lq(*<+y>4S)
zS2^>UeaAXdcMpcLAG=sw@PhYum|Cd_peuT>G`HwJkMF`tcS3D$@e60QEYFys@Oqn0
zM_$9g(XU)&06BGFth03<F&gE3)dz#{iC_0#s0D~fgPy16oAWX>N7=ZR-Pmtv#^~+K
z;_lY4S~H`Vk=ul|6JSGl3!EoxjjssR^X9_I)<Pv1oa)^Yz6l+0*co2y1(XMG>#+8t
zCrnQ?Y|kg6Y)rB;!68$zapj^g+$f!UI}sM(<6O*GQ~<BAiT!GnZ^S-BWTvyI;){}2
zA6Le<fz3=h#sTN;SaJ~PCGUr%6dpkb5hmbUmFLLZr)OhqS4B*5t-|96OD}txRA1P&
z2zG;1TB%I!o|h{W5874nZNrsc3LcTLhTUK?<(>cqM%YA?<(5ntol)f%jzP2S)_)`4
zcmM*&BQASs&PNI>vK0ya()BZy58PM$N{w%gAr@~xj(y4KxD@34nb1%<(XMZJ6Er_7
zbP9-G$Bn;a%wOJ-8A(AHJ*86Em0Az}&|+ng@DK(xy%qUoK2zy@v9%q;iv32qj0!+m
zErs}23C7UcxTJt1i|(-HDXlV8$VLvXCcn7CyOs@OWIlGv6J+7~2NfRg7&LW{16h>d
zOBIjSqQL9KC~rWt)8a%r68p&o`pf)(_D&a8ub%BZmeN}dH}+eE79?-IipRmeO!`Ot
zhwU`0{#NTi@d=trz5hBs0vIVa*86Iw*hRTl9OTo)Gb_iuTU7D-Fp<fgLmdKjb}Nw1
zv*T~h-c$ap=HZNOP5;<zvok1K#FCv<{p*$Va*d;TJc=cypk9-7OgvVI!b-AfaYv$U
zs`3K@stTd*-avavUufCw=;@Mt7iI%F+2#{9QSZ@8-V?)0*s(Z+Q3m#_7mu}#7R6Bg
zu%%uAHmyHY`j10gicTb`{5*3sf?_fMS?A$!QJYHEMLQ;5e*v~B;?=a&(UmtwRX+-;
zb@BW$lS)D?Z42hRTP_%DN2a0WElJoAsEwV2D~kL4F|HL1WRQX)l3EH*Ps0~fUN!`B
zfn!dZmU7pr4dX?)gv;=C9iAU>e&*#L&)OexOzxl5KrelkPHh9Db>Z^A{#h`$5IF9z
z0sk++8<k`yUWt4jN9W&^ar{Fase;OO2>R$Uj;)x>5SVqvpX2>@1Z!M+Rg<Xaa=z#`
zzC~rq#>1u#{L(cl^v`9@f+5Kr`I4!!li*-lYHVi|7ILr(Ck3I}PuQ$-6eTu^@a4g+
zz;lUd@VX47B=Y{`P5O@q-_*$f?uSVX2BNjnTC8>Ph{Fbwbbl;MDaV6q0`UqIEtMO>
zFGrZ*I~>&(T$SuD1B!Ajy<CoFkY8Cm^kn4)yCY|pR>O@}uq%o#oD22Z4o7!-;e~>L
zf8({fK{db}8blZ5M%tX(F=)I>Ge)vy+h-34gPg+u^@<a1ws2^vim3w*+9Q1*xSR#K
z*cun14SodY`NWov%wo2I&{n8!w0feAE*UZ!buQrek(F8z8an|fub!-teZ>H@cr{Mq
zl^8=kyJU{;>8&BAKrk7JbOVmBScOR7gNAD$sRt2QkTLfkKySA*&TcgN{1+|B)1vOS
z46W=x0+i8`;^cA1RMTtHUT-+~`z;>}J7wwlY%=teU#~X$ZsAq-E2{B#W}CJ-jluKk
zlEiQ_OiVi1qXj%!Ju5xwMU^^M#a#7}TYS|U6+Mx%F>GcKa4EG|4gRY6qOB!6c$xvg
zmG^OVe|e8NRl06Fsg1@D13Z=98PkR3IzL7bq3(2e<5Vml5w4<)jb;ua+Uh&y4THX%
zv}S@pjL{we2lY4|lJyaVFuuIjgRfJJ*OgAwQm+Ce5v2sdP={UUk+Ay+NOlumelVL=
zFrYMM%*Tk^Z?4&GGM4T8*nT-{Ik69aU5#x7b;y);b>u;0%Zt{rvUje%d`~bGQAReu
z*PAcV$OYl^v!02|YA%ts#WlLjil-dG1;CIjobks<x7io-uDN|jYQg{TigWegAfj#Q
z3LD{7P|LG|NlG1piWL_?n@Vg`e{h1snLrNu`KKGSnsuhRQjH?Ub5KhGPt?Qw29fGy
z=u{`Yy-T+Kt(J%(D~qK%^ZCm_<1nc6hBdqt&ePTY8=P3bT&lU5K8$Rz*QZ^)GakeX
z$IRxbrqrS}nZYP0Nh3%R4K~f8kNy@%*?F^IN3hOs5tDUXNY=!tpT0_mQ4Dw8KPSuY
zpWKrF!i;}mT~SOLHkBt_7MfQ9*JU4hex_1tf(z9q^n66do$pN3$1ad}RU<h1089>Y
zK|kxfZ49w;t`S3dHOKs2^h^amo}KjPWgjvHrV7fN8ItyM-QQF25i>OQtg$eR1AO~6
z;+B`%N&r)Q%j(-e4$%r8t?VIMYu1?*x;l6oKPmy%0$MKo5t@1Y69E}UU8rH1>*1fU
zn4$PX_tKSXNLO|C@`~+CWgskZS&{pYs+A#0K+5d`qaRcsgAg1oX$9BI6xX+Hs`PzX
z?wT0a6Qm8&KG1X6Utxw3vq5m&@UdAm@tnX25ePTU=`@28(<2G~M2`nPPUqnGWnR{G
zOnk~cT4z?2tsl%!ra}f~vTNuv;MUPWfepRcHb*=mcz&>7q#PS)P>9!Mro<p3*)k60
zi_H=Z0s}=b3djBXx`Jw;!Qv}T(h1G-Rjt_D5U0?Wr9|cA5z;%RtNFJ($jMgizgX`2
zES|#{+Z|h~V{_q2`6kva=lgX^lW9=nV@w?mdkO+rnKH9_Q1X#$ZTmi$(S9U#rhqp$
z!qJ907J{&PtUJmy@q27IK^=Ae_kGS0cZn0C;n)bUai=E0nQKxE@#-5E-d4Z72|VkX
zJK`!9qApqKbqdI3wCUR`yDe{kh8vXF7g0E<x#>#kAX48VKtCCS$_A!i;(pZ;V<z0I
z(D0LOKPy+BEqs){4iz(l0$|9~B+x&*4|K!BN}gVh$rjSQl+rmpa?v05W!+}-uz)v*
zfLC*PWTceeF<|j&utP`}bh?6eo10s53uDFa9;Dn*x9v4Hf!`9FHXn&uTR&u!O)aCx
z9`u8${XTH70+ou;b`qizsmBj+wUEs&M0p)VSK~`_PL9zJSCG*q;NbcgWcpHh1DI&r
zuQ=%=2L@GzcB4t1^vGKcmE|}}eyhty81_arnu7e*pKWtn6um??QhZNSAjhxFNpUjB
z9n#-SwwRag!;N@)&~PXSqeO}#^YA{21V!3?Uio3&8>y5vW8i-mJBNYtm#wdO-!b_l
z_oq)y9E5>W;xi!ksg$9z#-2{+P}M2Sl|vmacvJTd4^px`WPVL*)X8z!s&*fW8ns6?
zZPma<B~?}juh$C4468EqSbm}N*W3%al{Z{!y;?u6jhxFET8zPb9NW7>1aB&i#WW3q
zv}rfH8(MUDUy2`K<Q%lQ;oO86{S{G6kk8lKpPI1(;5(yDu<OM{EzaB|)i<^P$_6}v
zaX!^|xSu}^u3ESugY@0b)t*xajzL`DI*>|l$u5P+8H>%dUJXCPX4HWUl>%IiTM6f8
zEqWGwIKFXwPWg&lp6WfvI(b<yiwb~MSK`V4vfOZXT=&=FI}~dal#O4`3nIn#7AM@B
z%%{t#&g;v-3OYq|(+x5w;9!GDQw7AiETYPGBlI9^7DfQb4BMI|xH_z($NY}f*DIq}
zD&U#diMd*g7ydtX@cCWAPBbBHcG878Zfz>Vj8E2`Nb<4-#2x^+ssy+tUI&3D_^&(9
zrHkZf@Wh^yoz=bRI$x4(jsG}nOz)_!Q`iV+T5OElQV_Oq{P+SO4`oh=x#r0sl(<Lc
zV8le`x?kH3N2aXZtB7KkQa<ln>42Z1-cQ5SJ09dWZm|t}#YpDjx!Xd}-GeDTn%>{-
zqG_E~^=8H!sqmI85jk4qzGP#3LK5ZSAEjX}$w6?XB#1Zf%f@{%zGZ(}xyg_iH{F{8
zLKWPM)nK?>EP&>^pB!#VuP|4WEsB3!sJq>U`o%Ro-n>SZWympjM%>2uj9D6um5Pf+
zUMN^-3f!x2A_Hpwy01Rz6Lr|nYM03RVf!_#_Weq=?*)0pL1reuV*~XUZj(aOB<)np
zg=4=Wx4HT%t2yqvzK?Gd{mL<;tKs~y@-5d|n*@vT@dmzwadKwy!1`tWrFHx64_7zQ
zBMySZ{SJEhCpeyH+A<UBkpHbb(ceM6RA%SHUa*&YJkBf@C3gwpA0^W50mnw`yOi(s
z?|==szbBg@x&-Ih)hCj^SF@@KQqRQ0+Inz&{h6VEmObrL(X$RvH}u%^qNf9;c+@FW
zKAP78Ok7BbsTHcoG7a0G>{&DeK~S||!bAKMtAloCDZc;0`)&Uc-Yi5V4lDY-{$T>S
z`gFfZKS&@bfWIP0y#kFnFe4;9j{;dV%Z~b6S#d!;?LW2mtXd2i;4qct{Uk8K#&?zR
z$%ZxpT?u$(Edu#5|2N`YKryF-Dp~Ofmjax8S64ZGUG6bgT9}T(_;s)HAaV{A0Mh{?
z+^iaz5>qdM-;HZO@G=J+eZ|}*u)F5`5XsU=6QSQ^+%<<R-_D;&_t&)NLLLyybuTnG
zu(D|tM3}FXM7Q>RuSUqD7h%@+u3T(v{APD3lVW6-T~IRiL0BJi*C`qN?Gdv`0yK&P
zEtkFNa#JjW9ftQOQSJH2EnLH*Z`M-TTUrANyR<WpQ<WnTnUpp^^R(Nwq+<0b3o%;Y
zkg`yVe<crUFPCNryrL4}DymTJr7D6VK74lCwedG4Xv6Zgpc$DM1rZEwi*_zFgK*f7
zcO_*9uHBacxRj*lb~$_jZcd4e{p*$jccq`r0}Ju3?L{Lt=i9;-;nC|xt~D+?uY95_
z3uD~%CPta%dPr(Qq0JTH+hDO2=h^4Wu`)Uv4Vh%bI8l+x|00^kLj>3oRaytNPOik)
zIq`Y>O1DpOy^C)o%_n=1Xg}Jt2B3J*w7&>g22qT4jl7Rd>TCMup2k#P0hM<aJV{~p
z(8ikKd%nqpH|8aR6B8-wIAgBia9=AN$Luh&&t%0SW_J5Ra4)d>R9%cFXUOJ(J&OtA
z>f~&VpZaT9Ect1do26=_;-CsNr2xdsVS6dcmcN;3e^eII(~S=g;JRV<x!Dz&&jWvs
zme!ue8b=6&)HU!~NBP}C2{U)R6#chx?A$#!&dq6g6jQM8X2FoWzy^L`sJW-j{>osP
z7oaW2UR@k@2+**lyDJj7kQ34t7<J-7@`Q7se#z^wJ*uFc3?3?DUm9v9ESJxTp0tx|
zj0t;HlG4)tGw_uJ5y}^u#fco|O5w;8h!nChs}QVmDgc63v;ns|6^QX5R^TmD;S-O3
z=){{cQ^dCNHYM+fkpco$F_%sKTOG9plqf>V!-W|DrdS{EU!Rlepg&mdlk?jyKfpYa
z#W@b5puF<m2t9~<1T1ijlwP|mI1f3##G%gcM$Y4#2QB>M(w;IKO18~YpGH{gWw+Q`
z7*N%Iq1)(C%%O2CK(UDY)+C$K$Gy~gQ;iM{H3^EU+xHL!hui>*GA5q8+)0*5Eqz3J
z)5LfiyJRMJk;NN)+`Ot+hne6r5c_ia(&BG?pW*A@Jr*@qI+nrXaTB!$z`;K-5xv&0
zjI=ifc)BZwvjMP67hYi;YgzpF%pdb%yy}mm#n;<Hd@+0M+U9KI`#HonpiC~&r$s-D
zd^)ICA!V8fKNB?35JQn<SL+;xK2_vo%SSu_@k}7Lqywl+g;9{+SVVr@)J^%;wFaU5
z2@;wuPA4}u;)QE0j?V|Jq@UUqA2l@0POy#GK}Ofsc%DxdeCEAxjD);Y)#7s|9_4QK
zbEj=&YD)~ysmfN4MWwk#e^b6>RM3uXnF$x{UI=5LzuhFOss~|jQ8{OEx?^lK-!Xh%
zQbId@#d?l;Ppvaxc*y7$7b$QbdaHoqZ+fGQ%zR+Rfk`UmNYb0zn^FsFZ_mPn!tG4I
zp!-}`xHJ-$(|*Eoe3TT=e26%lhPCU<5uMRc>Ahb9M`j*3YOb$0C<xuiO#m{&bA^np
z9`q2ss;==gB?VE*9hC@lC&S$5m4?WPcGL7u%VZRj5De9OR1^rBsFD`yviz}{rv_#h
zGLi;Mu5s$&=N2vgB>Lsw`tmU2c{_NYsV7PaQi<h@LhtGNx}l7=<P#0nuC8%ntY!V|
z1s5@H5OC~otoIbTEcccL!uW{a^Zt9k2!PeT=(Sz}Keux?zeDAF-Vpr`y<A2WPMB}I
z{_8SQyy5;oHlfVLHs=d&qlM62{!@#8F>FU0en6i`f0%{Qq#gi9<(Fy&EqRT(cgYlo
z%x43rBAVPa#;UZ|+B9<FGsXYXlx1w<5J&G0FN8!thAp%?D|Wh-$%SRE&RQ^qeqdyQ
z+w}>v{}2UP;O=wRDd#x`=2z{%0$r)13@?4>-OfcH4LoLed;^`(ps76eIPa;*!ox^S
zrv3`W_u+%#mCrG>qO$u|-^-l-F;-R>9sn?>$BH5Y;xu>~4~6!NKJ9NB!e>L0nd^EU
zO|(h7iYB*Q^HOn}((~|y1FsU1!~m*EJ&LQ0C_X9dQkpo1{7IkP!bMYd$v*h4Fmm5S
zL%66v*Hd|?dWIdGDLVa2zZ|c^?J5|M8WN?S_9xdX69N|pH&ywI5o#tsaD*pUy41b0
z!w<bCeJ#+$>QR_@)^OJHW@SmG6dd#O880~|-j7LuJz)UG-&G|~X0W0gv>p|l%_Wu7
zf%k3NfuASQu(-L=M+PA?sEFmXM9mV6E5ORM#PQFB1p%R!PDri;%=_mfD*hS}&k8gJ
z>M#GBSD@Fdp|t|F3;LFB`xtbGA`Ljfe}86y7T@DBT4x}Db)ithp)HwH)q1bSMI?Rt
z7=EO0%S!yMF}5Xg&Hir|BnVq-mZ$^g4Sa;1+WKONfr~4p^pVW1#9#?DP$xIXg2|da
zme1{iBPgV|^agVEoP*YeMe|^GX4sw&HA!3qjM#63J^<BTbGCB1i-9+-*PnaFw}srr
zn08L`Jjlz1S^HWUE9D9j?<>O!kzvXB&|->E_ohAS7xf?FC4#D24JgHlwnEkqepzb9
z;(Xz{Tc>S)?nVB9=p!|<qTAIJXaCGKtyqyQmo2S5_jEm_upgAdUcFM5xWQeGDWa5e
zBXb-Wg$8Om-?CD{H&P$$@c$TNo_UugkROs2G-J)sfe*9f=S*_>AX~3!=a`njfaYQe
zd|aVEub^Anauc)C8{trO?=RS{iJ@9gi=YV}z|r_iONALdv4Sc1#b}ZxQWGvWBs`s$
zdWX@7M%L0Z3PR9Kw$Kso`}+-=8>o@iaXupvw(EqB&aL1E6+wZ)N1)fu_wf}F!)vGf
z_+F!7gz_QN*!uaM3ur!%<cq^k;<rnU>kWSu#rhF=MYCNcUU5Er;nr-d=v!t*us<84
zWNf;=oq<Y)|75)LuIP0SF|vM=zA}#xHmYw9zf)10$~p)6gVqdkTcdt++jYI)sQdlK
z2YX@$y7)H&__MBgRx=)Y3MVCo>kd8!)DK$!G#~O%JA-}!i)*c2pCp__;uw5bqKWbf
z?IATB06Rx>o6_u4t6q5$JjzZZJvL7Y@99Ro1IowVlIU6nQTAZdOA#fL5W_c#r1BAC
z_~<9^f6f<~gm$z_D7n66xeF(I+FP26txHO>73O~io7UK+gkAUISAMtIy8ah#dmThi
z6VR7ve$y@TI19P#MJjNIziY1AU$@Ys_iKJ0cV&0-A(_fii~<I6Lp{f|2xtAcp;7>J
z8Y+g5(oOlNg`91)WmZr_Z0E+mB_M9?MYMn{)Jh}iv0HpG?~W7yDG&P)Kqaf2*r+&-
z=sGubJP)^5OG<;;qv)3P(^A0E-`3gI6F5J_Xz+phqg&J;b%ZV1{o01>`}rHCv{a0q
z_7j~mFt0bn>E$GN&QKJUYJ-A@-0Z~&`5e~DQ%MvMT_gE5B@EMVp=>n4#3(-hE;O}j
zsYw%@w6h$K=h6y>=cHmZ>P$%2C9GkaCOY^T0s{u1SNB}>)pAB)Zv+!b**^@SeW9I#
zdv}wABBcCou^@9uqWO?7NN^CccQe$f{h<As;oqk``LP?QDBPVL{mOdn&@@x|8931`
zh}}C-xT^1Hjk!&mW&AYwn3at;l9LDxV<7*`IJ#eS-ovqIU5X%~6KQ%fnQysvaZRA0
zYsmXiZmVzfXhL&qLR!n|c?`}8xP7tbmjN4Tn}!wGit}R`4Y3=JBKsqA?c@N+`w9>-
zM6)+NR7W9}8zMuXHTbu>`Mjx$Ueva^`YLLqtj1H9U?w;sf7)IxM8hXi5*!hdZaZ1R
zCM4SF>9-hZ0R2k8H<XDhY4E^4XY^t9hox#APnw%Ur2T_@u1<FJjCM46j%JXN=)mg1
z_0Ka+;6)9~Ai@K!<@v2MIvus28RvYvFpUmp1m>%-IA{hrLi;gDna~X1vmfLs1ue>h
zSHRT%^R`=w6kGgoz0RxlDEw7C>41mLkAy0Oj&Sby6@I|jqi}EVHN-wSebz!&=m11z
ztIa#dLsX}vNGj<T{vmjk6bz(Pdna^?wMc1vGj5be;NzJxwuYuYpEl#9gKr|p3-20W
zbO_flz2T=dsedf!qKYAKPhf`_Vc0$C<j!G{s3fG<^&>ukwiiX_@>O~*Qp!}6B}fvB
zUXhMWbPjYejMY&9B~PaO*tideQP#jhfI-rhRwH}Y%s^|-g)KqreA{f>ZQPnD&<N6E
zFP(GBP*d$&xSqwffvdL$(q3~`vKH$z^_#09H}PG4i<1}1W*oXfxO0+}CO>oP_PGN}
zwt(o9nJeqj8(W0iA(I%0Z7rgsSQtG}5I~F##9>o9=My=X%Q^&9{<)xswG2C5X+Spz
zY|~DVg{7}KD)=Zrt))ABj)H7@Y^I;E=Nw=ELsn(mBU~U!B3?4M8WXrNV<*I}YtoSJ
zF_jfd=XS`xFHWD4#1g{T`Mlz@A9d_U%RDp^qPxsT95eupDT_RXG*0}K<xf%qRaDh(
z-{7#X;Is>C!fA==a?dy0t!U)tnv*#KEa<52(*(B($jEebj1clWB7!WiaJJRBl-^va
zW`Yg3W9g6OY{r=PU&@PHEcSOLSm=~<Bz`z2XsZ&rOmngXLbMQnHw}n5)cTLf8w}pE
zxV+FS(ffm`u6kL~x+y31w8cB8LeYZbirmPwC)o!`w1smgrH~X(i&6#$bpYLTV)SPE
zlR7+*7<pQ__wd)Nv6W~68O5DHQ&!c$D5Vq0KmUAyFmTpf*q?|b2jhbZWg39)<tEcE
z&;42g1<W^I0i7*Vgzo$U?cS~t>58S&e75Nt?T+?Nvh2nBcQQU11*G&)kl|7bO3tPI
z$jCtM8%QbiqGR=N`d^rXQEi_^5oNCl*m7nJ-H#Y$YR30WWnuABB%<$2q6KRrwO@Ev
za!ePbzgagTBGhB3tY{s7K^b8#o06Ta(1hI+UDDsuP#G<Vyrs~YTuXW>6u2!ylDZU8
z^s+0Ougpz=mLU!sG^6fS5h4ydTY>7vzkbr+Q+{MepvT;Tl00T%YTfS+3Bgnq*Ny#w
z#>i_uBgo9)Q4ejLo)x|h)Wp=r(rD&5!3n(#n-PlSw5{2f(N^V7x}B3f+wtbzs26wx
zpa%kr=440{MxAn-VyyKwXa>QbajXA)&;GQ}S+=J%1tk1Jl!OK!tVEdLzDZjxJTehv
zv34BI&{e}Bg|Gr{5_mW>jZ{bg3JqwN9fdJJ`IV27DHa#YNm=QpYV$}WWL@JahSm<}
z62&T`ih_YqsN1uR8w{flYBRQc*H)N1MLhF)0S*^y`(bP!oBeEA!+`!hzX(%iZmSHe
zN<<Ebav(;6r-^`NQo#NhZF^xnqd#fisMW?_>H{BM8(OI}Hd>2wK;12UfugFE&uz=x
zrwg)gEME4D@NZ1Zi7^6@y7nwys}R;X(E?W+4Czzeu%d8s+K$%Rr@|fk4jfy9Zj`jX
z>8ySU_YwPjr1!21K!y#eM^{p}2Xiii7?bdkt3O+i1Df)85D*I$oS_YTuJ6+O1VOr)
zW2A&b>cDkiltrmZJ-#j?C=4+a`%#*L`v}4tRsCl@19oc&2VR)O*DbB#M~Pr0>mtC0
z&POs#!KM;HE7mafPk-k+HESpvcXVV(J|>*onVb?~%EP#6zwTH6#1uAI!g;ZP9|&Uj
z8GC}{X?zbz*Vnt2fKSsr_6}eX+F%Jj5o*6q8sd0ng_*qU*Pfq*B<(@i{?if;5TkYP
zr{$&I${97(Tbnn2;B3r1&0Pu>#z)e%g9W*ZRrGV^Nu^`n9<T)Z0d-iFL8k8Y|1ln;
zR$H1GZq(wO(F$#h;6{#;13txdSyi<4aID1FM4@n;7qU|4B5e`9jCqed$3!{#@E!AL
zz2W{TtZ*dy=$$dX;|(A3Hs{E5OKYBgFt8hO(k&t>EAkcgh+6fJ`7+I)EoI#Hh082y
z_zSEvPdX)s^iI*C_aSD>x$RL4q6;qJpN9`nrkrS;(`|sPW{0&F3-(c^p*c+%$&2~h
z9?GiNnlL-M5wQr4fq%-5BduCQrY`y7ib(0gTRUz*$Z}<Hc+xFo7(fSCNzgaty3>ar
zmx9|~_FZq`SmaO-7SjM6P=xP3{++q1Rso;^x2(R4o@izo%kiDrzwIT^4A~$1ksA=%
zF>+(anw}ec{?-)Ve7#w$#9bc^SVkBm9!0aueDKs%{^NPa222bX{iLv>u!QH{K69)K
zhVaQXy%gjKFRv&r3@tw}0atJFX4dEMgu*e5AvP{AIb}}~XBF)h$k|VYe&T4)#paS)
ziiyNX8WrpgSchK2*RhapxzG711BFd5kIS3(i*suA2tVq$*xoQ3MRqx43_v*!bwD4+
zI{hD!%7AM@O8L7~I4lvU5aE9nR5)Z28M@kgi*|^@ahsJGVU8U25ix6ns=vM(=S^pR
zF!QHelwrbiN?oe*boaCt3OoV>>k|K#j1>vpy~TFy07Z1+nwXI86Cadrc{5&eKAxPA
z!s;5mFMp&|8|$n0Ssd*WT`rZUBi%+<2^tm9d`zxvj=Qq`+9S&D(-R>}i55t9;c&#_
zM_p`#U$`%9w9}?pB#kv<hg<AErHg0t)*CqQodl--C5g1};YS82IHy*!r-a{fHHI-#
z<_*Q~vPgBrIfefb#Ns87h^%*HK_>)24HY6ge29Xx>%-7gq5a%1{1cdJ)a!on%}Mpi
zSM#Kyn;K&F%=i;GQ8T1_uY!3r7#n+dG(#-VQnImnXHA^RO&E*$#r<Dae~`gRq5cib
zlJq(!s)+g&94tw>W>f?r;bt*&dn8O6kCLa?AnJwDoZc8)ZkBR_{i@n?@374VON#0D
z)$5>s-r|cx^FS8-5Phfzdv|8Gxp`{-4i_#THHENBBPg=IWvhW8gy%2F!}hMayp2Dp
zpB9Uhk=+T~_3&g%p)AzFR+ot!LRKkda*98V%Zebcu3VXI`sQ8m_;3yGq;c`={1@|H
zM{MI(=!LZZi7ghJ?T%1B3r;<K=x6YgRj&r4gxeC)_(2dwgbh1QG({joYA|6$aP>rn
zRAb<t#E5Vr5ztwIpPAVx2cvWx`SCwpvnf3)OSCFU>fhd+PyMr&{kFWO{p|u@b?KC(
z^6|;CUc8sX+!h~Cq$RPQiGmW?yYX+oAUXT%4b1IrY`Cl)aRqCWq$L*WEfA=rS8f!8
zKd>|4I*owsWYuJZjg^-Np?*5re=z&2n;FxyXWAz{y0zm3l#Y$QwrwD{3l}(qKFq@|
zo0Rull_8cpNa|N%E=))Gp8a@|RtKRlU>!E2bd*~OfL+`AvWbI6&;x*?S2m^+{v5R+
zyU`=&6uR6DCfuA^ly!V&{iFmCG9YGLDg?X;>w)^P;I5vNzy;-mJ27(tF*x#%8?dQf
z%1z8WWW0*afaOJ9Z<DPYS#C@e^3Y_>fZ_N?t&zW@HA$xcZ*MKhmD9fPz%v`=yKJ|f
z9A@X9sE*&fGopb)FiOHF6pW30s12!3*W+k3M@}qvr4_wG8%H*)V4K=f0}A5M@{nmS
zJ^lrntx*tVsVNN6(hZnmipWnBrEhZ)i4;h@)*B)vtfjWC&%s`#@A84d={+VjWrU}}
zlM{FsbD_9R3JJ?&FxdEMj2vbim`ip{r#1<Y&;tb@hjA6T{)QClhJ!S~rz8|r!1K11
zu-iJ_bi9J*&p_#jH{E>{><R&NjhZ<pZE_Lv46p@CD|=M+SKX+Iu?F77&A;wLyEm`y
zQr6Yz(a}rdvEFnUyOUTo<pfLu!|@*4j9@`m1szKi^nb!DHO+YIZtmOcRIp?fu8X@#
z+}hC!(E#IRa-Vi<f0V83T@qN!p3@WC9$&qx9%1HuAwMUJ`#oE@**ET{2RrTn+!GTd
z$Y&o5Kdz-*kZ`gYzRT2rl8ct#ewRJUi9>bd0wt>kQmCy2H~jub(m5X*$2gJbnFtiy
esSY2<4nG!n{!=R_PXe;4b&xTotZTYKLYOUKb<pSl

literal 20496
zcmV(xK<K|X+e#0FiA8w~&YGMTxxFDvk0|3oUvWJQg%}NntX?*EQp!Gk73>La^A1A{
zC?Psk9F9=0vs|yCO47y*ZIlD#&N(|qprYh0C41$`h#K2d`6H*ZNxL@aVeCa=2}u8D
z5Q5<HtLSUV=0pr$X^Oem0Xv8Qz$52_>|N$UM?6oFY1o+8H3bKI!)<OrWYS47ys>F!
zr{9+)UXwS^{4KZ)D9;gIr=-C1hI}j2u1gmKTOSwK_ql<OLTde(^AAo%nqZ!3$CSRc
zNT?LM(_id_<F-?M#&ni_Z;}iAp&EcMQ`1`n-z_UZK*OLGpx3Ql$hX>93X?;RiH1pV
zt3V&pEo>`inCnd$LvvpODYx5}oZwy!C&sT$O;d;KAjQff9hEWHs$o>1TWp}D(5PBs
zU-&trO@4xzzF6H6v^aC634_{Ek}yfX?+Bvb>m6t#Bfz?4$X=C}Rny9emz(KlY#AXg
ztAS>J7U3N{6O%p0#Y!k5H%4GiBEG#+`F#%(EnpQ{jeJ47^;9r<opf85{#MLqMx+_3
zAl&>U-+A;Zh~S8pmQ#Ok-8#dn^w+&Tm8eV*+djOfx03ep$2_y0%fo=!bVvN$Z~Erw
z_c?aRb=2=&I;Bh>p`z7swGXpXVRnKk$+aw=c2=SqNDa0eMRR50^a+FKk99A$E@y%&
zRMpT&<XRIzP+q=1szc)XQz<)Sbn8F~-^V)(MDkOO-y-~cZtLog^+szqc2(PNRRH^0
z73<yj#%&{UY{wVa*_AVW19KqT+kat=*vz?URBM1~F`3TrXDl<*qCO-CB$EvMfXA<+
zT_rZs<bp#xaeK3pkeF-iE0aS>(1CDy8~glgx52u1c9>+6@FT)&AmFx@{bUlaWa|}U
zZkjUUtQT}9TIMt4rEG^+Qli5hms$~+F8v&X$t#~*)ZWR8bs!JkLP;SIAYM&T>P!4C
z7-OtV9m<m0c9O9H%iu8?eR)}tr!_dR<UT-sul8rySod8_$qSXaJgjp|luLPRy~cdm
ztJ~G&0RDoCU<6Oh;3Q@CbP_yo7ylG2_hub0zSi@BDD5NPP;_kcu28A&64=q_ZDxjz
zk{n`9srM>hPLc)3-UL@VjYdl^(w!^(OB@UZ9IP1Z;ArZrKsG~dc%XS|L13s;y|7p@
z$%kG{RaOp@0Ej|1O<z{0#1-)$RDNhMyg5w?SxY7BV3&8GFJsZULZ`wEswRGzikbjr
z9QBL?w^v0k%4%~0PWI3gv0=C;H1|`Q-+19#P`Iww%|kTxW@(~t+MxK<ccOlgZosu<
zj^`h?YX~Nm7bQmY)w{Y>^J!V737*pHpy6Pt6U7WumTn574s?rEGay*kHayut3V%)(
zSuQ#F9rqx2jCFDGA_kK#Md`Rbb6)(Oi2Bs4gO`TIEFkF#qs4^#?CZ6Flu;6~<>f$B
zeLz{f5K#l7+%dhvDv?(w<HS8=34GS3@K>PddQocvHYwBdXCi&++s?vDJ6p&?Tg~Py
zcRcBMCV}UFl+E6a62rXSPGi1?=Q`7=djXe1al&0L!HLQ6MU<%5keb~LX$<~OOk#kh
z^)hbR?nDK`q)z{sW<<OZf}8G}pW>p&<~Kh1DU3IC2YkU`7bop2Wr-1Gu-Vw)HxWZB
z4cIj;w#ji+D}<9dp4*YE;r(H~BMM{)GsEU-cIy1S_eF{vuawEalG@ZPhna$0yZBK>
z;nwpJ*;`2F%DqH`opfspEJWBYBD1`V!bL~Dfz<RnrDpF2Nx_oJxxWeBAyj6^ZmT5W
z2mAk-Dpx*Ze~5BRLJP$iX#mNHT};FsfDM}{((S?Q0t4YEF~cP5ns`9%bx)>HQ(8!Q
zzG-!OA6GY{8lfpHrZaho<Km(h5hC23O1=$c94tPX^kdQJwBzlUngWtkJWA?=G#^yj
zl+8FB7M7EUowDjW=TgXhj2i2q)t;Hw>~lV=36aLYGr+!%w*Vzm+0nYIg6?ri6t{f;
zHgSsnOXKF+)fvN{Kj9xB(!FXC=~+=5p<bE&V<0GvFQPF3E#%j_y}`xKvy$pfyRi_p
z&4ar;T@q;BciHN2QaJ#(`_9_V9T~4mLSwpGF6C@_(iFtiBMkL#BL{HaWfr!rFJUJ%
zOyFF$>;x|ywAN=+v=l;*a4A(od*BU`OJri#-l^|Q{Uqj_68JGAWls<E(3ypc{0n{~
z@4;X!!u4#R2h~jMa}8%n-hT+R#e=$Z@gp12x+!lc4jUwSxkc%6QG>C*G!Srh#>jr<
zB1Ae$jw%pY1GJaZ<x!jEzERdX{KWZV;rR)4mPOIES`+&9CzgmHXW7-7mw)!qZw0@4
zi{}R_KPJr@)2G}YHD4%^mQrS&RUPZsPDrjoSyN6!n~r9}K1X6{Ol!_?sm$xkL`Ub~
zvvxA)IZ35a<5tNeoXu-;<s`EGoo^DO?=stDEOVM<vVK9af=wARIhl;MUiQXqxi=21
zCLDp*vfnoCM*@GijC(xSKH?#?<fRIRAnrn9Ag{)!VX~h7izdYp_}(bAfV{+MrRt8(
zK_?cW-|I1y2+E6ToS7hCa&i2)ZS*NORgtOhE=DEz=e~<eyARcO*URBqB*tl+J1e4^
zF~=N?*`zGjlGM<(AQrvJaP{D&A;q!)BHbljBag1Po`f$unbbz}Sv}=*`A>z(6hXcU
zS$@i}V1VwB04K_`Uq5$o%en#n%g@i`vd)t2E0t78&@U#?IYCfkv&iC_{`6WD_Qwrb
z{{N;y(Cz<ep#-<^;d(IQ(jv{$P5ugERI-U3S`u(QIzvEuCP&YwV5?NKj`9&mm$k;z
zv}0YPr|UWMGndph);Co#rnLphFuC#2IQ`;YF>WcAjRs6Y2=X+4r6-Zvxne4jTxn)o
zadyXOZmFmw`V)HI*a#E5QxMlfDguXI*ljvD^UT3=fk+`Wg}BdF)wVk+8ix`j21P!0
zR0pftB4o~SAp3;^s>RSE`keRD`dPada+I4ddVrD6B8AajH9g1dkYU;_L}p3c;4adm
z<Q-X~9cFJ_8eoTa85^pjxFKrXEs9f>xivL1Z^6@gQrvk989>X*ebBNK5fvP41z#Ej
z(xnv;a1L|H>%-S(XL=-idaa@g_3tJVuKDm7(1nYOFlOu-`yNLJ2WZFQhs?AJZX|5O
z)uc?A6XWoG-{%BLCo03w2GE9aI7YIYe*O|fH<g`Y!#rH3ng-kBYl$isE*r8}!eLTK
z#ak2X3YgjZxfT#1CR0yz1Csrp$1hxd!A`lTz-j#^8LC0VgtsY0*(3#&IaB#2xcVaH
zaEW733La2%fq2mlE)5E%Flx$OuXt3Vc!X6ke`yGI<2%aZd>2-h3F!FIj0q?;_mULV
z0{gYfpP_WSs;V7@^5|@=!5ph0B#=<wYiBF5+;o=KfAs8SDyO`?(XFypc@*m@M-Vu#
zj|$>>bSD^vS&(WlPyeLTd7+?8Ogg`D^k_Qu=&kzLY-+Z{YPA0-Dk7W9CSEh8ezjB4
zT8{fZrR|>&gd<|*BrSu$`Og-&r@^%8t*neTaK<u^q?K+wZdovTWLP~f-7hukB`J`@
zd(}!POA51MS>uH*EjV)>!UKaqiQ?S*3u*oz&eS#r0Ac>QBjiiJkmGGQU502txAIX*
zb3e#cLO5!WK$XD^i&jV9#NkHU$+~4}F)K8SrFtNkfUeK28A<t&AVn7U0@=`?I!xTv
zF}M#X-s_WIUnp#7JmNlu7@ulnZRT_YUi5c)AvvRpy(<n;&5GBMh%)UpNFKvzritt+
zAH3nnvEoyxzO3D)qm23p6I}h?d=t~&LVJ-%=8+Fi&bDYAX<<#6-#Z|P3B&ZoKU*Ae
z5woQX&qp(4m^ayLyWvQw?p&a2)u~9sfuTg=V1&}EocFim1PdX4!{z8x?_;mfR!_$2
z&Vd+YJ-sD2W;p(qlVDoY-Js$$!I7v#&KX28A>u6wwv{7tyXgd?V_V-fS(-b+Us};A
z7D9v64lwSi?slkeu9O;S>O$th=3gGf-Cyq<Q6%(^gUw`IPd?tNMJ<%6o>2edB%XWU
zXdi>=6suSa?D0?f`7`^l{Dv%`{$V$@TDyMn%C9SK$z2mEz(*xBbG-J^)<1ApV<s=b
zo5|Aad5u{o>rSohgX>*Ukmr%Ie{OWN`tUinJ2(wLaguo{vGY%vba8ZMb#b3Mx}W6%
z_f$nPk0L$YkZIE%6uU<(S%vu!5r)T3MxyM>hxPY0VBT3nDgaN~_wj@FVvNJ)-Ig2w
z)gADCy@1i`qqeMb8nIGX74|WZv(qU+p@>4h7Cq3R4svFsdvIM;?_Zy-S4m@(@<SXh
zXNnL>OuO;2cL{f|$4_c=c|$x5Dsp!!`zJ6>m;G_;1qvCzj{|Y2ZQnB4s%Z;2Z-w7*
zkIW>QpCkwI<pXXc>3&H@(ik)?AYDS!vM|M}H34|PK0O>>yG7Z)Oh02ppIzA32H&T*
z#h>-m1izeNeT*Z(9VPBQvV<aO+4epIdoI2`tLUI#=<FdX;y{s^6i2<O0FHE|G+Uaq
z9X7m>;IA4`RwmYVIBtdsR!f?sW9yW|ZYOH(Qt_!T1>?+3$0X9nObJWriq2I~ti*kJ
z#9(MOJD%Zc27oQf!kWY8)^6xbuz8}YyplMj|J`dME)cVPpHQA`tdo0Z8B_(%9qw&i
zY^Mp*MVL+Y`U<8_dwXf*%5x^*$CREo0U#0LY2r!f{xElZlNhs`qXcULj0-&9++#f<
zZ}KX^4YWUz9%&5pDq(v>n@rU~!hPHg?bT1cC?j|A2>iQCDT5XG(b-QdI2Q1=xm^a3
zHx%pE>Up9<6Zu6uj2r7FKD6yae{RM%KN|xdYsWDjnA!SksW)G&mveNDHlh>1BJ<Kq
zAC#{%0uua|_UMDzgA2MFM4g7C`P6<DJ8)4=FzF#xfvG8EXya{oJ*!_fBlI~HFFEn$
zDA~$fZ24;_jY)*GT#n41>ZAK2%&_-bC5S&F7def~K^o&|1nq=w@JHSp&n+Dx=!Vf2
zE8p&`X-j5fpMRb`WxGh<%{=;mmmR>)3oX!q7ipXt({9>?B)V?iQI1I`5O_Dh3G*h?
zSHtX#9X=RUlgoo1p*Y}{`kk_pCqU7+hY1J=L1Xv1Ja@^2IlysxXJNkj`;OV63yrBk
z;<HK~gE=gRSFiFUOJ0%)lwU0XXgo5%4brG6eB(vj3yKiP3-1JM93zvVTLuBdN$*4u
z^MC6Ok*SJt=raUo(2xf#045VYl-1R+96q9I{Ac(_;o6s+Q4SIp2>7iz$VX_az(k;U
z5nVy_8)gNZ^QikCpmx|_jh(=UWL#L9_udUj(2cKgvgI5?hr>(tv%UMJHh3~lC+=mF
zQHnzvYT&A5E^^SB{tu+%bHC^VfRkBTYW59{HEH!5v44Y0j>CpmIZ7xq{~cqlkxRpt
zuGN7gj=g|69;L0aznpAJX>r`tf0G-?Q2d=HPF0%qCb+``@)7|w@Txov9xM@q0U#(c
z{&3$fXl@7WJ_KXil8<|D{mAVWM6qX0j8n`w$PB_9t497DEw<u8tV&w@92m4sx0S4(
zvGC<(U{_Xt#AeCM9nJs^n~5!4JdF{-DR$>1vd8OHn3>y0{jho6|IBS2thnz|Jzd9(
zlrA?}QWBk7H7ERDmwi;T-CP->^JH1iXO|+`DI&l(!u;qT3-9J}%>D^xp1Yid4`Q(q
z+po(R$C%ChwlHInT*l1+U`!$gjryT>Mk-`4U;7T9+M9q!pxKA*z+*a7$Q>DREk=q)
zspuhwdV6I|gI6>gJ_Bf$YDA(^)a&sO;$P?Vb?}EwCc56t@)$YU>?o{Iy4<>fmeX!L
zJ-lNiYpXJ2q>CUj6<8A}O_q}nde^qpAO_9{E)!1O>%Dl?yX6E&&9qwZ(ac6fDJ4kX
z7l<9|O|Y55u+68G#9;^cRi>2^Fg&d;Z&kHeFhX-rZ83UEg3g<O3lIHq+?BB%i%s?t
z`lJTmMnCVTu2UL3BgyS?dFwzHrRssZPECk^In9@Wim;eLAHOyiO_=3@X5_H|53nMR
zmi|3xmi5wmcWT@t-4pn<kEoqxwde-z{Je$denI!O$*6;@ZvnxIygdrDVP+@%E5KUH
z7L!vRAPsjz7W;1GXT8=I8`DrRIt@QDLkhbK;0&rqTc_jkr3AAYyD*@hp4#~lwtw-u
z=gI@g79r6d_UGzxTy)OAeZ6VM`#T;o`QghewjL~8xWUbx2U(UfcyQXka<?%__AAR=
zDh}%{xeQb+oC0;uPXu>K*LjHhd&XBA*I)gmolo(BU__Z@yo!e~yroswP$rPCOwDit
ze-z(SVqhfdS5Jvt%8}$bVX_#BIC_Z$sRo8R&tlR$eBiZ35G~=DQaI5X1nVqN!Hc++
zq5K4dWvK@lfQawv1Q}bfx8kpP7=Z1fp@tj`-do3JoRF|G8aJmvkWwjl>-}tg{u6Vr
z6(E6hNc$A@8%-$T>VXQ&_|dgGYb_Oc(GK>fprwQ~_IO<8o2-?&GV<S48X`4L_Vpt)
zaqOKigq=i$2#krhNZImzTFxIcJXB}gR-%H2qFFvUC-Hd3264`ECIohHM9qzd8~Evx
zV38jg!M&vvxL8J<*xA=qpEkpIVDHhDJey>t_Xo~{e+xYG%ju|e6m=Mv^rb4Bz_A05
z{g0_m<l&>>1pcJaMZ`t|ge41Z54+Vf6Y}W!OP3+UXa`&BpR+*1DN?ZSmkwGEe_SP@
z9K<=ld2`l=Itov(tqeP%q)~-0Y#isGT3XE;Qf>BXuH9G%Ww?aN(EYQNa|*h!IH69|
zc^WMLNsb@!I03^-*Siss>xmW&C&Ps3Js`N*V&_e5_v(BYDDcF8W~|85j6YVmbpU4~
ze!44^#4#>Eh&89*=5TL~a#tynZW=UANMFTgwMh>zrqbI`_!JyX5!{*2UjhWhCUWwy
zLa^CdN?e88Z|$j9oJwty=JHpgx()J>$iA*q6YwcjGR%k)6=K_oPqu5+1>H_|-MQcQ
zNB^K7FknMR_%d_A!A#t>KJ$yaM%7ozCvKR-ce3%Hz{ltc;x;NV5f*|@YVIX7btAPf
z<NFTP0ZlHWaqk5_Sw7@ws-y_>=??PmgjHH`2!TBCtj)ljtI$}LXueAzVm=S`m@!Q&
ze6SqV=7q`MU&5e|^^4!CA+Anz=hQ0N9Y-Uf4A88>zl`;|`_@xg!<jcDM79|9;B%4m
zRcn;2(0dJd*1!*aox`B4THp;5Uz1wE{0^A)ov~)E<_a%VA^Ii(`d!Peq0|bMcVR6+
zZ8cO7j7*L(OntD$Kv&);Q)W9QKro$ilqScbu^YA7e-%&O{U^gguMnrEa+r2-wH$jt
zthFVk26R{0RgIeGHNMC$3pQK!s3N>d8I9sBGEr_2Z5`{rMt(KAwF>&tVknCZM6e{Y
z4Ih=bOP&?QE`_?>2#SlSmDH7NW&479JifaLu}3exz+tlivD)JUUCL@RYSqGUR0Jht
zM}^|=isFZZGIMdU<UD-rGuJ2<)l7Hf^=A<(#vz}2BXJ0-#sX2TUet$9=6M4?m2F?~
zu70o95O~R&F@H#RG^qru;Q*Y(FX2vLGDM?~RL!ZBk!yzC6fy`7XSWYjOwSxD!_>>!
zE*6_)USMDqC!SFZAb%RbN5AnTCQ86QO-D|KSO(F`&LW$qzy2jNFP7o^p2oXxGlH$M
zIFn8)z>$SGHHLWQItmIUHQnkb@XxI75nPYz1;5zphEJu%R=P`<jj0i<3$lR&-1Gc>
zK)Gv{hP9=qz1Xv=I<#h!5-pz<X-cgl=*AWizg3+fY-jl<Rz99*tGY@U|D7Ro+AlCi
zx8dw^w02BMi=PQQo=o*(ln{4WKVW7ZjUx_?0m1c1b-B%l`ZHJzzbP7qqx?!KRUE(Z
z)C(5oBbZ$Rxj*tR(5ujxF%nAmrgA=7>q|Z&F;mH8PBI6b(050LRAehwJKo!W#+3Cx
zc)p0xfoPWGPp^}Ht+Y04WE0@qVGCmwEL+dW%OjP*<xjYNoz#N7?_~i&l-zW9O+#PB
z^>|wsrU+sw5UHcvXh(W^n!Xv7tggs-w(I?`WG@Xs$q`lisV+|HXuP9V^|QhK`^P>c
z>2p9ewlzkRu8kRyfZP@F(o;%`%bBAm6^lACjw=>%QOr&VE&i$GMy7_~MHaM;ucUnC
z4K2+hXIBO8Ja1)Sl%YW|Kg;;!eBTs{9z<>>P~R6%GBJsTX>W*l%+1`Ds#hl3ehP+g
z);xkT>8!*7!Al(|(2Bp7p1sURCU0p7@<@RMZMEr|^A9%hq!CQPg@Fk#MlHX_*xvTe
z_zPFJYgH47;VX?E4Gcd?24?D)p+3?5*r+!aI&mWpkw=TIjvn-<8+H6g=3B43u$w)i
zI^1_-#S@p6KCn>~cv;P5)=`{LEbvj@ysw5Zs{HEPxoHU~%}aCE#aR}rXl#g%;P9#3
zHKMp{wX43(+NmOo6s)}7>!u_1kTB=Y^(D&wXI{8I)!oKVlNI;Ce5;$bK+-L)$r9?3
z#--I-!YTouI6(%8KB#LBkA2(jSp9G!U<=4(#8hd^8j$~?1~$1`xNAcJ_t?}Ml@qxv
z4fqa_{EF=n#Qf5m6}|_7<nuxt?N5iuG;bmtPSm-^Ufl-TIki=#`2p^0Z$|cMhN}_`
zh{^o%RS5XdF^b}Vz$R*Je3Bg20Bg&vwd((&I%3KkxWqDUD^*N~3@_X>Bi?XJsp<Qe
z!c!&b^`7$N9`w!l4KJe@D<1T^wMcn=G8>t8?}x!f(^f=E4K!jdR()!!i+0aEu7F)A
zRG?q5OpB&_qBcsL?XQdU9p`kAqjw&=OvWogII1oVXu4~Z(~gO6P5rp5emQ%{wCM$3
z?@hvW%+K)9{-Z?YiGablG=@I*Pv3u@Ug@|(^}dBEBA*nA!vmcP^5YXvY`1Jad>}~!
zw>$?DG-3#9?Hk|!0FcBulU_NG%wz|!-E4Tk{5^7Pa-i7{CptsZK!aq=yEo*xm%xDB
z2MKX>-;{cw7&spfG~|?dS5^OqdHb}wl_4TDidohvjPc_*{h0ZLl;TbL)pNvlTya1t
zXTv}ar{6L*AtZg!R3v{fNuZ|ekF(c_p#^}&2i!SB&>MpjQ`WD+XvoxT#lWhq^E*Eb
zsU)YW%vyx4Y;g|E#3g3V9x8_HtqbjBYuig&T_WC#^O~>=9Y|`T8|&JZ(F^r;Qw7wR
zAt&;xJhF|ErJK}sxH}rCB-PB}fwv>lW#r%TjjF3>kWMVlnAucEM$M@PBP|C6OWGC`
z=BfSVH;h-I6O}l7BT&2L&K<AJjhA|UvdUrROgu&1cxpV49_t08{AA_^rh22J;stj)
zkRp;ynH~BaTd<#5RrvjB_ry>h^;|9>1&5h25?qqth2EhMxC>98SrXCGZuEi7y6?<|
zw-yTZ_PPrH(5rV^KRPO_s?xlamP;@ip#;<`M%5DCn*xIdL8mFF?|xmTR3Ek=iP#qZ
zI`Xd?IUro1WIG}8k4@^G|Hl}h%0IBG`_#3e{SZ@{fQ#Zs$?BDMzfoZE2ArE}pUiO(
zV_j<?M8~uPoH{8ZyQq4wU@V)@?RW~NFbt-JtNth6u(~2s<S^NlJEfig&W+`fMdINI
z60Sdaj1J}0E}V^fsY8WP<*=-hdI#$;fTGw1yJye|2hSt>SV-wOU2#w|T8S;5wP8*=
z6*_c*W}L3DFB#bHu*Xk%>*QGoRmVrU>x`w&AB;t<BF?ScSOK^9c;q*#7Dx@kM40BF
z*2KN~jE5k;S)CC}=9ZC8&s7<QUQ>Lw^^VuTu@muy>)~<|KQ1|Yy5=p%MPE$TCPn5k
z@92o^#4wK`n1Mkrpmw!gZ``8>UZl?jhYH0bV@v2~=BjnMjOqT~{4UgU>~#u0U-I|w
zO?*M&KV}W6nfbm0c;@y4i|^MS#Eb}#HX&*r!-hTl<k?JYv}R;1Y%PkBMdRsiJY^9#
zL^3aZg(NWOtY`j+o6u2L_+0uDTg;N;XuZlUsxDEOR|*}UXkV}lI_ExK?PRXtipC~o
z6sUyFl5+)XgEyXP@QB3b!><28#ID8`JXB6%Jg%AcV&n`J1W9?8s2fr3nM@=^dlpim
zwIyti52$0~J*!Ui-=LGN&pdMl32@{3*9{Tk+3uG>b&obl{Y+ItV3_P4fL!>368ln)
zsByfuuR#6SzDn{rWFzV~-0tAAx-#EiCI|tyS*9fpz}XAVOM~$LIe!5hV2zY|@hAFj
zJ58PcR|LZ@)T!@dZeL3Dz;$$(XdWBRK&(2ys$eqAUq8r^5B8eK^MoeYSiLJGa7%qc
z2`;YfNvWS~EaBh!VJj8Outq74*KW@8-En|(Jxm+9>uf{b`yhEk{7)2k)qw=^m&U-B
z-sMexpB1acuhgVWha8nws}2fINQt~Z>?%U>tGj!?|1X#l4;$?`Xh}<l?VW4@NZ%X4
z;&9t~-OAE(I;<M##Dp-4`oX|dPcST^w_F|6i$~<*at=Baat&%tsu0WUVb;ZNFC@dl
zRh2Y!57+uIQ5HZ&VI6@jaXW*ag=u0$F{ur2mbHg_+rB^e12N3Ib7c#BB8!@o`Ywb&
zE2;gmo3Xy>86ZXLukC5a!QhYfrks^-Cv9|5`<~Y903uu!IJ}(cZZuu(PGnrTr@wio
zexHwi%a~3%D6$Q+dz^~KCxWWxLws-NuQ){(bkbsU_gdE?VmGFZEPRBra%dmg$G4g{
zgNJ*jQRp3wU|TW-?bUAkrQ5%vs#!tnGv(+X0n_HN)mV!5iAsjns*hH{EGM)KeWq_*
zPyin?EJ8yiE9(~g?<@UjEOgERxC}n@s54yn-+Z;Zku89{s1V#}(x6PY->U9?Yl%%y
zC0y0-q?i7|Mblu0Csf)mqGMzSrwqG-DM0NAq0%Ca#&t3~E9z5EJ%dL&k&;#oxlKr6
zD$50?aTk;F47%@Do>B?KRAsyUUpE*UnNE=AHUTtA3lUDdbs<^cw_yD1^V4keXOiYw
z<yQ&cuYl-EGDFsS@Z8CM3*wddqZos|n4_$CRzYnG72E=|OGM^Z|7cQL<=qUAwnO*N
znZmc6uZbpf-5~t+(1v)xE3Ey*i_5y_z>|ORJYVHt0Uu2Lon06nP;c|a{`t2P`o)+y
zWM74@GPQ<G6maytkr5WXBkc5}kW%`C{a<eo6(_~v^{6rT0lq08_O>n?6t0w|)Wgu0
zzvk|T8O-;Un$%oHqATA!`Ml0|nRTSJ>R|^VC-$GZ(_!Z!h3Qf-KZ<hWgM1uFYmn&H
zNK_P7jy3KyJui4;`2hB6BqOWBkfEi31LuURI8_qPrx!T)XoQKx=f#9Ip%20#9D3AB
z-OuZ$(i<K=myCK}yVy8|01d+_bBXpVy|GynE}iF+-SEPGCRai^&+pR0Yv$Xv&;Ph)
zuH5D}x)?O3`Yz2&QrpM&o|2M5UYuS4YAdHb%Fw3y2s?a7_1(avymsWLH?U2~Splxd
zDbi_dwJ_u7dA!a0VB!@>u~5SLjiSE|dIf0}R%8g-rX!(BlXd}<cJ#Z<7sEKSDy4*7
zYk)MAkaXnlnVzwIO%RVeGX;LmWUx~_iR81Ui=q<AJgFSwnJQ-kDqCM@PVURbwcW4{
zHsdTJ3Y;i>j5Rm+<}Bcn68$BK+?pS&ixNpm`OL@V^X+vz&5YZj+_JbN>WP7Kc<y}y
z<Y!YSGNV;3xRrSuPnc*7r-WrJiEu0|6GgS%d|q&FFT~8-nxhKeY<FzL*-cF;Ald`$
zqDiebW`3wjL2unfXIqJ)Bpxm=B~4$_^|d{R84slLm0r4ZH>ETQ>QU!zv^u(i7VZvP
zm37>YDweq@o2PeNmT&}yhULCX%(#aRok!Oew^2EVSmA;*UIsN}apN69++b>MkoGIM
zEk+1PkYj*MSigi4@&Khf#KaQWU6(I<+~SLSMmKBbhAXjpnigl$Xy%mrPJ5^H_qJJ$
zt0;!~AyDGq<j_Yc`1y?|QPn<&|C_&BnXb@*BW1UXKWo=jx)O{<S}I{7MeAE>;+T10
zUEK4bie7ldAaG5AQ&3vVrfqTTx>#>VEC5gCfu|i`5AEc-%ywH8twcj+{3^gappEj0
z`uaQJ=GTE3V2wb}eBhq$u2!hZ?qX@nxQk=L|0{*<5JweP_Q;Z>4o>G1zgsQ)ew+7;
zme<{Cx&~1F<FEcjtC+DaV1GrkV=#P2nxha++V{lI-ov_2gEbFFM(?i(SxOZ+6FF1M
z24&O7_BDXaS#m-DJbg3ZJHlq`SN`1kxyYF{xLzU{$IpFyN<gBX<kq!Ayl5h_NvhZ!
z!Cc=oDl+$P8r{rHF!bPIZBg0F=H}^mT<C*>{L+h&z&V-9V96qT=mOA?y6B}?uL}Y(
zC=LgWw@O}+9<I-uMpuz1m{6!{`VrLT;_)Az>-;i|aKx-zk5O-ZuZ}@YWDe#!u7ZAU
zUX1RR%xU{_ty5S&nD)h{26kG4J1U7;O@V@M?2YS}?CsLA3kr_k)CJPX*qQe_?a29`
zCA9!9mjETn<!lOW<g50ARfn{!TwO?o0JJ*DKweVpdyr#}Df86qVv*tYFkIPwg`dz^
zBD?j4T{{)RS7Ekj>{}WFVSfxA*A23ApE43Vth-n3rKByq=^=rrV}1!T{B0EI$A7hY
z=C%k>&y%VB0ichsofqbyG{iNb8$4^pW(l2id1>{O7z(w$Z@{2wL8f}E`RM|Qw9KR3
zWnLpfrgA3w>)?{9o|e2#u9F}<RPwRDE^PqWiDxDBOf_oR+Tmod@cbyip&*f>DEmAM
zlPPp7S1}qoVx6i#1CLQwG@!TL6vj6$oVcC8+Cz{~sZn8xp`+1FRCXZVty|Ojoea0?
zq~aK_W#0!<8yhM|l{#Kce;&vG$!{x7D1OTc_ngS0X|=nOLp!_oMABZeC4*1yBI}u5
z*qAPN%%}ZuaP@>iDtqDQ2Z_N${ZvS5Gu*;&G=|xEXMaLrbW$hdEFGXQ`5L=)*{yoK
z=|lPf7*>(44<J^}Gz`1At)ZLShIpxCp$eg`4bQa)BmU~}3eq_`B2JL_xSn$fObA{S
zPRU)J;*!$_ck?5}Q&lDFYchTP5|Tx7zL?qk0wyKw=oeQEywSgU8Sp!EoIjIH3npk2
zKyCNH{3ClP)G7mxRm9yw2^1^C|ClE5qo6#F^<7dS;Y+`h|3`XH)S_v|^k-TtG>&@D
zt7PkVUc92T%k5mY3hVN^^H0~!%7<yrp_sI!o*>gLdd<3>XuR6!Tp?>yKFImo-{fAc
zpl4wMW3gjW?K#cvS_OcMth$QFQBg%=oJnVbk<*4*o6sC<UkB(5LHBc%fYG$fMx+4_
zJuzgwx53p{6)=F{mPZ&`d#*_e-EaNVaJY6Phyx=f+2w9NwmQ`IDJUW0v-JY&^>CGR
zhF*+t@XkgI%OUcB5?7!Zol~OjwTnW|&A9^Xi|6$<PnBi_F+!6~QEkLOt&=PTlSo}P
z*Hl0A5{aB$I48=IOZQ!&M5R;fWcw^lxaNZ8Qm}wvp!63&{%tKmQCjM^a`%>MqN!c?
z;zS%84-H>yYR?)%aqT&mi@jXICXA~5>XW0|S}y5}V3!)YF!_m9-rNq!$QDq(4X(AF
zwXHei>RK3@tAuMp3Dj$#Sp|f+V&cQAw{7&RAIhS`*?z+hvZP@r10Z*FAg(KU!2#RR
zcum5K@C+ZC%LeUW_!uZP=ir+`SSkX-)9LMkM*jpmL5=%+MGeKp(<P2^b?s}F`=Uy;
z^VTR0<J{6p@hP>>!3)YgJ5fQ2{8CWEngRo9QcWvWGZ!UN+7G3u<(y3pWs(&!@9E)*
z%{C(O7GL%euv)xQi5Z8P9zzi!S_L>hZFL;dEOAv6fmUk*puX4!^o4#&OOIj(DxT$S
z?xvaj=AeiUbE2NDVAPJrDfh3jxBhqx@4>-(7yujG0|)`d?`=SIwTU71(E9WtlMQ44
z^_Pj50(%TFK4N6}p1v5xfm)oPL#P<pEtN(RazV0E_6PB^NT$QuJj*%$pRIB3@ZIPM
z9&3={x5JA@PKwV+Uv=V9c}NJ9miT>``&c{Ze8X5#)M7HJZ2Sy_%<y%?b(#`gkKyl=
z_{_R^YB|ms8|`9$i(!Z}!9!d=j>hk5FZq8GOBL64SZ`F<U(ripL%(!B@gIA4w2Vb<
z0-YH18me{OEB|9@K{=DrJ`{{0`iej#JhY3ALaq;>smQ~L=(Y6omePB(cL%|Tv(>{+
zVB9W4MLO$B;yGrCCqowSZG68X$*`+LB5e&)HE>|G;jU0sj|7D(X(M{RGXe)l?}p+Y
zSZ=AU`ln`{C2lsa?tEBaMJxL#uAd`HW<AbrM2bc4L6Rx5{^n;jysO-Usnb-js-wNZ
zy*OF`R$7XVGU_`NDsF)`D{yPlXhtz<8~U_ig!%})28$Re+3F%#TJ?>g<)iA`kZVIA
zv$drk)sGun*N=))eVE$t#}=S%YArnq5+hp_QZ@`?zu~r!?9w8VWUX!tA2@HJQ<wt}
zFOsJ2yE*#Y4Sb(212CU}G0?o4=4y>uu7mg24`e*4m~LudjA^-xU&>L5V>neREkF+F
zZvse^sNehQ#MVRoO&aO9tc5=%T;L?;G)}8r>BR(mE&Qks7exR_eJR@c2{m;zts62A
z1rZl|%&|V`F$9PW>?79Zk_8;x0_;ZGc_bsRC>2f51bS)N^T3qEZ=*qnd@||hT$I;s
z`k!*9KC&XPBzAwUvbvzHxnft1W_Ir#5~i62BgO9;`4A$=ft)nwH+0Xbc06i5Z=?Ss
z+&DF;@(w(g2)ILJIQ)O6iQ2r;_30Zx%ERCU$)aUtbhxnHtD?1+FZ8V>C0wS<XZh*1
zJWo61Yl(K=G8iEKqqz|&<$1S!uo!OW)D{`Q8fYq{$tO^*4@{jgR)wz{`D1oMhWTP;
zh6A}ySXexip^X<~N#BX^M@h_8y^ZnXn!xhDY3NzFPL*>o!tVu0jS^YS9Ui;9o#aTS
zX>V~v>V2pGpEl3bN<+fWdHc{*cybJUCd8zT&Mo-&<HC}M2?Qi~god{Yc8XzlFVF#9
z(YNybfyIRyJ<m}3%!TPlMWIoI8rQ5=NQD;c#paqt@m-~+`h;UVmaZ93+&|@wF_7@5
z-@-2N_{z36h)<G)0(bb7W?%d$;eogFY+Nmu7=QW&lL48i4SG%P`d(9Pgq@3PC1vrq
zu`$2g5*xOfDpZD8Bsyr{MP%ZU>gL=JAdnMX<J9KU|DDG1@ze5mco?_w>!6t6l>TEZ
z-F)y5mvT%67rgCy>Rv0$p(&Nh*X-0_NQc>uu3dcQadcG@Y}s(TOCcJ;1p8gHzEN#%
z;VUtmaF`!fod*|><DIb1*dSvnDS={+c!^vaqqVGU+3Q3+n9vHEoWI_T=^UGPNAHy^
z%jb?5$uZcwYHb3wnYg)j2TM33XnCVo1J^`08FGuvA**_g17TBZ&=PI`BvnT){CG9y
zL<Q^nYcgAeR?Mr!mJJ<+=m^$aL<XK&2{nzI-r`>u2m)@AN=u<ee=)0?Gj=<J>%{4|
z?#u+@T<RqwWw_#hrTS_m?(eW8zhQVF0w-JC|3B!1PwiVXVx{#u2Kn0WW)*p-`?3sE
zY-=6P1Jrb#WF!Ud<FR$}8YIMpHK}M@_=cf~oQ+$eev!0OuNN5T5N#>ZT#|qY>&Lf&
zar<@%aMX0O-q|Cb!rNyzL@mK%0WgBg(4X)VTZD?;C4JI4?K8q=pN9D}_GG45v}JJ!
zf#TYEsjcbs(6jF;%9o)pNqj6ckP3Lh25vLh3^fDNYy*opRL^^%6T@x>dW=$RKI>4o
z<m;-!)k;bICD4^x;+Xy>Yzm^6FgqH^vp-aZv-0rAJ#IW-?c*4MPdBfLC0OBKbRh|L
zu1BSq<X~<bL~F{$jIe^gLQQGOh8C#8=uiOa=H&{nx(B2tDrW*`%K|}Q^9@2rxZ+`=
zG9Im<9|LL@*0K?C26%D=SReuH0p2%7)W#JAwE!-io4t8*`TQQm4sFw6-hlZKyQ~>}
z_k*SCaA?Q+xv4=1H}Z8=Y;yyKY7W>qX29ErK#zSyBkB>KpH&SiB-BB~XC=czb{L6b
z|F<GWkogBttzaI^=mb|#4(B|aCQ<C0Vz?UL09#^wT%XAYIta+uZlZ8zl0n}wAJ05_
z@YJ=y7Nyfyoe>YTUE)F5Ol9yWNu)EtYW%e53a6n^iePe*+Y&PLm&x?tha=nasi(M%
z8OK+>Ac<g=ddq)$sK~59GeMun@WGBURiP)U=ydR+*9UBYE3&XVP4f9Cx8H<V%@$r<
z|JE_F4)mFvbaM7Bl2WA?ngp^dS~!g!j_%k}dnN%d%NpjvMEo83>+xu~2rf#T+XJRV
zRbfPnMoi6iFy=B-HsD`;kva1RGdE8b8@Qksd1X!kI>5<$vN&o*PJ=IA_lLu~Fb&2m
z-Ev~|8$u!1-7tHiqid=RB~K92Rkcqnm_*Yn1tiFf{?o0GDai^XbTOWYTNaR?v<ifY
z287!Q8gcp`vWKOijX}){oAJVD+QUJ~CAoJd@Jwf$)p)u`NvA8*+2|!qXQ3%xu^Lrp
z{GY3`U^FMW-*8+yFRS1^k*W3top)BFdH)#EHFp$5o(XrSze2DWY8+=JHLuL4RFyx7
zNQ5?e6|y*6IaHbNB-p71H>UlJny~tBaR5m_pZSRTikV`-6OcwSl8V4pYH~tfkr>kV
zR7@BeWQ@(W2J=eSTo#eSG}8n%cHD$v=Qe#cjQ7okR&r6<UvDLAagKdPhHDkoAt&N~
zUsRKEhj2ukMA<a!Gj2~6%}TP;5Ih1bL3&bkb~yac)_=SSRVsNTup8lG#sx8?M0zmc
zdnoo8h)@Qlsn7w62v4lU<9d%8Ix2rgpg!i01ZKI+Hu@BpA+k<+wl(*fIhL#9$p?i2
zMw&x&Rx#BjZMQHyIY|~1tsodA0$Th?cJ)^k`Yn%X(OX9;O1#^3MMbd6StBMoeJ#8c
zLzWCe-mhuE=HXdAMqNJPEky0NB2`Tt?*66DM|<Bc{<J5na=6W@fJ$1ixt+1E4MIZI
zL_(iMRJGWNLk*Sxc0;#OL<f-vB@W3lLnxbb-WC$M4-@TX9<5)K(xe%JwCsaIm}%x6
zI2i-wiE@A}L4G|5d}=ls+%h5AqBHbigwi2w(P`LGAaQ`zv!%S9xH+&dK|`1Lyw0)Z
z>Cs@FRA_@VZD{=_Pmq4|9fZY|jMd)Au7#KXXCvP%Hyk5^9lG^jcO=J+Q$CTp=15S=
z_EKb;RF+Ivq7!%cs{2n-40Jp6UdY4W7B9e;liSf#o90u)ct~)U3jb&<e9oFTJosHg
z<Zsi9-gIUyVY+-4Sx_qI&>G!j{Bw@Ng=qBj)y!TRzCDIXK5$6n<EJBk*QYBJrSDGV
zQ>aK`kXl<X`?^Hce<WwQ5g?jA1qVaxvCxD+xYnrJT;v;t&n%`8LdD$nD$wkF#&-EJ
z-8Sgqx~?aDQqBZ}ZZV4$#sOwIiA8cePluYdS$(wl@<!>))7XPlqL!<qf#vS4mv*oX
zk_dKsMP|(P@j7k$zG<ENf$VeqeD~UMt=$_h`e|z#;r}OZaI6A=IJJQ(GHa#$3=AZg
zG_G(|1}!!AsuYz|4dg8uT~q7j><$D6Io_3+5hZ7Zyl9j%M>RqVbhU=n-#$ziFq)`6
z>8-RHk>E`@?JOUgap4<0_7vYYSt^~Gw$yq!*jI;L$Z``4Gq8qXZ~mgp0in=~0hu<J
zTgpp!6j=MUM#^z%YDa5$Cj;eP=bnAGaO;T^3q<X$E>$FH^kk_^`BG*$ySOt?oe23A
z2c2sX4ACkmAgL)H4B>^`h;GaA1jtGzR3tNIvtCPPDF39CssxC$ybG-R5Nv4Lgo?2~
z3t|HY{q<xs<<psvGzT)|%RjFpsZyeVe`E}wboaBr|GPG0ce2^0|5)HFZgIQQIyjbD
z?|**CmW&GVD{VM~%IFk}I%AJv<@(R}W0(!oF{orqLNJV$FAAT<tGDadyjmM3kr6gm
zTYYM11;?ql=%$!RcYSHkw>uk`0<U}hPSJ~eAM5{+u&zQ@PQ{bY_36?D17YF*1O4E&
zmVA_%<*KQowAF%lFkMXEJyMd*F1wD0DMH_*^7?iuj3{*@yU{wLLTy#rML%uH;7_68
z_2n>uh}e~8dpAA1{0OX%)zES{eat46-RLDtnfB&von~H90hn?Pnb_;o<*H&LwQmgK
zGoi$-`=y14iQoFY#z_iS#rvWG7K--*|4&?!WJuV^U>Pzbvx!yZdl>H>8Sm;NJk^bm
zTq2E#+@i2)SH~QVmag^sJ8&nSGU>9EXm@$FdTeBQH-P~a#kr_&;&$}W%kuD3=>l>D
ze=Xjc-aUZm4r>9_YrS+l&SsD!{5Kly_TlIdx|pmHV&W`tKM(@%1qoKs>CN5d(;=Dm
zZ1raIxi7&&w#Cx&)g7hLD*R?xk+<>t0~l$x9Iki5yS%w6Um$A1O-51XIUJMQPHfj8
z_qJ~W4Q`i6-m;#74fp@@Vthq|tV_0TKWE|7#)tQ-i#nSJOV%xiaWuM-x?54h-^6dq
zmzSuNGMpN>U2}rk3Z<ex{Sh-az0XczXUu8s)FdVpUNfI=YV0J=OqO0d@@-{<t)2-<
zPW#GMRcg9!BpGVE2mRlRb4xIXk_YvF9_w#_)8t~TUV8S0LP1UySJSVC7dT8dc`FW;
z3|8K&M}OLKGM4A(3b=JQlp)u&EvPmlwx8gtznLNlZ)ZdE$MoF;zjau#T{^O%XwK?)
zyf6e5-FXTiJc;@q$MtWU-VL1@U%Y@l*Fk=}nXBhK)#N^=*6%JaykjTB8_aYIMK>(;
zLGFfF&U&%(!GtW8vpOI6-zysCnt)8I`wI6Q*wzSV%F<b=dY{|Rtvo~(rv||4*z+9>
zNK##HR<I7lTJBjU-+a@h@q*WIk^7{M)IWD2qv|7=y{^FxN;-CLSuOy?DelmT)vU_<
zE|<=@Z*~LBICurBWM*5?dZ1?@1ZUx(Y0*bs^#fVU1$Mrzo<b+=9ud|0??iyq?x33_
zRK3oIHuWjcMm2$5uWL2T2(GI1rGGGPuf#P@)z5M0jxkbK#wN;nq?fS3I9S3s-6pmx
zb%3B{kZEZKq|*zy$gU#wU0pELwR(kkSmp6wKiR6XZnU-%&@R4Q@oUlg0d{hVmwu~o
zOhdSOCSiX6yc8WQobQPC3ZI+p=gmVynaMXEV_YPwy76}o9)#@-N&u4XE3%1Lhc5L=
zs%<Q6!Co~^4LjdTBqNsmAxsj&K5rh@xK(O`Hs~hpdzPxYPwmV`yYU+IaZ}TSYVl!A
z0bm5??52|PJO~2(L9Q0HQK|xCQV>96x3!mN=;u8`C$Y30j-lZFF;=LU#rh*#&OtJA
zXIb$&sPf9X%zR>X+IBM0dIWKW=KBs+;y6DVV&a&(%K^Mf_@h2!&W5_9nZ8<$iV*Uw
zOKu)jfo8-X_v{sR5}A`@7Zi{PkaX4<(2$sPHz|}<>g5k;hABL1Drc$T*vVfLaVS3z
zOeG)rE@XYDd4A&alO9bI1bt}GK+bYE1vZP;#@mRCwobOz#TwN9)TU;S0w7=<A(FR0
zN|mXxkc+61vo6xi3a1MWao;GP|0&Oz+Fm{xCd#5E*v&u|oHd?h)b9&P^dnSygrb!-
z!73ug*U~g#<x}P&nS@=a%;L3q2cW6$+zj&1<A;;bC4_+At4P571TMX$li2hynQj-9
zW|>$^w8;NtS$mwZ&;R${5wUcc;pK#I-wIP0XqdUf9RU89^#Q&&<K(!Z<y*>6<BAvr
zDEO^F)qS+8=;3^{`_5>a8j3##(pOE}W3MhuR7)v<lzB$$nQX|kDw@27W4x74K@d-=
z`vtQ@&-jl$wgu@K$0~kidg}0?es0fOg3Tlr=bl6M+G}Zq#$D}T9fFt#I)Gq~XGrh~
zTL?cg1NJJUi`(YL|7cw&r7)IR|ICG+%AW}_eRyS5f%X5gk$=7+`G@e{=H-WHVk({O
zTVI<D^j;%YdBL9t&lg`t=*uGG3=d4dcrK-(Z<uN**3ou5cO0=P(5Z?F;^t`fB5faD
z0#L$#9FBE}{7R(pNxOj~245c23A{IHFT?#%dF-`s^Ir6@si5mF43sm|D+cdmt0Zi*
zBM3(p_$mgn*9obk6nNyo1k<rwuck2=Gihh6j~8zO<88jA#j9hX3>k(_6Y%E;e$_FW
zdo|A(63!ij{o6a)SR^~T)*$;)x2ILmamdE|$c1<hy^M(#7wPX7Xr}6sZ$#JIe}CGq
zxte^faC;x&p`IE8o1;sNeM-Y)l1HB$-(^i`!;&;O!$$Cf?FfgnOPYDriivcXx|jnS
zY#K?08Br5m6%e{ovUpH&Utc&=p@U)>V)0EmH1`U6V@FZzxYHh=RUiF>?jdZ%SY~>l
zNHaqzzzb8dmD<chhWk7^cpc5H=(6#jYQL4$rOO`kxJ}u9lP;2Zu@@dTw4)}?$pqDS
zg6!9NPPdod(7GB}`2+TW`B&>Zars2#Bu*=8j<IX1vtAO_AXe`dfkh6~UBVw_0`Sel
zG%MS3%!`fmz1GEeJ=ge8tA1NbASDiTHvW^;EJeaF{uY{h)s*Eu->WLz_xMwNd7I<G
zo3}EgnP(d@6(|E6l--YKsgT|Qw_FMNRdR%IO<@iD9bE?BKY~0@G0Ptsns4n07%j0~
z%@LS0kv~mNxku(oL#|V<7h0?5TJpc}2N8C8i%O2WSmse!asD$|IleTSHa|YIYKcWF
z$wH?U(w-D2sSe5XHJrOYT&v>~{#}&asrSuY^QaUH*!z1xNIod(y(v|k(s(f{7y)27
z$a+ZyJnP<1p6>`>#|j1d6ZY@|tC(Juc2}9Gin`wEp47T!`zxeH2bC9nFl7YN3v|;~
zzn8!12I_!h;2{G$!tN0HQKna^tHh4boH?ID&$V<38VVmIbu_K2StonQp0;LwO@dmh
zrTNVHG!))9vY|>6bJNavilljOfUXil$kVcyxsf~yl-rzy9CbmTw6*sn9?~kIVRQ6y
zZ=Tbc&r0>&iRLH!JxYUz%<+AP|JC1&H#v+QN)U=zK&Eoah;#FT(saVjR-H&f|CGb5
zj%0+Mm}0G#qD}ESP8?FV5-QL&D=6G$jFVfwSQ(BE25IVKB|hS@8z6Zs%}jOD6@JTq
z30GJr^$>)>cZ~4`MQGuM?YH4qj2*ri+(2nKjpg;c{27&`f9{DnIJfN4#d`TnVd2%Z
zDAHeG9oTL5m`p_~qA3xlIU@3^q5%$Wp`TW;4g|EK{;01?967%Ji69{|?x{EVb@~sA
zyyH;1i-xQii0ZF4vw{;{Q!Zn-J_hK!nyn*bkGfS9Brs;@DmAhVH%geT)lmPwN{y=Z
ztx^qn*2EFNhE}1@@*9nErLSKWNVzD8T#4Eiw$PQMSw!BdNF%U`AL4B0mrr^(l;^Hb
zt?=>kkB}AEK`p)$6%U+uH_TOfbNFeJw@bt*@2ReFwSFN8w=~zLA={4iONpl!Z4Dm&
zP6XV;=yLajxzCfII{<&=W~U9UG|;8vcwfQl%1}8muTbxo`jl#@Jodqr5I9DOHK}i-
z5<M=n4n6Nb=g&D63Vr$q!j{BB+=>kksTrM3B&!F-)Df?hBYaRBR09@4m(y*~1L{f;
z(4jmw=a$|a|M!m%gtS)HV$F-a+ElwlEEO`*Tv=Vh1VjP(4+iky)SIT-LUVap75eao
zPPHlyg+(5`vXP&TG`qoMEi3b9Z5C)R#d8zJhzy_K>%?QO4uxss9`pRg61<RcitwX0
znKr9uTuuQJys&Xs=`dT|TKJF29Mzuth#ZaQUc@{cC*!aN4!D{bnDKCOV5T81og9ga
zoMhifV{9s!J4e6hSUGxLZwEEpCmns7wb=*2vZu-RN(n>sLJx2ZZUVzE-OIj4OKYRp
zY-GJ~t&QK^bIxZzi!P9u`k8ewE$nL8!8GCr0C4193vV?Kn3G%?8I?+G_{NfvrDDX)
zX#UB1JKT{`R;v3F8|3J+W!54}l~R3?nng>HKo<ZpnZ%n=&Nr0nLOMskZlX}Bvkl|P
zPWDiW88`3pttlaCKVMHJ=(s7bMupKV-T--Rh7`w{Ef87dQPQJh5XAvu^pky2{w;!~
z_r9w0TtM0lMxLOSxvSn=S0WYimg*+yk@ENx*Ci)eZ77uvcdX-)X7|&n^1MU>)!s_=
zRjRpwa4L)m*KM}Wy{QHdYYr{#H2AuGDRE@I3tlC^FA+U65j$ncK~J9v1F$j6bKm%l
z_tQYOfuRLLROZ2MTCHAt?x9E6QS2}d8~z<Lg5vLa0k&E1pJ+|IrHlvdL4e$m_HXJF
z!TGE~7f1%ZgHd87a4m+s69pHTo$fUq1M|RfhSZ2z#|HlIIcHL@nXYfMTy27h`oNm&
zf<j%e!Gv1}2c#6&wetzp8FTcx5A#Gasd#0NL8m4c$_%s8aU50FzFiH3u1gSPI&ZkB
zBpG^k<_OQ2?O8*2Jf?LAAr-1eQ%(};rAvs-VkZ&#d_pr*x?u(C@P@qmX>wP|ED4Pn
z$(1w*@75;R*t5H=0E6^2Xfb;-7T5xZQ<e%cuud*ZNO#Si+@Gj1)&i%<rWPVY#bFtN
z!VL;rZ8@@E@!gK?5RbH#e3yH4nA(&jPBvs+lqioMhx)!P_IL1m1Cv!f=bx#H6Nlr`
zr_4^N-h#WYrb;-i)~gi{Za@-{|GZ6Ee;L1b0Po=i<kbDIm$TJFJU>DzgFQIo2=`w8
zNQ`YUCIaBt%c5L2pkRaz<hiKIDY7!e?)h`Gv;F23Gx|=B3Bz#u1HaR~;mgL=Cv6b?
z{13_T<xHKfN3W;!n^!$ayo~trnKHDmr@%m}I|()ke${lDdqw_0Hye=Y1F0{~or`pW
zU-*B`;U;(TQZDRAg*SL*{8`4GnJMevR>z}>UvN-6Ap93||F~J(mQx{KJK>$ZR8{1q
z@RTl58ThfNb^uXOkjN^z`#RE*f_b}R6d~^$AFn4mscFnhraXa3Q`>7;dI4%e!SjAv
zSXG>49EUZc0g1j3x%N)u*Jj3Z7l4l12pj33mbxFu_gCPI*pdKE^a7?N5|vx^_wG|N
zVtHO|qwFdMMU922Mvdmnb<7i=14LlPyxu>Ms2xXDWY0&)k897bDS-JTbIUm)0wl-|
zpNje<P%)D?VgEqvyo8waF}B)J^;Fiwd0b2WmcLJ31N<m0m8O8#;SkeHbs?dnF3zsl
zm;fTvnZGB{4I@#KpGT}3lh9c!`(%eZs7eu~aSBGppR3oacRNK*csh;1|LMpvczv6l
z8zO`MEtZ^Nfr-JP2hXKW>kRYLHq!ec7kWBFCVzKFA)5!4dTh4$D=zTU)nz)aW($Zz
z<GO=BnM9eG#(F*gwu*xrTgZ#wt0n0veC4;2BJle-C<3BJ1%ENNhpb#7v~HdA%>_u#
zH<&rXtV1?CLWI-2r+5t3{XhUe##}UmnBTg(>3(nuF0fiCPQ1+NQQgtBVpj%o@~_rb
zy&9YW)XW3XnH)wuH^!vSUk`!Evd$1~Q%Bl~Qc!*)_TjRz5eV)oKPZJ=-<?D!A~*`_
ziw^G0<4#DB-}Bq)fgMD3=TcT!T?r#)8*AKK#D_5GQ>e)f4LpvyE5Eb{c8?KaN`1<F
z3{il-<z?A=mh}YDUKWXG$SU;G_WA!oP|mRJCT}bwXKV}`%H<wAo2NE8aB(Nb+42O<
zVEykXbZY+<Q8M~dL}s`^RZu6B>31RVS=X3GjPi6F-Dvx*&+esGKI;shn=su<^)@cP
zPI;DI{d^AxE462kt$*hmkYL`!{d5CUv<o!7eb(UW4z$Jh_xz1Pk9S!8bTKrw6`0G^
zbt~jJWGlZn&98tic@t@sg^8PS6;feUxkNGg=*5Pywb5iXEJN(WVOr;@7$59zIbipO
z;kOKjJzJIh<9T648rlC71Dj7rIkHn&lh|kNv@|~*K3=xqKN=i+A!{cyaAjI@06J`+
zDgj>)PVt<Jse~~UwbfrPUo)$W)q3Rdb2mTsqRra`^D2~Y7nsKv(8lfxEev*2-5O~6
zJ#n)k9N}%CR<}aNmhyXzP_)K(z6b2f#?@y5K+?&7@dm6<v5xyenH<987=Wr{bd+_a
z^aJ-nT6+@qns#tlRiCBu=B5XgnVZ`}^TxssnU>gj;lgwzgD)m&IxY*XBdWqUu}#Je
z30lMpNQC6Td4E>&B!!_kO6}AA3;>J1IyI4~VuMEEnj1@eBVTHWFp3SWj7!zl_D1mD
zGQ@<RhH^(1N<7&h9+px2c0}#svaDuD2JeqMs%q*^qGaS$WQfYO>wqPel=~lvgeGeh
zv1T_4K)GhOo0nmjsnSyfP!OmCbsVtH^KNu&S}(lzsh%af!4$*qo@!YwZD9Aa<#3~Y
z#=#!b>hz4yW6p{TUz+D8TCk5^H?H*Y_Df5fPJtM6%>cgdX*YEPPhpIGIK>pgJRSp$
z`swa7X1$}zBR#?^;|3j))=|tkD|1lM!wJbOXNM7WsGa@gg$UbQF$8+_oR(aE;J1H4
zO_5}DV3^!B^~9Q%BJqE}_7rRPEbxRz3%))(D)enpv0L1YY}#Ajk^eX+CMlUp%%T;>
zdtkDm?+;7P+*&6s`(Wv8rrE{Hl#cgFeNV&-G-Yk)_E(5;TQ<N~cXAKWoi)C~jix5t
zlCZjGY4C&tT!H<9!M<|^H0{C!@AXtm9EIYfgMDk5GQr^m6R86Xqx!Qx>QMeB-IcK)
z{F+7oJ0{+p^Ycbq_fEj{MFM4w<%)GdM|^%GZyHm)_N}pEH}Wd(nL{fz&6udiw*O6Y
zP9VTs%zH%6^Fgi9{l7u+Gh$b!`nU`m17R?&K6GH6@pqN21)%HV^EsDNcH;HMpp;w-
zX0`DmD^iUPdKGs$(ZNArFctBD9Hx{N(^FE;9k#5O5Q`XUtNp2N1bDS)0b58Qmj4&<
z0#S<c{Wqixa<7r%{g)mItpQC(tNM;+Ku)ukQ8c3ej;)J!YRCicu%)JRwqHbudtfQe
zT~G^3pHk!i?7ryGH3b}gKhGZouXfDqXLc%RQ{N-5HdH8NDQN%|%!eV-ukwR}SPGrz
z2LPz)>~az)m>BDjox96YN<K5(=)1y)REK1bcLi5h0$wQo@~ce^J=CGX!r8E|?!_ev
zK^^V-mlpoD-qlp!Yc<Z-`boRHXX80KK~TQNau<Yd;ageA;m+MqJguw&BCOV~eCqQi
z92<tkHoVE)-yfMVBg~VN%^CyN+fbx@Q@4+F2Iv9c3#R^+%=R`ti<*TX|46!WyY~}E
zrvb<7)a<KOA84S?8-c-~=%@X`zoUp}vj-kI%E)M?kX#(S1KR`IPk3Gn-}38{xNb}D
z!+iT^EFA)Dv3Pb@cVBv>USVlgD?}?rSn>gly?(^(XOQPcjyl$*ZPc(+akYIQkqaPF
zeGuh?$wwM0$(%fgXY6SeTLl6RwUq3o+jXDWn`^o$YM^!E;#~O{yEQYzvEDMDxd*GI
zXsflfUy>&iFAT#43cV7Z-Io918tlAH&C~>|8gTFX5FN3uFLG^2=^P&opEUl36EBt~
z8>Lu&C=K7J$b(ufl0a(n(S!(9SMNM-QX|L;@y2QlY*Ue4NC&pov0)~dU*%A(=g%i6
zCar3W>Nd~Z@31w6wX9&PZ%1P<R-NM_lM(EX!_=(Kg;u*!_e9~sxP*os9=8q-tP2rV
zN|7u1iW>*pOa`Q^96|r#{T7dU?Ca$q+%(u1IRm&2dVwUA7~r5vSw`u*E){lQ`)X3+
z^!7)tM>3s3W^()drtybJ3%^T+8J|~?vFjFB&ISu68&=?Rd@8r2B2){|+0ELgj7&4-
zSOj1-aIyS1e<F0TZ>{YX%HrYMEW$u`9*R$q`)zU($*2f12PKgt>M&KS&Mad#?OF3e
z89`fV_bi*ksIvJNls+u5r?LP2*y_ZvBd25&5Swu)n!ja!eKA9DmL%Zn&$mA4v+LG#
zN2u8%%7we?c=;g-b<K(Lgx3fPK_}#br4E%TXkqo;*#Do>?!~%Tll-<M3v75<3#G;;
zs(uYVI%<@|1~mn_WLs>l^Ly_u48obApE=Aw>G?P1-3s<lxfFtK-1G4Dx(u!LMx4pK
zpQg^5ad{pCk@j2=0sL+0uf&d_r`E;O-QC`51(rJYe0hq?F@J8m%>T3OlSBwJmruyu
z*XMuqW(*8HY=_p=V89BeKsdZXzAB04PmdDzMc(%Ro&GP~QKL}3DzURpBP+ZaciRUy
zx{g)#1Dd*qa50F4u@Ikrsq!7ENp1>S;wl+A3k1|CM0Or(N>1=oEQlWx%vC;^c>lK9
zOtUe)lj`oKRZvW2#_NQ(e|6YDv)M{K0nF=zDdcsoDmZ~%$w00<2*T@xc;j+-lkXk)
zXUI+m&Z0`mLsyDc0ig<Yq#ZC<v0mjNxu0M6QhIy5)s)cw6)4S;Bm+ITrB5OM%C`5F
zXtv(-<2$UDs}X>DZ6+!b!;&FO#D*<rZe0skBjb_DFjR$U9aS8f2>C0gI|xj#?aZKK
z-6%9<ea4rVP5ejb^bS^if2yM(xEE7Keop*HYPoBsFKRf{6Utv^NMFkstiYI$P`Kqk
z9M=*gr^3d4DiJR-c=r!AM0!J1FsR9RyOcllkQ11twBTu-NlYR#c~2xO(5tISb?tLR
z<ivBnS(@#t>fw``<AYlY@_v}W?>7lR9M9Bs;$1$ZfJA)Gg-FN=a#178ewzLAcHQI-
z4oJRPV|coMad~fZf9u`R-_s*EKV7|@PF2Q5MX1}I9nZb@-I32YyVo@b&!jeNu7zH~
z1?*#na9~6=yIP<@yC3dSNLB0bq4b=~8_?)-oaMz1@b=5V6;)In!mYPo5oc{@%}6wS
z{NgIhG#JUSEo{h>97t$m4bn{KFRdA`-Z%OR@>*KT($Ti3@$y1%#+<))UWP9PHt67}
zOD<e`4H8pM&0zK>!fAz&Slyc=hkQY^O(GJG9-VV!T8mQUe#}#BxM-T#1FgFdIpm6R
ziOIO#q6-Q)7WUf}$p!diDFrkAM3D)pDmPk>L-Rnekh8-o1VbX)ZXXfTT~BpGQjYXG
X7`ZwHoUi@sY32V=!hr^)<vl0ag9Pfb

diff --git a/src/bin/travis/testCentaurJes.sh b/src/bin/travis/testCentaurJes.sh
index 96e5e8e8b..059de28b2 100755
--- a/src/bin/travis/testCentaurJes.sh
+++ b/src/bin/travis/testCentaurJes.sh
@@ -32,7 +32,7 @@ printTravisHeartbeat
 set -x
 
 # Unpack our credentials and such
-openssl aes-256-cbc -K "$encrypted_5b9e82629fa8_key" -iv "$encrypted_5b9e82629fa8_iv" -in src/bin/travis/resources/jesConf.tar.enc -out jesConf.tar -d
+openssl aes-256-cbc -K "$encrypted_e1386386891c_key" -iv "$encrypted_e1386386891c_iv" -in src/bin/travis/resources/jesConf.tar.enc -out jesConf.tar -d
 tar xvf jesConf.tar
 
 # Do a bunch of crap to enable gsutil. It's possible this is overkill but it doesn't take long anyways
@@ -48,7 +48,7 @@ pyenv global 2.7.10
 sudo pip install --upgrade pip
 sudo pip install pyopenssl ndg-httpsclient pyasn1 --upgrade
 export CLOUDSDK_PYTHON_SITEPACKAGES=1
-gcloud auth activate-service-account --key-file=broad-dsde-cromwell-dev-d9c443bb4a94.json "$CROMWELL_SERVICE_ACCOUNT"
+gcloud auth activate-service-account --key-file=broad-dsde-cromwell-dev-ed9fc19865dc.json "$CROMWELL_SERVICE_ACCOUNT"
 
 echo "RUNNING TRAVIS CENTAUR"
 sbt assembly

From e726c41fa9f0257b7d842885fbcfac1e0cc2f020 Mon Sep 17 00:00:00 2001
From: Chris Llanwarne <cjllanwarne@gmail.com>
Date: Thu, 2 Feb 2017 17:50:11 -0500
Subject: [PATCH 41/84] Files in more complex types

---
 project/Dependencies.scala                            |  2 +-
 .../impl/jes/JesAsyncBackendJobExecutionActor.scala   |  2 ++
 .../scala/cromwell/backend/sfs/SharedFileSystem.scala | 19 ++++++++++++++++++-
 .../sfs/SharedFileSystemAsyncJobExecutionActor.scala  |  9 ++++++---
 4 files changed, 27 insertions(+), 5 deletions(-)

diff --git a/project/Dependencies.scala b/project/Dependencies.scala
index bd730569d..a964db00f 100644
--- a/project/Dependencies.scala
+++ b/project/Dependencies.scala
@@ -2,7 +2,7 @@ import sbt._
 
 object Dependencies {
   lazy val lenthallV = "0.20"
-  lazy val wdl4sV = "0.10-3ad63bf-SNAP"
+  lazy val wdl4sV = "0.10-76bf06b-SNAP"
   lazy val sprayV = "1.3.3"
   /*
   spray-json is an independent project from the "spray suite"
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala
index 11982fa89..b510c2f58 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala
@@ -500,6 +500,8 @@ class JesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn
         }
       case wdlArray: WdlArray => wdlArray map gcsPathToLocal
       case wdlMap: WdlMap => wdlMap map { case (k, v) => gcsPathToLocal(k) -> gcsPathToLocal(v) }
+      case wdlPair: WdlPair => WdlPair(gcsPathToLocal(wdlPair.left), gcsPathToLocal(wdlPair.right))
+      case wdlOptional: WdlOptionalValue => wdlOptional.copy(value = wdlOptional.value map gcsPathToLocal)
       case _ => wdlValue
     }
   }
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala
index 83d3e67bf..e72c201ae 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala
@@ -203,7 +203,7 @@ trait SharedFileSystem extends PathFactory {
   }
 
   /**
-   * Try to localize a WdlValue if it is or contains a WdlFile.
+   * Try to localize a WdlValue if it is or contains one or more WdlFiles.
    *
    * @param toDestPath function specifying how to generate the destination path from the source path
    * @param strategies strategies to use for localization
@@ -229,6 +229,21 @@ trait SharedFileSystem extends PathFactory {
       }
     }
 
+    def adjustPair(p: WdlPair): Try[WdlPair] = {
+      for {
+        l <- localizeWdlValue(toDestPath, strategies)(p.left)
+        r <- localizeWdlValue(toDestPath, strategies)(p.right)
+      } yield WdlPair(l, r)
+    }
+
+    def adjustOptional(v: WdlOptionalValue): Try[WdlOptionalValue] = {
+      v.value map localizeWdlValue(toDestPath, strategies) match {
+        case Some(Success(adjusted)) => Success(WdlOptionalValue(v.innerType, Option(adjusted)))
+        case Some(Failure(t)) => Failure(t)
+        case None => Success(v)
+      }
+    }
+
     def adjustFile(path: String) = {
       toDestPath(path) flatMap {
         case PairOfFiles(src, dst) => duplicate("localize", src, dst, strategies) map { _ => WdlFile(dst.toString) }
@@ -239,6 +254,8 @@ trait SharedFileSystem extends PathFactory {
       case wdlFile: WdlFile => adjustFile(wdlFile.value)
       case WdlArray(t, values) => adjustArray(t, values)
       case WdlMap(t, values) => adjustMap(t, values)
+      case pair: WdlPair => adjustPair(pair)
+      case opt: WdlOptionalValue => adjustOptional(opt)
       case x => Success(x)
     }
   }
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala
index 09cac5a8e..5cdf0e0a7 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala
@@ -14,7 +14,7 @@ import cromwell.core.path.PathFactory._
 import cromwell.core.path.{DefaultPathBuilder, PathBuilder}
 import cromwell.core.retry.SimpleExponentialBackoff
 import wdl4s.EvaluatedTaskInputs
-import wdl4s.values.{WdlArray, WdlFile, WdlGlobFile, WdlMap, WdlValue}
+import wdl4s.values.{WdlArray, WdlFile, WdlGlobFile, WdlMap, WdlOptionalValue, WdlPair, WdlValue}
 
 import scala.concurrent.duration._
 import scala.util.{Failure, Success, Try}
@@ -101,12 +101,15 @@ trait SharedFileSystemAsyncJobExecutionActor
   lazy val jobPathsWithDocker: JobPathsWithDocker = jobPaths.asInstanceOf[JobPathsWithDocker]
 
   def toUnixPath(docker: Boolean)(path: WdlValue): WdlValue = {
+    val toUnixPathFunc: WdlValue => WdlValue = toUnixPath(docker) _
     path match {
       case _: WdlFile =>
         val cleanPath = DefaultPathBuilder.build(path.valueString).get
         WdlFile(if (docker) jobPathsWithDocker.toDockerPath(cleanPath).toString else cleanPath.toString)
-      case array: WdlArray => WdlArray(array.wdlType, array.value map toUnixPath(docker))
-      case map: WdlMap => WdlMap(map.wdlType, map.value mapValues toUnixPath(docker))
+      case array: WdlArray => WdlArray(array.wdlType, array.value map toUnixPathFunc)
+      case map: WdlMap => WdlMap(map.wdlType, map.value mapValues toUnixPathFunc)
+      case pair: WdlPair => WdlPair(toUnixPathFunc(pair.left), toUnixPathFunc(pair.right))
+      case opt: WdlOptionalValue => WdlOptionalValue(opt.innerType, opt.value map toUnixPathFunc)
       case wdlValue => wdlValue
     }
   }

From 7716a07ecb3bbada030cc6ff21b23efb1e4729e7 Mon Sep 17 00:00:00 2001
From: Khalid Shakir <kshakir@broadinstitute.org>
Date: Thu, 2 Feb 2017 17:45:14 -0500
Subject: [PATCH 42/84] Asynced the standard backend execute/recover. Added an
 extra akka message check just in case we miss the abort message when our
 scala future eventually runs.

---
 .../standard/StandardAsyncExecutionActor.scala     | 97 ++++++++++++++++------
 1 file changed, 70 insertions(+), 27 deletions(-)

diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala b/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala
index 24b906494..5d30b63aa 100644
--- a/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala
+++ b/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala
@@ -105,7 +105,19 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
     *
     * @return the execution handle for the job.
     */
-  def execute(): ExecutionHandle
+  def execute(): ExecutionHandle = {
+    throw new NotImplementedError(s"Neither execute() nor executeAsync() implemented by $getClass")
+  }
+
+  /**
+    * Async execute the job specified in the params. Should return a `StandardAsyncPendingExecutionHandle`, or a
+    * `FailedExecutionHandle`.
+    *
+    * @return the execution handle for the job.
+    */
+  def executeAsync()(implicit ec: ExecutionContext): Future[ExecutionHandle] = {
+    Future.fromTry(Try(execute()))
+  }
 
   /**
     * Recovers the specified job id, or starts a new job. The default implementation simply calls execute().
@@ -116,13 +128,23 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
   def recover(jobId: StandardAsyncJob): ExecutionHandle = execute()
 
   /**
+    * Async recovers the specified job id, or starts a new job. The default implementation simply calls execute().
+    *
+    * @param jobId The previously recorded job id.
+    * @return the execution handle for the job.
+    */
+  def recoverAsync(jobId: StandardAsyncJob)(implicit ec: ExecutionContext): Future[ExecutionHandle] = {
+    Future.fromTry(Try(recover(jobId)))
+  }
+
+  /**
     * Returns the run status for the job.
     *
     * @param handle The handle of the running job.
     * @return The status of the job.
     */
   def pollStatus(handle: StandardAsyncPendingExecutionHandle): StandardAsyncRunStatus = {
-    throw new NotImplementedError(s"pollStatus nor pollStatusAsync not implemented by $getClass")
+    throw new NotImplementedError(s"Neither pollStatus nor pollStatusAsync implemented by $getClass")
   }
 
   /**
@@ -238,46 +260,67 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
     FailedNonRetryableExecutionHandle(new Exception(s"Task failed for unknown reason: $runStatus"), returnCode)
   }
 
+  // See executeOrRecoverSuccess
+  private var missedAbort = false
+  private case object CheckMissedAbort
+
   context.become(standardReceiveBehavior(None) orElse receive)
 
   def standardReceiveBehavior(jobIdOption: Option[StandardAsyncJob]): Receive = LoggingReceive {
     case AbortJobCommand =>
-      jobIdOption foreach { jobId =>
-        Try(tryAbort(jobId)) match {
-          case Success(_) => jobLogger.info("{} Aborted {}", tag: Any, jobId)
-          case Failure(ex) => jobLogger.warn("{} Failed to abort {}: {}", tag, jobId, ex.getMessage)
-        }
+      jobIdOption match {
+        case Some(jobId) =>
+          Try(tryAbort(jobId)) match {
+            case Success(_) => jobLogger.info("{} Aborted {}", tag: Any, jobId)
+            case Failure(ex) => jobLogger.warn("{} Failed to abort {}: {}", tag, jobId, ex.getMessage)
+          }
+        case None => missedAbort = true
       }
       postAbort()
+    case CheckMissedAbort =>
+      if (missedAbort)
+        self ! AbortJobCommand
     case KvPutSuccess(_) => // expected after the KvPut for the operation ID
   }
 
   override def retryable = false
 
   override def executeOrRecover(mode: ExecutionMode)(implicit ec: ExecutionContext): Future[ExecutionHandle] = {
-    Future.fromTry(Try {
-      val result = mode match {
-        case Recover(jobId: StandardAsyncJob@unchecked) =>
-          recover(jobId)
+    val executeOrRecoverFuture = {
+      mode match {
+        case Recover(jobId: StandardAsyncJob@unchecked) => recoverAsync(jobId)
         case _ =>
           tellMetadata(startMetadataKeyValues)
-          execute()
+          executeAsync()
       }
-      result match {
-        case handle: PendingExecutionHandle[
-          StandardAsyncJob@unchecked, StandardAsyncRunInfo@unchecked, StandardAsyncRunStatus@unchecked] =>
-          tellKvJobId(handle.pendingJob)
-          jobLogger.info(s"job id: ${handle.pendingJob.jobId}")
-          tellMetadata(Map(CallMetadataKeys.JobId -> handle.pendingJob.jobId))
-          context.become(standardReceiveBehavior(Option(handle.pendingJob)) orElse receive)
-        case _ => /* ignore */
+    }
+
+    executeOrRecoverFuture map executeOrRecoverSuccess recoverWith {
+      case throwable: Throwable => Future failed {
+        jobLogger.error(s"Error attempting to $mode", throwable)
+        throwable
       }
-      result
-    } recoverWith {
-      case exception: Exception =>
-        jobLogger.error(s"Error attempting to $mode", exception)
-        Failure(exception)
-    })
+    }
+  }
+
+  private def executeOrRecoverSuccess(executionHandle: ExecutionHandle): ExecutionHandle = {
+    executionHandle match {
+      case handle: PendingExecutionHandle[
+        StandardAsyncJob@unchecked, StandardAsyncRunInfo@unchecked, StandardAsyncRunStatus@unchecked] =>
+        tellKvJobId(handle.pendingJob)
+        jobLogger.info(s"job id: ${handle.pendingJob.jobId}")
+        tellMetadata(Map(CallMetadataKeys.JobId -> handle.pendingJob.jobId))
+        /*
+        NOTE: Because of the async nature of the Scala Futures, there is a point in time where we have submitted this or
+        the prior runnable to the thread pool this actor doesn't know the job id for aborting. These runnables are
+        queued up and may still be run by the thread pool anytime in the future. Issue #1218 may address this
+        inconsistency at a later time. For now, just go back and check if we missed the abort command.
+        */
+        context.become(standardReceiveBehavior(Option(handle.pendingJob)) orElse receive)
+        self ! CheckMissedAbort
+      case _ =>
+    }
+    executionHandle
   }
 
   override def poll(previous: ExecutionHandle)(implicit ec: ExecutionContext): Future[ExecutionHandle] = {
@@ -381,7 +424,7 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
             jobLogger.warn(s"could not download return code file, retrying", exception)
             oldHandle
             // Failed to convert return code content to Int -> Fail
-          case (_, _, Failure(exception)) =>
+          case (_, _, Failure(_)) =>
             FailedNonRetryableExecutionHandle(ReturnCodeIsNotAnInt(jobDescriptor.key.tag, returnCodeAsString.get, stderrAsOption))
             // Stderr is not empty and failOnStdErr is true -> Fail
           case (Success(length), _, _) if failOnStdErr && length.intValue > 0 =>

From 6736143def5805a3c0fa00c63db39b69260bea3c Mon Sep 17 00:00:00 2001
From: Khalid Shakir <kshakir@broadinstitute.org>
Date: Thu, 2 Feb 2017 19:27:51 -0500
Subject: [PATCH 43/84] Moved conf override from ref.conf to app.conf and added
 test.

---
 core/src/main/resources/reference.conf             |  2 -
 .../scala/cromwell/core/ReferenceConfSpec.scala    | 63 ++++++++++++++++++++++
 src/main/resources/application.conf                |  1 +
 3 files changed, 64 insertions(+), 2 deletions(-)
 create mode 100644 core/src/test/scala/cromwell/core/ReferenceConfSpec.scala

diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf
index d2aad5422..2cf93549a 100644
--- a/core/src/main/resources/reference.conf
+++ b/core/src/main/resources/reference.conf
@@ -21,8 +21,6 @@ akka {
     #parallelism-max = 64
   }
 
-  actor.guardian-supervisor-strategy = "cromwell.core.CromwellUserGuardianStrategy"
-
   dispatchers {
     # A dispatcher for actors performing blocking io operations
     # Prevents the whole system from being slowed down when waiting for responses from external resources for instance
diff --git a/core/src/test/scala/cromwell/core/ReferenceConfSpec.scala b/core/src/test/scala/cromwell/core/ReferenceConfSpec.scala
new file mode 100644
index 000000000..0bd9febf3
--- /dev/null
+++ b/core/src/test/scala/cromwell/core/ReferenceConfSpec.scala
@@ -0,0 +1,63 @@
+package cromwell.core
+
+import java.net.URL
+
+import com.typesafe.config.{Config, ConfigFactory}
+import org.scalatest.{FlatSpecLike, Matchers, Suite}
+
+import scala.collection.JavaConverters._
+
+class ReferenceConfSpec extends Suite with FlatSpecLike with Matchers {
+
+  behavior of "reference.conf"
+
+  it should "check for colliding cromwell keys" in {
+    val classLoader = Thread.currentThread().getContextClassLoader
+    val references = classLoader.getResources("reference.conf").asScala.toSeq
+    val (fileUrls, nonFileUrls) = references.partition(_.getProtocol == "file")
+
+    if (fileUrls.isEmpty)
+      fail(s"Could not locate any file urls within:\n  ${nonFileUrls.mkString("\n  ")}")
+
+    for (fileUrl <- fileUrls) {
+      val fileKeys = getKeys(fileUrl)
+      for (nonFileUrl <- nonFileUrls) {
+        val nonFileKeys = getKeys(nonFileUrl)
+
+        /*
+        Key definitions go in reference.conf.
+        Key overrides go in application.conf.
+        FYI: Stanzas, for ex: "my { empty { stanza {} } }" never collide.
+        FYI: Lists count as values, not stanzas, and will collide.
+
+        We should never define keys within cromwell's reference.conf that collide with keys from a library
+        reference.conf. Overriding a key's value should be placed into application.conf.
+
+        Examples:
+        reference.conf definitions:
+          akka.dispatchers.my-new-dispatcher.type = Dispatcher
+          akka.dispatchers.my-new-dispatcher.executor = "fork-join-executor"
+
+        application.conf overrides:
+          akka.loggers = ["my.list", "of.loggers"]
+        */
+        val overlappingKeys = fileKeys.intersect(nonFileKeys).toSeq.sorted
+        if (overlappingKeys.nonEmpty)
+          fail(
+            s"""|Key(s) overlapping
+                |=> ${overlappingKeys.mkString("=> ")}
+                |between $fileUrl
+                |    and $nonFileUrl
+                |""".stripMargin)
+      }
+    }
+  }
+
+  private def getKeys(configUrl: URL): Set[String] = {
+    getKeys(ConfigFactory.parseURL(configUrl))
+  }
+
+  private def getKeys(config: Config): Set[String] = {
+    config.entrySet().asScala.map(_.getKey).toSet
+  }
+}
diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf
index aaf3d4148..a27527934 100644
--- a/src/main/resources/application.conf
+++ b/src/main/resources/application.conf
@@ -1,6 +1,7 @@
 akka {
   loggers = ["akka.event.slf4j.Slf4jLogger"]
   logging-filter = "akka.event.slf4j.Slf4jLoggingFilter"
+  actor.guardian-supervisor-strategy = "cromwell.core.CromwellUserGuardianStrategy"
 }
 
 spray.can {

From 0b2c2900d52be66d234caabba093511724f8ef8c Mon Sep 17 00:00:00 2001
From: Khalid Shakir <kshakir@broadinstitute.org>
Date: Thu, 2 Feb 2017 19:29:27 -0500
Subject: [PATCH 44/84] Added tests that will deadlock and fail on empty mysql
 lobs.

---
 .../cromwell/services/ServicesStoreSpec.scala      | 121 ++++++++++++++++++++-
 1 file changed, 117 insertions(+), 4 deletions(-)

diff --git a/services/src/test/scala/cromwell/services/ServicesStoreSpec.scala b/services/src/test/scala/cromwell/services/ServicesStoreSpec.scala
index a23637974..0e0cfbfd7 100644
--- a/services/src/test/scala/cromwell/services/ServicesStoreSpec.scala
+++ b/services/src/test/scala/cromwell/services/ServicesStoreSpec.scala
@@ -3,9 +3,10 @@ package cromwell.services
 import java.io.{ByteArrayOutputStream, PrintStream}
 import java.sql.{Connection, JDBCType}
 import java.time.OffsetDateTime
+import javax.sql.rowset.serial.{SerialBlob, SerialClob, SerialException}
 
 import better.files._
-import com.typesafe.config.ConfigFactory
+import com.typesafe.config.{Config, ConfigFactory}
 import cromwell.core.Tags._
 import cromwell.core.WorkflowId
 import cromwell.database.migration.liquibase.LiquibaseUtils
@@ -26,7 +27,7 @@ import slick.driver.JdbcProfile
 import slick.jdbc.meta._
 
 import scala.concurrent.duration._
-import scala.concurrent.{Await, ExecutionContext, Future}
+import scala.concurrent.{Await, ExecutionContext, Future, TimeoutException}
 import scala.util.Try
 import scala.xml._
 
@@ -40,6 +41,53 @@ class ServicesStoreSpec extends FlatSpec with Matchers with ScalaFutures with St
 
   behavior of "ServicesStore"
 
+  it should "deadlock and timeout after 10 seconds" in {
+    /*
+    Tests that we still need our deadlock workaround in `database.run` to deal with problems in Slick 3.1.
+    If/when this test fails, the workaround should be removed.
+    */
+
+    class DeadlockingSlickDatabase(config: Config) extends SlickDatabase(config) {
+
+      import dataAccess.driver.api._
+
+      override protected[this] def runTransaction[R](action: DBIO[R]): Future[R] = {
+        database.run(action.transactionally) //<-- https://github.com/slick/slick/issues/1274
+      }
+    }
+
+    // Test based on https://github.com/kwark/slick-deadlock/blob/82525fc/src/main/scala/SlickDeadlock.scala
+    val databaseConfig = ConfigFactory.parseString(
+      s"""|db.url = "jdbc:hsqldb:mem:$${uniqueSchema};shutdown=false;hsqldb.tx=mvcc"
+          |db.driver = "org.hsqldb.jdbcDriver"
+          |db.connectionTimeout = 3000
+          |db.numThreads = 2
+          |driver = "slick.driver.HsqldbDriver$$"
+          |""".stripMargin)
+    import ServicesStore.EnhancedSqlDatabase
+    for {
+      database <- new DeadlockingSlickDatabase(databaseConfig).initialized.autoClosed
+    } {
+      val futures = 1 to 20 map { _ =>
+        val workflowUuid = WorkflowId.randomId().toString
+        val callFqn = "call.fqn"
+        val jobIndex = 1
+        val jobAttempt = 1
+        val jobSuccessful = false
+        val jobStoreEntry = JobStoreEntry(workflowUuid, callFqn, jobIndex, jobAttempt, jobSuccessful, None, None, None)
+        val jobStoreJoins = Seq(JobStoreJoin(jobStoreEntry, Seq()))
+        // NOTE: This test just needs to repeatedly read/write from a table that acts as a PK for a FK.
+        for {
+          _ <- database.addJobStores(jobStoreJoins)
+          queried <- database.queryJobStores(workflowUuid, callFqn, jobIndex, jobAttempt)
+          _ = queried.get.jobStoreEntry.workflowExecutionUuid should be(workflowUuid)
+        } yield ()
+      }
+
+      intercept[TimeoutException](Await.result(Future.sequence(futures), 10.seconds))
+    }
+  }
+
   it should "not deadlock" in {
     // Test based on https://github.com/kwark/slick-deadlock/blob/82525fc/src/main/scala/SlickDeadlock.scala
     val databaseConfig = ConfigFactory.parseString(
@@ -249,10 +297,13 @@ class ServicesStoreSpec extends FlatSpec with Matchers with ScalaFutures with St
     lazy val databaseConfig = ConfigFactory.load.getConfig(configPath)
     lazy val dataAccess = new SlickDatabase(databaseConfig).initialized
 
-    it should "(if hsqldb) have transaction isolation mvcc" taggedAs DbmsTest in {
+    lazy val getProduct = {
       import dataAccess.dataAccess.driver.api._
+      SimpleDBIO[String](_.connection.getMetaData.getDatabaseProductName)
+    }
 
-      val getProduct = SimpleDBIO[String](_.connection.getMetaData.getDatabaseProductName)
+    it should "(if hsqldb) have transaction isolation mvcc" taggedAs DbmsTest in {
+      import dataAccess.dataAccess.driver.api._
       //noinspection SqlDialectInspection
       val getHsqldbTx = sql"""SELECT PROPERTY_VALUE
                               FROM INFORMATION_SCHEMA.SYSTEM_PROPERTIES
@@ -270,6 +321,68 @@ class ServicesStoreSpec extends FlatSpec with Matchers with ScalaFutures with St
       } yield ()).futureValue
     }
 
+    it should "fail to store and retrieve empty clobs in MySQL" taggedAs DbmsTest in {
+      /*
+      Tests that we still need our empty clob workaround in `SqlConverters.toClob`. The current mysql jdbc driver throws
+      an exception when serializing an empty clob. If/when this test fails, the workaround should be removed.
+      */
+      val emptyClob = new SerialClob(Array.empty[Char])
+
+      val workflowUuid = WorkflowId.randomId().toString
+      val callFqn = "call.fqn"
+      val jobIndex = 1
+      val jobAttempt = 1
+      val jobSuccessful = false
+      val jobStoreEntry = JobStoreEntry(workflowUuid, callFqn, jobIndex, jobAttempt, jobSuccessful, None, None, None)
+      val jobStoreSimpletonEntries = Seq(JobStoreSimpletonEntry("empty", Option(emptyClob), "WdlString"))
+      val jobStoreJoins = Seq(JobStoreJoin(jobStoreEntry, jobStoreSimpletonEntries))
+
+      val future = for {
+        product <- dataAccess.database.run(getProduct)
+        _ = product match {
+          case "MySQL" =>
+            val exception = intercept[SerialException] {
+              Await.result(dataAccess.addJobStores(jobStoreJoins), Duration.Inf)
+            }
+            exception should be(a[SerialException])
+            exception.getMessage should be("Invalid position in SerialClob object set")
+          case _ => ()
+        }
+      } yield ()
+
+      future.futureValue
+    }
+
+    it should "fail to store and retrieve empty blobs in MySQL" taggedAs DbmsTest in {
+      /*
+      Tests that we still need our empty blob workaround in `SqlConverters.toBlob`. The current mysql jdbc driver throws
+      an exception when serializing an empty blob. If/when this test fails, the workaround should be removed.
+      */
+      val emptyBlob = new SerialBlob(Array.empty[Byte])
+
+      val workflowUuid = WorkflowId.randomId().toString
+      val workflowStoreEntry = WorkflowStoreEntry(workflowUuid, "{}".toClob, "{}".toClob, "{}".toClob,
+        "Testing", OffsetDateTime.now.toSystemTimestamp, Option(emptyBlob), "{}".toClob)
+
+      val workflowStoreEntries = Seq(workflowStoreEntry)
+
+      val future = for {
+        product <- dataAccess.database.run(getProduct)
+        _ = product match {
+          case "MySQL" =>
+            val exception = intercept[SerialException] {
+              Await.result(dataAccess.addWorkflowStoreEntries(workflowStoreEntries), Duration.Inf)
+            }
+            exception should be(a[SerialException])
+            exception.getMessage should
+              be("Invalid arguments: position cannot be less than 1 or greater than the length of the SerialBlob")
+          case _ => ()
+        }
+      } yield ()
+
+      future.futureValue
+    }
+
     it should "store and retrieve empty clobs" taggedAs DbmsTest in {
       val workflowUuid = WorkflowId.randomId().toString
       val callFqn = "call.fqn"

From 119bb11c574a0b917e10a82561415c9b54c72948 Mon Sep 17 00:00:00 2001
From: Khalid Shakir <kshakir@broadinstitute.org>
Date: Mon, 6 Feb 2017 11:20:44 -0500
Subject: [PATCH 45/84] Moved globbing behavior to standard. Standard Backend
 has even more default implementations, and a few breaking changes. Moved some
 code for testing JES out of main. Increased the timeout for the ask inside
 SingleWorkflowRunnerActorSpec. Bumped apache http client to the latest
 version.

---
 .../main/scala/cromwell/backend/io/JobPaths.scala  |  16 +-
 .../standard/StandardAsyncExecutionActor.scala     | 247 +++++++++++++++++++--
 .../standard/StandardCacheHitCopyingActor.scala    |   7 +-
 .../standard/StandardExpressionFunctions.scala     |  41 ++++
 .../standard/StandardFinalizationActor.scala       |  64 +++++-
 .../standard/StandardInitializationActor.scala     |  44 +++-
 .../standard/StandardInitializationData.scala      |  22 +-
 .../standard/StandardJobExecutionActorParams.scala |  48 ----
 .../standard/StandardLifecycleActorFactory.scala   |  50 +++--
 .../standard/StandardSyncExecutionActor.scala      |  19 +-
 .../scala/cromwell/backend/wdl/WdlFileMapper.scala |  28 +++
 core/src/main/scala/cromwell/core/core.scala       |   4 +-
 .../src/test/scala/cromwell/MapWorkflowSpec.scala  |   9 +-
 .../workflow/SingleWorkflowRunnerActorSpec.scala   |   6 +-
 project/Dependencies.scala                         |   2 +
 .../jes/JesAsyncBackendJobExecutionActor.scala     | 168 +++-----------
 .../impl/jes/JesBackendInitializationData.scala    |   2 +-
 .../impl/jes/JesBackendLifecycleActorFactory.scala |  23 +-
 .../backend/impl/jes/JesCacheHitCopyingActor.scala |  11 -
 .../backend/impl/jes/JesExpressionFunctions.scala  |  35 +--
 .../backend/impl/jes/JesFinalizationActor.scala    |  57 +----
 .../backend/impl/jes/JesInitializationActor.scala  |  63 ++----
 .../cromwell/backend/impl/jes/JesJobPaths.scala    |   7 +-
 .../backend/impl/jes/JesRuntimeAttributes.scala    |   9 -
 .../jes/JesAsyncBackendJobExecutionActorSpec.scala |  71 +++---
 .../backend/impl/jes/JesCallPathsSpec.scala        |   6 +-
 .../impl/jes/JesInitializationActorSpec.scala      |  14 +-
 .../impl/jes/JesRuntimeAttributesSpec.scala        |  21 +-
 .../ConfigBackendLifecycleActorFactory.scala       |   4 +-
 .../sfs/config/ConfigInitializationActor.scala     |   7 +-
 .../sfs/BackgroundAsyncJobExecutionActor.scala     |  54 ++---
 .../cromwell/backend/sfs/SharedFileSystem.scala    | 106 +++------
 .../SharedFileSystemAsyncJobExecutionActor.scala   | 111 +++------
 ...redFileSystemBackendLifecycleActorFactory.scala |  11 +-
 .../sfs/SharedFileSystemCacheHitCopyingActor.scala |   4 +-
 .../sfs/SharedFileSystemExpressionFunctions.scala  |  64 +-----
 .../sfs/SharedFileSystemInitializationActor.scala  |  23 +-
 .../SharedFileSystemJobCachingActorHelper.scala    |   7 +-
 .../sfs/TestLocalAsyncJobExecutionActor.scala      |   3 +-
 39 files changed, 754 insertions(+), 734 deletions(-)
 create mode 100644 backend/src/main/scala/cromwell/backend/standard/StandardExpressionFunctions.scala
 create mode 100644 backend/src/main/scala/cromwell/backend/wdl/WdlFileMapper.scala
 delete mode 100644 supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesCacheHitCopyingActor.scala

diff --git a/backend/src/main/scala/cromwell/backend/io/JobPaths.scala b/backend/src/main/scala/cromwell/backend/io/JobPaths.scala
index 5c3a0c9f7..24339d86c 100644
--- a/backend/src/main/scala/cromwell/backend/io/JobPaths.scala
+++ b/backend/src/main/scala/cromwell/backend/io/JobPaths.scala
@@ -2,7 +2,7 @@ package cromwell.backend.io
 
 import java.nio.file.Path
 
-import cromwell.core.JobKey
+import cromwell.core.{CallContext, JobKey}
 import cromwell.services.metadata.CallMetadataKeys
 
 object JobPaths {
@@ -27,6 +27,7 @@ object JobPaths {
 
 trait JobPaths { this: WorkflowPaths =>
   import JobPaths._
+  import cromwell.core.path.PathImplicits._
 
   def returnCodeFilename: String = "rc"
   def stdoutFilename: String = "stdout"
@@ -54,10 +55,19 @@ trait JobPaths { this: WorkflowPaths =>
     JobPaths.StdErrPathKey -> stderr,
     JobPaths.ReturnCodePathKey -> returnCode
   )
-  
+
+  private lazy val commonLogPaths: Map[String, Path] = Map(
+    JobPaths.StdoutPathKey -> stdout,
+    JobPaths.StdErrPathKey -> stderr
+  )
+
   protected lazy val customMetadataPaths: Map[String, Path] = Map.empty
   protected lazy val customDetritusPaths: Map[String, Path] = Map.empty
-  
+  protected lazy val customLogPaths: Map[String, Path] = Map.empty
+
   lazy val metadataPaths = commonMetadataPaths ++ customMetadataPaths
   lazy val detritusPaths = commonDetritusPaths ++ customDetritusPaths
+  lazy val logPaths = commonLogPaths ++ customLogPaths
+
+  lazy val callContext = CallContext(callExecutionRoot, stdout.toRealString, stderr.toRealString)
 }
diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala b/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala
index 5d30b63aa..0adb0dc38 100644
--- a/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala
+++ b/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala
@@ -10,17 +10,35 @@ import cromwell.backend.BackendLifecycleActor.AbortJobCommand
 import cromwell.backend.async.AsyncBackendJobExecutionActor.{ExecutionMode, JobId, Recover}
 import cromwell.backend.async.{AbortedExecutionHandle, AsyncBackendJobExecutionActor, ExecutionHandle, FailedNonRetryableExecutionHandle, PendingExecutionHandle, ReturnCodeIsNotAnInt, StderrNonEmpty, SuccessfulExecutionHandle, WrongReturnCode}
 import cromwell.backend.validation._
-import cromwell.backend.wdl.Command
+import cromwell.backend.wdl.{Command, OutputEvaluator, WdlFileMapper}
 import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendJobDescriptor, BackendJobLifecycleActor}
+import cromwell.core.path.PathFactory.pathPlusSuffix
+import cromwell.core.{CallOutputs, CromwellAggregatedException, ExecutionEvent}
 import cromwell.services.keyvalue.KeyValueServiceActor._
 import cromwell.services.metadata.CallMetadataKeys
+import lenthall.util.TryUtil
 import wdl4s._
-import wdl4s.expression.{PureStandardLibraryFunctions, WdlFunctions}
-import wdl4s.values.WdlValue
+import wdl4s.values.{WdlFile, WdlGlobFile, WdlSingleFile, WdlValue}
 
 import scala.concurrent.{ExecutionContext, ExecutionContextExecutor, Future, Promise}
 import scala.util.{Failure, Success, Try}
 
+trait StandardAsyncExecutionActorParams extends StandardJobExecutionActorParams {
+  /** The promise that will be completed when the async run is complete. */
+  def completionPromise: Promise[BackendJobExecutionResponse]
+}
+
+case class DefaultStandardAsyncExecutionActorParams
+(
+  override val jobIdKey: String,
+  override val serviceRegistryActor: ActorRef,
+  override val jobDescriptor: BackendJobDescriptor,
+  override val configurationDescriptor: BackendConfigurationDescriptor,
+  override val backendInitializationDataOption: Option[BackendInitializationData],
+  override val backendSingletonActorOption: Option[ActorRef],
+  override val completionPromise: Promise[BackendJobExecutionResponse]
+) extends StandardAsyncExecutionActorParams
+
 /**
   * An extension of the generic AsyncBackendJobExecutionActor providing a standard abstract implementation of an
   * asynchronous polling backend.
@@ -65,20 +83,128 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
   override lazy val jobDescriptor: BackendJobDescriptor = standardParams.jobDescriptor
 
   /** @see [[StandardAsyncExecutionActorParams.jobIdKey]] */
-  def jobIdKey: String = standardParams.jobIdKey
+  lazy val jobIdKey: String = standardParams.jobIdKey
 
   /** @see [[Command.instantiate]] */
-  def commandLineFunctions: WdlFunctions[WdlValue] = PureStandardLibraryFunctions
+  lazy val backendEngineFunctions: StandardExpressionFunctions =
+    standardInitializationData.expressionFunctions(jobPaths)
+
+  /**
+    * Maps WdlFile objects for use in the commandLinePreProcessor.
+    *
+    * By default just calls the pass through mapper mapCommandLineWdlFile.
+    *
+    * Sometimes a preprocessor may need to localize the files, etc.
+    *
+    * @param wdlFile The wdlFile.
+    * @return The updated wdlFile.
+    */
+  def preProcessWdlFile(wdlFile: WdlFile): WdlFile = wdlFile
 
   /** @see [[Command.instantiate]] */
-  def commandLinePreProcessor: EvaluatedTaskInputs => Try[EvaluatedTaskInputs] = Success.apply
+  final lazy val commandLinePreProcessor: EvaluatedTaskInputs => Try[EvaluatedTaskInputs] = {
+    inputs => TryUtil.sequenceMap(inputs mapValues WdlFileMapper.mapWdlFiles(preProcessWdlFile))
+  }
+
+  /**
+    * Maps WdlFile to a local path, often maping Path.toRealString to Path.toString, for use in the
+    * commandLineValueMapper.
+    *
+    * @param wdlFile The wdlFile.
+    * @return The updated wdlFile.
+    */
+  def mapCommandLineWdlFile(wdlFile: WdlFile): WdlFile = WdlSingleFile(workflowPaths.buildPath(wdlFile.value).toString)
 
   /** @see [[Command.instantiate]] */
-  def commandLineValueMapper: WdlValue => WdlValue = identity
+  final lazy val commandLineValueMapper: WdlValue => WdlValue = {
+    wdlValue => WdlFileMapper.mapWdlFiles(mapCommandLineWdlFile)(wdlValue).get
+  }
+
+  /**
+    * The local path where the command will run.
+    */
+  lazy val commandDirectory: Path = jobPaths.callExecutionRoot
+
+  /**
+    * The local parent directory of the glob file. By default this is the same as the commandDirectory.
+    *
+    * In some cases, to allow the hard linking by ln to operate, a different mount point must be returned.
+    *
+    * @param wdlGlobFile The glob.
+    * @return The parent directory for writing the wdl glob.
+    */
+  def globParentDirectory(wdlGlobFile: WdlGlobFile): Path = commandDirectory
+
+  /**
+    * Returns the shell scripting for hard linking the glob results using ln.
+    *
+    * @param globFiles The globs.
+    * @return The shell scripting.
+    */
+  def globManipulations(globFiles: Traversable[WdlGlobFile]): String = globFiles map globManipulation mkString "\n"
+
+  /**
+    * Returns the shell scripting for hard linking a glob results using ln.
+    *
+    * @param globFile The glob.
+    * @return The shell scripting.
+    */
+  def globManipulation(globFile: WdlGlobFile): String = {
+    val parentDirectory = globParentDirectory(globFile)
+    val globDir = backendEngineFunctions.globName(globFile.value)
+    val globDirectory = File(parentDirectory)./(globDir)
+    val globList = File(parentDirectory)./(s"$globDir.list")
+
+    s"""|mkdir $globDirectory
+        |( ln -L ${globFile.value} $globDirectory 2> /dev/null ) || ( ln ${globFile.value} $globDirectory )
+        |ls -1 $globDirectory > $globList
+        |""".stripMargin
+  }
+
+  /** Any custom code that should be run within commandScriptContents before the instantiated command. */
+  def commandScriptPreamble: String = ""
+
+  /** A bash script containing the custom preamble, the instantiated command, and output globbing behavior. */
+  def commandScriptContents: String = {
+    jobLogger.info(s"`$instantiatedCommand`")
+
+    val cwd = commandDirectory
+    val tmpDir = File(cwd)./("tmp").path
+    val rcPath = File(cwd)./(jobPaths.returnCodeFilename).path
+    val rcTmpPath = pathPlusSuffix(rcPath, "tmp").path
+
+    val globFiles = backendEngineFunctions.findGlobOutputs(call, jobDescriptor)
+
+    s"""|#!/bin/bash
+        |export _JAVA_OPTIONS=-Djava.io.tmpdir=$tmpDir
+        |export TMPDIR=$tmpDir
+        |$commandScriptPreamble
+        |(
+        |cd $cwd
+        |INSTANTIATED_COMMAND
+        |)
+        |echo $$? > $rcTmpPath
+        |(
+        |cd $cwd
+        |${globManipulations(globFiles)}
+        |)
+        |mv $rcTmpPath $rcPath
+        |""".stripMargin.replace("INSTANTIATED_COMMAND", instantiatedCommand)
+  }
 
   /** The instantiated command. */
   lazy val instantiatedCommand: String = Command.instantiate(
-    jobDescriptor, commandLineFunctions, commandLinePreProcessor, commandLineValueMapper).get
+    jobDescriptor, backendEngineFunctions, commandLinePreProcessor, commandLineValueMapper).get
+
+  /**
+    * Redirect the stdout and stderr to the appropriate files. While not necessary, mark the job as not receiving any
+    * stdin by pointing it at /dev/null.
+    *
+    * If the `command` errors for some reason, put a "-1" into the rc file.
+    */
+  def redirectOutputs(command: String): String = {
+    s"$command > ${jobPaths.stdout} 2> ${jobPaths.stderr} < /dev/null || echo -1 > ${jobPaths.returnCode}"
+  }
 
   /** A tag that may be used for logging. */
   lazy val tag = s"${this.getClass.getSimpleName} [UUID(${workflowId.shortString}):${jobDescriptor.key.tag}]"
@@ -180,6 +306,14 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
   def isTerminal(runStatus: StandardAsyncRunStatus): Boolean
 
   /**
+    * Returns any events retrieved from the terminal run status.
+    *
+    * @param runStatus The terminal run status, as defined by isTerminal.
+    * @return The execution events.
+    */
+  def getTerminalEvents(runStatus: StandardAsyncRunStatus): Seq[ExecutionEvent] = Seq.empty
+
+  /**
     * Returns true if the status represents a success.
     *
     * @param runStatus The run status.
@@ -237,6 +371,73 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
   }
 
   /**
+    * Output value mapper.
+    *
+    * @param wdlValue The original wdl value.
+    * @return The Try wrapped and mapped wdl value.
+    */
+  final def outputValueMapper(wdlValue: WdlValue): Try[WdlValue] = {
+    WdlFileMapper.mapWdlFiles(mapOutputWdlFile)(wdlValue)
+  }
+
+  /**
+    * Used to convert to output paths.
+    *
+    * @param wdlFile The original file.
+    * @return The mapped output file.
+    */
+  def mapOutputWdlFile(wdlFile: WdlFile): WdlFile = wdlFile
+
+  /**
+    * Tries to evaluate the outputs.
+    *
+    * Used during handleExecutionSuccess.
+    *
+    * @return A Try wrapping evaluated outputs.
+    */
+  def evaluateOutputs: Try[CallOutputs] = {
+    OutputEvaluator.evaluateOutputs(jobDescriptor, backendEngineFunctions, outputValueMapper)
+  }
+
+  /**
+    * Tests whether an attempted result of evaluateOutputs should possibly be retried.
+    *
+    * If the exception is a CromwellAggregatedException, this method will recursively call into itself checking if the
+    * inner exceptions should be retried by using retryEvaluateOutputs.
+    *
+    * Custom implementations of handleExecutionSuccess should use this method when testing the results of
+    * evaluateOutputs.
+    *
+    * However, to implement the actual check, override the function retryEvaluateOutputs.
+    *
+    * @param exception The exception, possibly an CromwellAggregatedException.
+    * @return True if evaluateOutputs should be retried later.
+    */
+  final def retryEvaluateOutputsAggregated(exception: Exception): Boolean = {
+    exception match {
+      case aggregated: CromwellAggregatedException =>
+        aggregated.throwables.collectFirst {
+          case exception: Exception if retryEvaluateOutputsAggregated(exception) => exception
+        }.isDefined
+      case _ => retryEvaluateOutputs(exception)
+    }
+  }
+
+  /**
+    * Tests whether an attempted result of evaluateOutputs should possibly be retried.
+    *
+    * Override this function to check for different types of exceptions.
+    *
+    * Custom implementations of handleExecutionSuccess should NOT use method when testing the results of
+    * evaluateOutputs, as this method does not recurse into aggregated exceptions. Instead custom overrides of
+    * handleExecutionSuccess should call into retryEvaluateOutputsAggregated.
+    *
+    * @param exception The exception, possibly an internal instance retrieved from within a CromwellAggregatedException.
+    * @return True if evaluateOutputs should be retried later.
+    */
+  def retryEvaluateOutputs(exception: Exception): Boolean = false
+
+  /**
     * Process a successful run, as defined by `isSuccess`.
     *
     * @param runStatus  The run status.
@@ -244,8 +445,18 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
     * @param returnCode The return code.
     * @return The execution handle.
     */
-  def handleExecutionSuccess(runStatus: StandardAsyncRunStatus, handle: StandardAsyncPendingExecutionHandle,
-                             returnCode: Int): ExecutionHandle
+  def handleExecutionSuccess(runStatus: StandardAsyncRunStatus,
+                             handle: StandardAsyncPendingExecutionHandle,
+                             returnCode: Int): ExecutionHandle = {
+    evaluateOutputs match {
+      case Success(outputs) =>
+        SuccessfulExecutionHandle(outputs, returnCode, jobPaths.detritusPaths, getTerminalEvents(runStatus))
+      case Failure(exception: Exception) if retryEvaluateOutputsAggregated(exception) =>
+        // Return the execution handle in this case to retry the operation
+        handle
+      case Failure(ex) => FailedNonRetryableExecutionHandle(ex)
+    }
+  }
 
   /**
     * Process an unsuccessful run, as defined by `isSuccess`.
@@ -384,9 +595,13 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
       case exception: Exception =>
         val handler: PartialFunction[(ExecutionHandle, Exception), ExecutionHandle] =
           customPollStatusFailure orElse {
+            case (_: ExecutionHandle, exception: Exception) if isFatal(exception) =>
+              // Log exceptions and return the original handle to try again.
+              jobLogger.warn(s"Caught fatal exception attempting to poll", exception)
+              FailedNonRetryableExecutionHandle(exception)
             case (handle: ExecutionHandle, exception: Exception) =>
               // Log exceptions and return the original handle to try again.
-              jobLogger.warn(s"Caught exception, retrying", exception)
+              jobLogger.warn(s"Caught exception trying to poll, retrying", exception)
               handle
           }
         handler((oldHandle, exception))
@@ -411,7 +626,7 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
       lazy val returnCodeAsString: Try[String] = Try(File(jobPaths.returnCode).contentAsString)
       lazy val returnCodeAsInt: Try[Int] = returnCodeAsString.map(_.trim.toInt)
       lazy val stderrAsOption: Option[Path] = Option(jobPaths.stderr)
-      
+
       if (isSuccess(status)) {
         lazy val stderrLength: Try[Long] = Try(File(jobPaths.stderr).size)
         (stderrLength, returnCodeAsString, returnCodeAsInt) match {
@@ -443,8 +658,12 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
         handleExecutionFailure(status, oldHandle, returnCodeAsInt.toOption)
       }
     } catch {
-      case e: Exception =>
-        jobLogger.warn("Caught exception processing job result, retrying", e)
+      case exception: Exception if isFatal(exception) =>
+        jobLogger.warn("Caught fatal exception processing job result", exception)
+        // Return the original handle to try again.
+        FailedNonRetryableExecutionHandle(exception)
+      case exception: Exception =>
+        jobLogger.warn("Caught exception processing job result, retrying", exception)
         // Return the original handle to try again.
         oldHandle
     }
diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardCacheHitCopyingActor.scala b/backend/src/main/scala/cromwell/backend/standard/StandardCacheHitCopyingActor.scala
index 7f5531e94..41d1b9118 100644
--- a/backend/src/main/scala/cromwell/backend/standard/StandardCacheHitCopyingActor.scala
+++ b/backend/src/main/scala/cromwell/backend/standard/StandardCacheHitCopyingActor.scala
@@ -5,6 +5,7 @@ import java.nio.file.Path
 import akka.actor.ActorRef
 import cromwell.backend.callcaching.CacheHitDuplicating
 import cromwell.backend.{BackendCacheHitCopyingActor, BackendConfigurationDescriptor, BackendInitializationData, BackendJobDescriptor}
+import cromwell.core.path.PathCopier
 
 /**
   * Trait of parameters passed to a StandardCacheHitCopyingActor.
@@ -31,10 +32,10 @@ case class DefaultStandardCacheHitCopyingActorParams
 /**
   * Standard implementation of a BackendCacheHitCopyingActor.
   */
-trait StandardCacheHitCopyingActor extends BackendCacheHitCopyingActor with CacheHitDuplicating
-  with StandardCachingActorHelper {
+class StandardCacheHitCopyingActor(val standardParams: StandardCacheHitCopyingActorParams)
+  extends BackendCacheHitCopyingActor with CacheHitDuplicating with StandardCachingActorHelper {
 
-  def standardParams: StandardCacheHitCopyingActorParams
+  override protected def duplicate(source: Path, destination: Path): Unit = PathCopier.copy(source, destination).get
 
   override lazy val jobDescriptor: BackendJobDescriptor = standardParams.jobDescriptor
   override lazy val backendInitializationDataOption: Option[BackendInitializationData] =
diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardExpressionFunctions.scala b/backend/src/main/scala/cromwell/backend/standard/StandardExpressionFunctions.scala
new file mode 100644
index 000000000..3f0ea002f
--- /dev/null
+++ b/backend/src/main/scala/cromwell/backend/standard/StandardExpressionFunctions.scala
@@ -0,0 +1,41 @@
+package cromwell.backend.standard
+
+import java.nio.file.Path
+
+import cromwell.backend.io.GlobFunctions
+import cromwell.backend.wdl.{ReadLikeFunctions, WriteFunctions}
+import cromwell.core.CallContext
+import cromwell.core.path.PathBuilder
+import wdl4s.expression.PureStandardLibraryFunctionsLike
+import wdl4s.values.{WdlFile, WdlValue}
+
+import scala.util.{Success, Try}
+
+trait StandardExpressionFunctionsParams {
+  def pathBuilders: List[PathBuilder]
+
+  def callContext: CallContext
+}
+
+case class DefaultStandardExpressionFunctionsParams(override val pathBuilders: List[PathBuilder],
+                                                    override val callContext: CallContext
+                                                   ) extends StandardExpressionFunctionsParams
+
+// TODO: Once we figure out premapping and postmapping, maybe we can standardize that behavior. Currently that's the most important feature that subclasses override.
+class StandardExpressionFunctions(val standardParams: StandardExpressionFunctionsParams)
+  extends PureStandardLibraryFunctionsLike with ReadLikeFunctions with WriteFunctions with GlobFunctions {
+
+  override lazy val pathBuilders: List[PathBuilder] = standardParams.pathBuilders
+
+  override lazy val callContext: CallContext = standardParams.callContext
+
+  override lazy val writeDirectory: Path = callContext.root
+
+  override def stdout(params: Seq[Try[WdlValue]]): Try[WdlFile] = Success(WdlFile(callContext.stdout))
+
+  override def stderr(params: Seq[Try[WdlValue]]): Try[WdlFile] = Success(WdlFile(callContext.stderr))
+
+  override def writeTempFile(path: String, prefix: String, suffix: String, content: String): String = {
+    super[WriteFunctions].writeTempFile(path, prefix, suffix, content)
+  }
+}
diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardFinalizationActor.scala b/backend/src/main/scala/cromwell/backend/standard/StandardFinalizationActor.scala
index 19500a90d..588697a16 100644
--- a/backend/src/main/scala/cromwell/backend/standard/StandardFinalizationActor.scala
+++ b/backend/src/main/scala/cromwell/backend/standard/StandardFinalizationActor.scala
@@ -1,9 +1,17 @@
 package cromwell.backend.standard
 
+import java.nio.file.Path
+
+import akka.dispatch.MessageDispatcher
 import cromwell.backend._
+import cromwell.backend.io.WorkflowPaths
 import cromwell.core.CallOutputs
+import cromwell.core.Dispatcher.IoDispatcher
+import cromwell.core.path.PathCopier
 import wdl4s.TaskCall
 
+import scala.concurrent.Future
+
 trait StandardFinalizationActorParams {
   def workflowDescriptor: BackendWorkflowDescriptor
 
@@ -28,9 +36,15 @@ case class DefaultStandardFinalizationActorParams
   configurationDescriptor: BackendConfigurationDescriptor
 ) extends StandardFinalizationActorParams
 
-trait StandardFinalizationActor extends BackendWorkflowFinalizationActor {
-
-  def standardParams: StandardFinalizationActorParams
+/**
+  * Implements BackendWorkflowFinalizationActor.afterAll() by calling copyCallLogs.
+  *
+  * Subclasses should either call super.afterAll(), or invoke copyCallLogs().
+  *
+  * @param standardParams Standard parameters.
+  */
+class StandardFinalizationActor(val standardParams: StandardFinalizationActorParams)
+  extends BackendWorkflowFinalizationActor {
 
   override lazy val workflowDescriptor: BackendWorkflowDescriptor = standardParams.workflowDescriptor
   override lazy val calls: Set[TaskCall] = standardParams.calls
@@ -38,4 +52,48 @@ trait StandardFinalizationActor extends BackendWorkflowFinalizationActor {
   lazy val jobExecutionMap: JobExecutionMap = standardParams.jobExecutionMap
   lazy val workflowOutputs: CallOutputs = standardParams.workflowOutputs
   override lazy val configurationDescriptor: BackendConfigurationDescriptor = standardParams.configurationDescriptor
+
+  protected val workflowPaths: Option[WorkflowPaths] = initializationDataOption.map {
+    _.asInstanceOf[StandardInitializationData].workflowPaths
+  }
+
+  protected val ioExecutionContext: MessageDispatcher = context.system.dispatchers.lookup(IoDispatcher)
+
+  override def afterAll(): Future[Unit] = copyCallLogs()
+
+  lazy val logPaths: Seq[Path] = {
+    for {
+      actualWorkflowPath <- workflowPaths.toSeq
+      (backendWorkflowDescriptor, keys) <- jobExecutionMap.toSeq
+      key <- keys
+      jobPaths = actualWorkflowPath.toJobPaths(key, backendWorkflowDescriptor)
+      logPath <- jobPaths.logPaths.values
+    } yield logPath
+  }
+
+  protected def copyCallLogs(): Future[Unit] = {
+    /*
+    NOTE: Only using one thread pool slot here to upload all the files for all the calls.
+    Using the io-dispatcher defined in application.conf because this might take a while.
+    One could also use Future.sequence to flood the dispatcher, or even create a separate jes final call specific thread
+    pool for parallel uploads.
+
+    Measure and optimize as necessary. Will likely need retry code at some level as well.
+    */
+    workflowPaths match {
+      case Some(paths) => Future(paths.finalCallLogsPath foreach copyCallLogs)(ioExecutionContext)
+      case _ => Future.successful(())
+    }
+  }
+
+  private def copyCallLogs(callLogsPath: Path): Unit = {
+    copyLogs(callLogsPath, logPaths)
+  }
+
+  private def copyLogs(callLogsDirPath: Path, logPaths: Seq[Path]): Unit = {
+    workflowPaths match {
+      case Some(paths) => logPaths.foreach(PathCopier.copy(paths.executionRoot, _, callLogsDirPath))
+      case None =>
+    }
+  }
 }
diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardInitializationActor.scala b/backend/src/main/scala/cromwell/backend/standard/StandardInitializationActor.scala
index 1fbfc92d1..d958cdefc 100644
--- a/backend/src/main/scala/cromwell/backend/standard/StandardInitializationActor.scala
+++ b/backend/src/main/scala/cromwell/backend/standard/StandardInitializationActor.scala
@@ -1,9 +1,12 @@
 package cromwell.backend.standard
 
 import akka.actor.ActorRef
+import cromwell.backend.io.WorkflowPaths
 import cromwell.backend.validation.RuntimeAttributesDefault
-import cromwell.backend.{BackendConfigurationDescriptor, BackendWorkflowDescriptor, BackendWorkflowInitializationActor}
+import cromwell.backend.wfs.WorkflowPathBuilder
+import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendWorkflowDescriptor, BackendWorkflowInitializationActor}
 import cromwell.core.WorkflowOptions
+import cromwell.core.path.{DefaultPathBuilder, PathBuilder}
 import wdl4s.TaskCall
 import wdl4s.values.WdlValue
 
@@ -28,18 +31,45 @@ case class DefaultInitializationActorParams
   configurationDescriptor: BackendConfigurationDescriptor
 ) extends StandardInitializationActorParams
 
-trait StandardInitializationActor extends BackendWorkflowInitializationActor {
-
-  def standardParams: StandardInitializationActorParams
+/**
+  * Implements BackendWorkflowInitializationActor.beforeAll() by returning a basic initialization data.
+  *
+  * Most implementors will want to do custom initialization in a beforeAll() and/or return custom initialization data.
+  *
+  * @param standardParams Standard parameters
+  */
+class StandardInitializationActor(val standardParams: StandardInitializationActorParams)
+  extends BackendWorkflowInitializationActor {
 
   override lazy val serviceRegistryActor: ActorRef = standardParams.serviceRegistryActor
 
   override lazy val calls: Set[TaskCall] = standardParams.calls
 
+  override def beforeAll(): Future[Option[BackendInitializationData]] = {
+    Future.fromTry(Try(Option(initializationData)))
+  }
+
+  lazy val initializationData: StandardInitializationData =
+    new StandardInitializationData(workflowPaths, runtimeAttributesBuilder, classOf[StandardExpressionFunctions])
+
+  lazy val expressionFunctions: Class[_ <: StandardExpressionFunctions] = classOf[StandardExpressionFunctions]
+
+  lazy val pathBuilders: List[PathBuilder] = List(DefaultPathBuilder)
+
+  lazy val workflowPaths: WorkflowPaths =
+    WorkflowPathBuilder.workflowPaths(configurationDescriptor, workflowDescriptor, pathBuilders)
+
+  /**
+    * Returns the runtime attribute builder for this backend.
+    *
+    * NOTE: Implemented as a def instead of a lazy val to allow calls to super.runtimeAttributesBuilder.withValidation()
+    *
+    * @return runtime attributes builder with possible custom validations
+    */
   def runtimeAttributesBuilder: StandardValidatedRuntimeAttributesBuilder =
     StandardValidatedRuntimeAttributesBuilder.default
 
-  override protected def runtimeAttributeValidators: Map[String, (Option[WdlValue]) => Boolean] = {
+  override protected lazy val runtimeAttributeValidators: Map[String, (Option[WdlValue]) => Boolean] = {
     runtimeAttributesBuilder.validatorMap
   }
 
@@ -63,8 +93,8 @@ trait StandardInitializationActor extends BackendWorkflowInitializationActor {
     })
   }
 
-  override protected def workflowDescriptor: BackendWorkflowDescriptor = standardParams.workflowDescriptor
+  override protected lazy val workflowDescriptor: BackendWorkflowDescriptor = standardParams.workflowDescriptor
 
-  override protected def configurationDescriptor: BackendConfigurationDescriptor =
+  override protected lazy val configurationDescriptor: BackendConfigurationDescriptor =
     standardParams.configurationDescriptor
 }
diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardInitializationData.scala b/backend/src/main/scala/cromwell/backend/standard/StandardInitializationData.scala
index e9cdb2f35..e4dae0e0b 100644
--- a/backend/src/main/scala/cromwell/backend/standard/StandardInitializationData.scala
+++ b/backend/src/main/scala/cromwell/backend/standard/StandardInitializationData.scala
@@ -1,21 +1,23 @@
 package cromwell.backend.standard
 
 import cromwell.backend.BackendInitializationData
-import cromwell.backend.io.WorkflowPaths
-import cromwell.core.path.PathBuilder
+import cromwell.backend.io.{JobPaths, WorkflowPaths}
 
 class StandardInitializationData
 (
   val workflowPaths: WorkflowPaths,
-  val runtimeAttributesBuilder: StandardValidatedRuntimeAttributesBuilder
-) extends BackendInitializationData
+  val runtimeAttributesBuilder: StandardValidatedRuntimeAttributesBuilder,
+  val standardExpressionFunctionsClass: Class[_ <: StandardExpressionFunctions]
+) extends BackendInitializationData {
 
-object StandardInitializationData {
-  def workflowPaths(initializationDataOption: Option[BackendInitializationData]): WorkflowPaths = {
-    BackendInitializationData.as[StandardInitializationData](initializationDataOption).workflowPaths
-  }
+  /* TODO: This could (should?) be monadic instead of reflection. */
+  private lazy val standardExpressionFunctionsConstructor =
+    standardExpressionFunctionsClass.getConstructor(classOf[StandardExpressionFunctionsParams])
 
-  def pathBuilders(initializationDataOption: Option[BackendInitializationData]): List[PathBuilder] = {
-    workflowPaths(initializationDataOption).pathBuilders
+  def expressionFunctions(jobPaths: JobPaths): StandardExpressionFunctions = {
+    val pathBuilders = jobPaths.asInstanceOf[WorkflowPaths].pathBuilders
+    val callContext = jobPaths.callContext
+    val standardParams = DefaultStandardExpressionFunctionsParams(pathBuilders, callContext)
+    standardExpressionFunctionsConstructor.newInstance(standardParams)
   }
 }
diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardJobExecutionActorParams.scala b/backend/src/main/scala/cromwell/backend/standard/StandardJobExecutionActorParams.scala
index 1261119d9..83e29a14e 100644
--- a/backend/src/main/scala/cromwell/backend/standard/StandardJobExecutionActorParams.scala
+++ b/backend/src/main/scala/cromwell/backend/standard/StandardJobExecutionActorParams.scala
@@ -1,12 +1,8 @@
 package cromwell.backend.standard
 
 import akka.actor.ActorRef
-import cromwell.backend.BackendJobExecutionActor.BackendJobExecutionResponse
 import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendJobDescriptor}
 
-import scala.concurrent.Promise
-import scala.language.existentials
-
 /**
   * Base trait for params passed to both the sync and async backend actors.
   */
@@ -29,47 +25,3 @@ trait StandardJobExecutionActorParams {
   /** The singleton actor. */
   def backendSingletonActorOption: Option[ActorRef]
 }
-
-/**
-  * Extended trait for params passed to synchronous backend actors.
-  */
-trait StandardSyncExecutionActorParams extends StandardJobExecutionActorParams {
-  /**
-    * The class for creating an async backend.
-    *
-    * @see [[StandardSyncExecutionActor]]
-    */
-  def asyncJobExecutionActorClass: Class[_ <: StandardAsyncExecutionActor]
-}
-
-/** A default implementation of the sync params. */
-case class DefaultStandardSyncExecutionActorParams
-(
-  override val jobIdKey: String,
-  override val serviceRegistryActor: ActorRef,
-  override val jobDescriptor: BackendJobDescriptor,
-  override val configurationDescriptor: BackendConfigurationDescriptor,
-  override val backendInitializationDataOption: Option[BackendInitializationData],
-  override val backendSingletonActorOption: Option[ActorRef],
-  override val asyncJobExecutionActorClass: Class[_ <: StandardAsyncExecutionActor]
-) extends StandardSyncExecutionActorParams
-
-/**
-  * Extended trait for params passed to asynchronous backend actors.
-  */
-trait StandardAsyncExecutionActorParams extends StandardJobExecutionActorParams {
-  /** The promise that will be completed when the async run is complete. */
-  def completionPromise: Promise[BackendJobExecutionResponse]
-}
-
-/** A default implementation of the async params. */
-case class DefaultStandardAsyncExecutionActorParams
-(
-  override val jobIdKey: String,
-  override val serviceRegistryActor: ActorRef,
-  override val jobDescriptor: BackendJobDescriptor,
-  override val configurationDescriptor: BackendConfigurationDescriptor,
-  override val backendInitializationDataOption: Option[BackendInitializationData],
-  override val backendSingletonActorOption: Option[ActorRef],
-  override val completionPromise: Promise[BackendJobExecutionResponse]
-) extends StandardAsyncExecutionActorParams
diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardLifecycleActorFactory.scala b/backend/src/main/scala/cromwell/backend/standard/StandardLifecycleActorFactory.scala
index 01cbe6eef..61d980e15 100644
--- a/backend/src/main/scala/cromwell/backend/standard/StandardLifecycleActorFactory.scala
+++ b/backend/src/main/scala/cromwell/backend/standard/StandardLifecycleActorFactory.scala
@@ -8,6 +8,7 @@ import cromwell.backend._
 import cromwell.core.{CallOutputs, Dispatcher}
 import cromwell.core.Dispatcher.BackendDispatcher
 import wdl4s.TaskCall
+import wdl4s.expression.WdlStandardLibraryFunctions
 
 /**
   * May be extended for using the standard sync/async backend pattern.
@@ -23,11 +24,25 @@ trait StandardLifecycleActorFactory extends BackendLifecycleActorFactory {
   def configurationDescriptor: BackendConfigurationDescriptor
 
   /**
+    * Returns the key to use for storing and looking up the job id.
+    *
+    * @return the key to use for storing and looking up the job id.
+    */
+  def jobIdKey: String
+
+  /**
+    * Returns the asynchronous executor class.
+    *
+    * @return the asynchronous executor class.
+    */
+  def asyncExecutionActorClass: Class[_ <: StandardAsyncExecutionActor]
+
+  /**
     * Returns the initialization class.
     *
     * @return the initialization class.
     */
-  def initializationActorClass: Class[_ <: StandardInitializationActor]
+  lazy val initializationActorClass: Class[_ <: StandardInitializationActor] = classOf[StandardInitializationActor]
 
   /**
     * Returns the synchronous executor class. By default using the standard sync executor should be sufficient for most
@@ -35,35 +50,23 @@ trait StandardLifecycleActorFactory extends BackendLifecycleActorFactory {
     *
     * @return the synchronous executor class.
     */
-  def syncExecutionActorClass: Class[_ <: StandardSyncExecutionActor] = classOf[StandardSyncExecutionActor]
-
-  /**
-    * Returns the asynchronous executor class.
-    *
-    * @return the asynchronous executor class.
-    */
-  def asyncExecutionActorClass: Class[_ <: StandardAsyncExecutionActor]
+  lazy val syncExecutionActorClass: Class[_ <: StandardSyncExecutionActor] = classOf[StandardSyncExecutionActor]
 
   /**
     * Returns the cache hit copying class.
     *
     * @return the cache hit copying class.
     */
-  def standardCacheHitCopyingActorOption: Option[Class[_ <: StandardCacheHitCopyingActor]] = None
+  lazy val cacheHitCopyingActorClassOption: Option[Class[_ <: StandardCacheHitCopyingActor]] =
+    Option(classOf[StandardCacheHitCopyingActor])
 
   /**
     * Returns the finalization class.
     *
     * @return the finalization class.
     */
-  def finalizationActorClassOption: Option[Class[_ <: StandardFinalizationActor]] = None
-
-  /**
-    * Returns the key to use for storing and looking up the job id.
-    *
-    * @return the key to use for storing and looking up the job id.
-    */
-  def jobIdKey: String
+  lazy val finalizationActorClassOption: Option[Class[_ <: StandardFinalizationActor]] =
+    Option(classOf[StandardFinalizationActor])
 
   override def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[TaskCall],
                                                 serviceRegistryActor: ActorRef): Option[Props] = {
@@ -96,7 +99,7 @@ trait StandardLifecycleActorFactory extends BackendLifecycleActorFactory {
 
   override def cacheHitCopyingActorProps:
   Option[(BackendJobDescriptor, Option[BackendInitializationData], ActorRef) => Props] = {
-    standardCacheHitCopyingActorOption map {
+    cacheHitCopyingActorClassOption map {
       standardCacheHitCopyingActor => cacheHitCopyingActorInner(standardCacheHitCopyingActor) _
     }
   }
@@ -134,6 +137,15 @@ trait StandardLifecycleActorFactory extends BackendLifecycleActorFactory {
       initializationDataOption, configurationDescriptor)
   }
 
+  override def expressionLanguageFunctions(workflowDescriptor: BackendWorkflowDescriptor,
+                                           jobKey: BackendJobDescriptorKey,
+                                           initializationDataOption: Option[BackendInitializationData]):
+  WdlStandardLibraryFunctions = {
+    val standardInitializationData = BackendInitializationData.as[StandardInitializationData](initializationDataOption)
+    val jobPaths = standardInitializationData.workflowPaths.toJobPaths(jobKey, workflowDescriptor)
+    standardInitializationData.expressionFunctions(jobPaths)
+  }
+
   override def getExecutionRootPath(workflowDescriptor: BackendWorkflowDescriptor, backendConfig: Config,
                                     initializationData: Option[BackendInitializationData]): Path = {
     initializationData.get.asInstanceOf[StandardInitializationData].workflowPaths.executionRoot
diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardSyncExecutionActor.scala b/backend/src/main/scala/cromwell/backend/standard/StandardSyncExecutionActor.scala
index 472fffa0f..11e487a0b 100644
--- a/backend/src/main/scala/cromwell/backend/standard/StandardSyncExecutionActor.scala
+++ b/backend/src/main/scala/cromwell/backend/standard/StandardSyncExecutionActor.scala
@@ -5,11 +5,28 @@ import akka.actor.{ActorRef, OneForOneStrategy, Props}
 import cromwell.backend.BackendJobExecutionActor.{AbortedResponse, BackendJobExecutionResponse}
 import cromwell.backend.BackendLifecycleActor.AbortJobCommand
 import cromwell.backend.async.AsyncBackendJobExecutionActor.{Execute, Recover}
-import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptor, BackendJobExecutionActor}
+import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendJobDescriptor, BackendJobExecutionActor}
 import cromwell.core.Dispatcher
 import cromwell.services.keyvalue.KeyValueServiceActor._
 
 import scala.concurrent.{Future, Promise}
+import scala.language.existentials
+
+trait StandardSyncExecutionActorParams extends StandardJobExecutionActorParams {
+  /** The class for creating an async backend. */
+  def asyncJobExecutionActorClass: Class[_ <: StandardAsyncExecutionActor]
+}
+
+case class DefaultStandardSyncExecutionActorParams
+(
+  override val jobIdKey: String,
+  override val serviceRegistryActor: ActorRef,
+  override val jobDescriptor: BackendJobDescriptor,
+  override val configurationDescriptor: BackendConfigurationDescriptor,
+  override val backendInitializationDataOption: Option[BackendInitializationData],
+  override val backendSingletonActorOption: Option[ActorRef],
+  override val asyncJobExecutionActorClass: Class[_ <: StandardAsyncExecutionActor]
+) extends StandardSyncExecutionActorParams
 
 /**
   * Facade to the asynchronous execution actor.
diff --git a/backend/src/main/scala/cromwell/backend/wdl/WdlFileMapper.scala b/backend/src/main/scala/cromwell/backend/wdl/WdlFileMapper.scala
new file mode 100644
index 000000000..b079f8cc5
--- /dev/null
+++ b/backend/src/main/scala/cromwell/backend/wdl/WdlFileMapper.scala
@@ -0,0 +1,28 @@
+package cromwell.backend.wdl
+
+import lenthall.util.TryUtil
+import wdl4s.values.{WdlArray, WdlFile, WdlMap, WdlValue}
+
+import scala.util.{Success, Try}
+
+object WdlFileMapper {
+  def mapWdlFiles(mapper: (WdlFile => WdlFile))(wdlValue: WdlValue): Try[WdlValue] = {
+    wdlValue match {
+      case file: WdlFile => Try(mapper(file))
+      case array: WdlArray =>
+        val mappedArray = array.value map mapWdlFiles(mapper)
+        TryUtil.sequence(mappedArray) map {
+          WdlArray(array.wdlType, _)
+        }
+      case map: WdlMap =>
+        val mappedMap = map.value map {
+          // TODO: TryUtil.sequenceMap doesn't currently handle keys of Try[WdlValue], so for now calling .get here
+          case (key, value) => mapWdlFiles(mapper)(key).get -> mapWdlFiles(mapper)(value)
+        }
+        TryUtil.sequenceMap(mappedMap) map {
+          WdlMap(map.wdlType, _)
+        }
+      case other => Success(other)
+    }
+  }
+}
diff --git a/core/src/main/scala/cromwell/core/core.scala b/core/src/main/scala/cromwell/core/core.scala
index 8f47182c0..bf7bd72d1 100644
--- a/core/src/main/scala/cromwell/core/core.scala
+++ b/core/src/main/scala/cromwell/core/core.scala
@@ -21,4 +21,6 @@ object CromwellFatalException {
 }
 
 class CromwellFatalException(val exception: Throwable) extends Exception(exception) with CromwellFatalExceptionMarker
-case class CromwellAggregatedException(throwables: Seq[Throwable], exceptionContext: String = "") extends ThrowableAggregation
+
+case class CromwellAggregatedException(throwables: Seq[Throwable], exceptionContext: String = "")
+  extends Exception with ThrowableAggregation
diff --git a/engine/src/test/scala/cromwell/MapWorkflowSpec.scala b/engine/src/test/scala/cromwell/MapWorkflowSpec.scala
index ccf6ace5a..bc3b1f740 100644
--- a/engine/src/test/scala/cromwell/MapWorkflowSpec.scala
+++ b/engine/src/test/scala/cromwell/MapWorkflowSpec.scala
@@ -13,7 +13,7 @@ import scala.util.{Success, Try}
 class MapWorkflowSpec extends CromwellTestKitSpec {
   private val pwd = File(".")
   private val sampleWdl = SampleWdl.MapLiteral(pwd.path)
-  val ns = WdlNamespaceWithWorkflow.load(sampleWdl.wdlSource(""), Seq.empty[ImportResolver])
+  val ns = WdlNamespaceWithWorkflow.load(sampleWdl.wdlSource(), Seq.empty[ImportResolver])
   val expectedMap = WdlMap(WdlMapType(WdlFileType, WdlStringType), Map(
     WdlFile("f1") -> WdlString("alice"),
     WdlFile("f2") -> WdlString("bob"),
@@ -32,8 +32,9 @@ class MapWorkflowSpec extends CromwellTestKitSpec {
             WdlString("x") -> WdlInteger(500),
             WdlString("y") -> WdlInteger(600),
             WdlString("z") -> WdlInteger(700)
-          )),
-          "wf.write_map.contents" -> WdlString("f1\talice\nf2\tbob\nf3\tchuck")
+          )) //, TODO: We now process file paths used as keys. So file paths like 'f1' will actually be mapped
+          // to ${PWD}/cromwell-executions/wf/<id>/call-write_map/inputs${PWD}/f1
+          //"wf.write_map.contents" -> WdlString("f1\talice\nf2\tbob\nf3\tchuck")
         )
       )
       sampleWdl.cleanup()
@@ -48,7 +49,7 @@ class MapWorkflowSpec extends CromwellTestKitSpec {
       val expression = declaration.expression.getOrElse {
         fail("Expected an expression for declaration 'map'")
       }
-      val value = expression.evaluate((s:String) => fail("No lookups"), NoFunctions).getOrElse {
+      val value = expression.evaluate((_: String) => fail("No lookups"), NoFunctions).getOrElse {
         fail("Expected expression for 'map' to evaluate")
       }
       expectedMap.wdlType.coerceRawValue(value).get shouldEqual expectedMap
diff --git a/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala
index 90f3c0fa5..73b65c5c5 100644
--- a/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala
+++ b/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala
@@ -10,7 +10,7 @@ import akka.util.Timeout
 import better.files._
 import com.typesafe.config.ConfigFactory
 import cromwell.CromwellTestKitSpec._
-import cromwell.core.{WorkflowSourceFilesCollection}
+import cromwell.core.WorkflowSourceFilesCollection
 import cromwell.engine.backend.BackendSingletonCollection
 import cromwell.engine.workflow.SingleWorkflowRunnerActor.RunWorkflow
 import cromwell.engine.workflow.SingleWorkflowRunnerActorSpec._
@@ -23,7 +23,7 @@ import org.scalatest.prop.{TableDrivenPropertyChecks, TableFor3}
 import spray.json._
 
 import scala.concurrent.Await
-import scala.concurrent.duration.Duration
+import scala.concurrent.duration._
 import scala.util._
 
 /**
@@ -242,7 +242,7 @@ class SingleWorkflowRunnerActorWithBadMetadataSpec extends SingleWorkflowRunnerA
       within(TimeoutDuration) {
         val runner = createRunnerActor(outputFile = Option(metadataDir.path))
         waitForErrorWithException(s"Specified metadata path is a directory, should be a file: $metadataDir") {
-          val futureResult = runner ? RunWorkflow
+          val futureResult = runner.ask(RunWorkflow)(30.seconds, implicitly)
           Await.ready(futureResult, Duration.Inf)
           futureResult.value.get match {
             case Success(_) =>
diff --git a/project/Dependencies.scala b/project/Dependencies.scala
index a964db00f..b78ffabb7 100644
--- a/project/Dependencies.scala
+++ b/project/Dependencies.scala
@@ -84,6 +84,8 @@ object Dependencies {
     "com.google.cloud" % "google-cloud-nio" % "0.3.0"
       exclude("com.google.api.grpc", "grpc-google-common-protos")
       exclude("com.google.cloud.datastore", "datastore-v1-protos")
+      exclude("org.apache.httpcomponents", "httpclient"),
+    "org.apache.httpcomponents" % "httpclient" % "4.5.2"
   )
 
   private val dbmsDependencies = List(
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala
index b510c2f58..a9e5d761c 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala
@@ -4,23 +4,20 @@ import java.net.SocketTimeoutException
 import java.nio.file.{Path, Paths}
 
 import akka.actor.ActorRef
-import better.files._
 import com.google.api.client.googleapis.json.GoogleJsonResponseException
 import com.google.cloud.storage.contrib.nio.CloudStoragePath
 import cromwell.backend._
-import cromwell.backend.async.{AbortedExecutionHandle, ExecutionHandle, FailedNonRetryableExecutionHandle, FailedRetryableExecutionHandle, PendingExecutionHandle, SuccessfulExecutionHandle}
+import cromwell.backend.async.{AbortedExecutionHandle, ExecutionHandle, FailedNonRetryableExecutionHandle, FailedRetryableExecutionHandle, PendingExecutionHandle}
 import cromwell.backend.impl.jes.RunStatus.TerminalRunStatus
 import cromwell.backend.impl.jes.io._
 import cromwell.backend.impl.jes.statuspolling.JesPollingActorClient
 import cromwell.backend.standard.{StandardAsyncExecutionActor, StandardAsyncExecutionActorParams, StandardAsyncJob}
-import cromwell.backend.wdl.OutputEvaluator
 import cromwell.core._
-import cromwell.core.path.PathFactory._
 import cromwell.core.path.PathImplicits._
 import cromwell.core.path.proxy.PathProxy
 import cromwell.core.retry.SimpleExponentialBackoff
 import wdl4s._
-import wdl4s.expression.{NoFunctions, WdlFunctions}
+import wdl4s.expression.NoFunctions
 import wdl4s.values._
 
 import scala.concurrent.duration._
@@ -89,14 +86,6 @@ class JesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn
     else None
   }
 
-  private lazy val callContext = CallContext(
-    callRootPath,
-    jesStdoutFile.toRealString,
-    jesStderrFile.toRealString
-  )
-
-  private[jes] lazy val backendEngineFunctions = new JesExpressionFunctions(List(jesCallPaths.gcsPathBuilder), callContext)
-
   /**
     * Takes two arrays of remote and local WDL File paths and generates the necessary JesInputs.
     */
@@ -205,61 +194,19 @@ class JesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn
     )
   }
 
-  override lazy val commandLineFunctions: WdlFunctions[WdlValue] = backendEngineFunctions
+  override lazy val commandDirectory: Path = JesWorkingDisk.MountPoint
 
-  override lazy val commandLinePreProcessor: (EvaluatedTaskInputs) => Try[EvaluatedTaskInputs] = mapGcsValues
-
-  def mapGcsValues(inputs: EvaluatedTaskInputs): Try[EvaluatedTaskInputs] = {
-    Try(inputs mapValues gcsPathToLocal)
-  }
-
-  override def commandLineValueMapper: (WdlValue) => WdlValue = gcsPathToLocal
-
-  private def uploadCommandScript(command: String, withMonitoring: Boolean, globFiles: Set[WdlGlobFile]): Unit = {
-    val monitoring = if (withMonitoring) {
+  override def commandScriptPreamble: String = {
+    if (monitoringOutput.isDefined) {
       s"""|touch $JesMonitoringLogFile
           |chmod u+x $JesMonitoringScript
           |$JesMonitoringScript > $JesMonitoringLogFile &""".stripMargin
     } else ""
+  }
 
-    val tmpDir = File(JesWorkingDisk.MountPoint)./("tmp").path
-    val rcPath = File(JesWorkingDisk.MountPoint)./(returnCodeFilename).path
-    val rcTmpPath = pathPlusSuffix(rcPath, "tmp").path
-
-    def globManipulation(globFile: WdlGlobFile) = {
-
-      val globDir = backendEngineFunctions.globName(globFile.value)
-      val (_, disk) = relativePathAndAttachedDisk(globFile.value, runtimeAttributes.disks)
-      val globDirectory = File(disk.mountPoint)./(globDir)
-      val globList = File(disk.mountPoint)./(s"$globDir.list")
-
-      s"""|mkdir $globDirectory
-          |( ln -L ${globFile.value} $globDirectory 2> /dev/null ) || ( ln ${globFile.value} $globDirectory )
-          |ls -1 $globDirectory > $globList
-          |""".stripMargin
-    }
-
-    val globManipulations = globFiles.map(globManipulation).mkString("\n")
-
-    val fileContent =
-      s"""|#!/bin/bash
-          |export _JAVA_OPTIONS=-Djava.io.tmpdir=$tmpDir
-          |export TMPDIR=$tmpDir
-          |$monitoring
-          |(
-          |cd ${JesWorkingDisk.MountPoint}
-          |INSTANTIATED_COMMAND
-          |)
-          |echo $$? > $rcTmpPath
-          |(
-          |cd ${JesWorkingDisk.MountPoint}
-          |$globManipulations
-          |)
-          |mv $rcTmpPath $rcPath
-          |""".stripMargin.replace("INSTANTIATED_COMMAND", command)
-
-    jesCallPaths.script.writeAsText(fileContent)
-    ()
+  override def globParentDirectory(wdlGlobFile: WdlGlobFile): Path = {
+    val (_, disk) = relativePathAndAttachedDisk(wdlGlobFile.value, runtimeAttributes.disks)
+    disk.mountPoint
   }
 
   private def googleProject(descriptor: BackendWorkflowDescriptor): String = {
@@ -305,14 +252,12 @@ class JesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn
     // Force runtimeAttributes to evaluate so we can fail quickly now if we need to:
     Try(runtimeAttributes) match {
       case Success(_) =>
-        val command = instantiatedCommand
         val jesInputs: Set[JesInput] = generateJesInputs(jobDescriptor) ++ monitoringScript + cmdInput
         val jesOutputs: Set[JesFileOutput] = generateJesOutputs(jobDescriptor) ++ monitoringOutput
-        val withMonitoring = monitoringOutput.isDefined
 
         val jesParameters = standardParameters ++ gcsAuthParameter ++ jesInputs ++ jesOutputs
 
-        uploadCommandScript(command, withMonitoring, backendEngineFunctions.findGlobOutputs(call, jobDescriptor))
+        jobPaths.script.writeAsText(commandScriptContents)
         val run = createJesRun(jesParameters, runIdForResumption)
         PendingExecutionHandle(jobDescriptor, StandardAsyncJob(run.runId), Option(run), previousStatus = None)
       case Failure(e) => FailedNonRetryableExecutionHandle(e)
@@ -345,29 +290,14 @@ class JesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn
     }
   }
 
-  private[jes] def wdlValueToGcsPath(jesOutputs: Set[JesFileOutput])(value: WdlValue): WdlValue = {
-    def toGcsPath(wdlFile: WdlFile) = jesOutputs collectFirst {
-      case o if o.name == makeSafeJesReferenceName(wdlFile.valueString) => WdlFile(o.gcs)
-    } getOrElse value
-
-    value match {
-      case wdlArray: WdlArray => wdlArray map wdlValueToGcsPath(jesOutputs)
-      case wdlMap: WdlMap => wdlMap map {
-        case (k, v) => wdlValueToGcsPath(jesOutputs)(k) -> wdlValueToGcsPath(jesOutputs)(v)
-      }
-      case file: WdlFile => toGcsPath(file)
-      case other => other
-    }
+  override def mapOutputWdlFile(wdlFile: WdlFile): WdlFile = {
+    wdlFileToGcsPath(generateJesOutputs(jobDescriptor))(wdlFile)
   }
 
-  private def postProcess: Try[CallOutputs] = {
-    def wdlValueToSuccess(value: WdlValue): Try[WdlValue] = Success(value)
-
-    OutputEvaluator.evaluateOutputs(
-      jobDescriptor,
-      backendEngineFunctions,
-      (wdlValueToSuccess _).compose(wdlValueToGcsPath(generateJesOutputs(jobDescriptor)))
-    )
+  private[jes] def wdlFileToGcsPath(jesOutputs: Set[JesFileOutput])(wdlFile: WdlFile): WdlFile = {
+    jesOutputs collectFirst {
+      case jesOutput if jesOutput.name == makeSafeJesReferenceName(wdlFile.valueString) => WdlFile(jesOutput.gcs)
+    } getOrElse wdlFile
   }
 
   override def isSuccess(runStatus: RunStatus): Boolean = {
@@ -380,24 +310,19 @@ class JesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn
     }
   }
 
-  override def handleExecutionSuccess(runStatus: RunStatus,
-                                      handle: StandardAsyncPendingExecutionHandle,
-                                      returnCode: Int): ExecutionHandle = {
-    val success = runStatus match {
-      case successStatus: RunStatus.Success => successStatus
+  override def getTerminalEvents(runStatus: RunStatus): Seq[ExecutionEvent] = {
+    runStatus match {
+      case successStatus: RunStatus.Success => successStatus.eventList
       case unknown =>
         throw new RuntimeException(s"handleExecutionSuccess not called with RunStatus.Success. Instead got $unknown")
     }
-    val outputMappings = postProcess
-    val jobDetritusFiles = jesCallPaths.detritusPaths
-    val executionHandle = handle
-    val events = success.eventList
-    outputMappings match {
-      case Success(outputs) => SuccessfulExecutionHandle(outputs, returnCode, jobDetritusFiles, events)
-      case Failure(ex: CromwellAggregatedException) if ex.throwables collectFirst { case s: SocketTimeoutException => s } isDefined =>
-        // Return the execution handle in this case to retry the operation
-        executionHandle
-      case Failure(ex) => FailedNonRetryableExecutionHandle(ex)
+  }
+
+  override def retryEvaluateOutputs(exception: Exception): Boolean = {
+    exception match {
+      case aggregated: CromwellAggregatedException =>
+        aggregated.throwables.collectFirst { case s: SocketTimeoutException => s }.isDefined
+      case _ => false
     }
   }
 
@@ -432,7 +357,7 @@ class JesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn
       val exception = failed.toFailure(jobPaths.jobKey.tag, Option(jobPaths.stderr))
       FailedNonRetryableExecutionHandle(exception, returnCode)
     }
-    
+
     val errorCode = failed.errorCode
     val taskName = s"${workflowDescriptor.id}:${call.unqualifiedName}"
     val attempt = jobDescriptor.key.attempt
@@ -460,12 +385,6 @@ class JesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn
     }
   }
 
-  // TODO: Adapter for left over test code. Not used by main.
-  private[jes] def executionResult(status: TerminalRunStatus, handle: JesPendingExecutionHandle)
-                                  (implicit ec: ExecutionContext): Future[ExecutionHandle] = {
-    Future.fromTry(Try(handleExecutionResult(status, handle)))
-  }
-
   /**
     * Takes a path in GCS and comes up with a local path which is unique for the given GCS path.
     *
@@ -479,30 +398,15 @@ class JesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn
     mountPoint.resolve(gcsPath.bucket()).resolve(gcsPath.toUri.getPath.stripPrefix("/"))
   }
 
-  /**
-    * Takes a single WdlValue and maps google cloud storage (GCS) paths into an appropriate local file path.
-    * If the input is not a WdlFile, or the WdlFile is not a GCS path, the mapping is a noop.
-    *
-    * @param wdlValue the value of the input
-    * @return a new FQN to WdlValue pair, with WdlFile paths modified if appropriate.
-    */
-  private[jes] def gcsPathToLocal(wdlValue: WdlValue): WdlValue = {
-    wdlValue match {
-      case wdlFile: WdlFile =>
-        getPath(wdlFile.valueString) match {
-          case Success(gcsPath: CloudStoragePath) =>
-            WdlFile(localFilePathFromCloudStoragePath(workingDisk.mountPoint, gcsPath).toString, wdlFile.isGlob)
-          case Success(proxy: PathProxy) =>
-            proxy.unbox(classOf[CloudStoragePath]) map { gcsPath =>
-              WdlFile(localFilePathFromCloudStoragePath(workingDisk.mountPoint, gcsPath).toString, wdlFile.isGlob)
-            } getOrElse wdlValue
-          case _ => wdlValue
-        }
-      case wdlArray: WdlArray => wdlArray map gcsPathToLocal
-      case wdlMap: WdlMap => wdlMap map { case (k, v) => gcsPathToLocal(k) -> gcsPathToLocal(v) }
-      case wdlPair: WdlPair => WdlPair(gcsPathToLocal(wdlPair.left), gcsPathToLocal(wdlPair.right))
-      case wdlOptional: WdlOptionalValue => wdlOptional.copy(value = wdlOptional.value map gcsPathToLocal)
-      case _ => wdlValue
+  override def mapCommandLineWdlFile(wdlFile: WdlFile): WdlFile = {
+    getPath(wdlFile.valueString) match {
+      case Success(gcsPath: CloudStoragePath) =>
+        WdlFile(localFilePathFromCloudStoragePath(workingDisk.mountPoint, gcsPath).toString, wdlFile.isGlob)
+      case Success(proxy: PathProxy) =>
+        proxy.unbox(classOf[CloudStoragePath]) map { gcsPath =>
+          WdlFile(localFilePathFromCloudStoragePath(workingDisk.mountPoint, gcsPath).toString, wdlFile.isGlob)
+        } getOrElse wdlFile
+      case _ => wdlFile
     }
   }
 }
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendInitializationData.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendInitializationData.scala
index 7662b79e6..06eb747b2 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendInitializationData.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendInitializationData.scala
@@ -9,4 +9,4 @@ case class JesBackendInitializationData
   override val runtimeAttributesBuilder: StandardValidatedRuntimeAttributesBuilder,
   jesConfiguration: JesConfiguration,
   genomics: Genomics
-) extends StandardInitializationData(workflowPaths, runtimeAttributesBuilder)
+) extends StandardInitializationData(workflowPaths, runtimeAttributesBuilder, classOf[JesExpressionFunctions])
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala
index 7026c8cc2..5bd04e7f9 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala
@@ -7,23 +7,19 @@ import cromwell.backend.impl.jes.callcaching.JesBackendFileHashing
 import cromwell.backend.standard._
 import cromwell.core.CallOutputs
 import wdl4s.TaskCall
-import wdl4s.expression.WdlStandardLibraryFunctions
 
 case class JesBackendLifecycleActorFactory(name: String, configurationDescriptor: BackendConfigurationDescriptor)
   extends StandardLifecycleActorFactory {
 
-  override def initializationActorClass: Class[_ <: StandardInitializationActor] = classOf[JesInitializationActor]
+  override lazy val initializationActorClass: Class[_ <: StandardInitializationActor] = classOf[JesInitializationActor]
 
-  override def asyncExecutionActorClass: Class[_ <: StandardAsyncExecutionActor] =
+  override lazy val asyncExecutionActorClass: Class[_ <: StandardAsyncExecutionActor] =
     classOf[JesAsyncBackendJobExecutionActor]
 
-  override def standardCacheHitCopyingActorOption: Option[Class[_ <: StandardCacheHitCopyingActor]] =
-    Option(classOf[JesCacheHitCopyingActor])
-
-  override def finalizationActorClassOption: Option[Class[_ <: StandardFinalizationActor]] =
+  override lazy val finalizationActorClassOption: Option[Class[_ <: StandardFinalizationActor]] =
     Option(classOf[JesFinalizationActor])
 
-  override def jobIdKey: String = JesAsyncBackendJobExecutionActor.JesOperationIdKey
+  override lazy val jobIdKey: String = JesAsyncBackendJobExecutionActor.JesOperationIdKey
 
   val jesConfiguration = new JesConfiguration(configurationDescriptor)
 
@@ -44,17 +40,6 @@ case class JesBackendLifecycleActorFactory(name: String, configurationDescriptor
       initializationDataOption)
   }
 
-  override def expressionLanguageFunctions(workflowDescriptor: BackendWorkflowDescriptor,
-                                           jobKey: BackendJobDescriptorKey,
-                                           initializationDataOption: Option[BackendInitializationData]
-                                          ): WdlStandardLibraryFunctions = {
-    val initializationData = BackendInitializationData.
-      as[JesBackendInitializationData](initializationDataOption)
-
-    val jesCallPaths = initializationData.workflowPaths.toJobPaths(jobKey, workflowDescriptor)
-    new JesExpressionFunctions(List(jesCallPaths.gcsPathBuilder), jesCallPaths.callContext)
-  }
-
   override def backendSingletonActorProps = Option(JesBackendSingletonActor.props(jesConfiguration.qps))
 
   override lazy val fileHashingFunction: Option[FileHashingFunction] = Option(FileHashingFunction(JesBackendFileHashing.getCrc32c))
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesCacheHitCopyingActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesCacheHitCopyingActor.scala
deleted file mode 100644
index 631eda1b8..000000000
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesCacheHitCopyingActor.scala
+++ /dev/null
@@ -1,11 +0,0 @@
-package cromwell.backend.impl.jes
-
-import java.nio.file.Path
-
-import cromwell.backend.standard.{StandardCacheHitCopyingActor, StandardCacheHitCopyingActorParams}
-import cromwell.core.path.PathCopier
-
-case class JesCacheHitCopyingActor(override val standardParams: StandardCacheHitCopyingActorParams)
-  extends StandardCacheHitCopyingActor with JesJobCachingActorHelper {
-  override protected def duplicate(source: Path, destination: Path): Unit = PathCopier.copy(source, destination).get
-}
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesExpressionFunctions.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesExpressionFunctions.scala
index 8e16b08a5..dbfc866d7 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesExpressionFunctions.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesExpressionFunctions.scala
@@ -1,31 +1,16 @@
 package cromwell.backend.impl.jes
 
-import java.nio.file.Path
-
-import cromwell.backend.io.GlobFunctions
-import cromwell.backend.wdl.{ReadLikeFunctions, WriteFunctions}
-import cromwell.core.CallContext
-import cromwell.core.path.PathBuilder
-import cromwell.filesystems.gcs.GcsPathBuilder
+import cromwell.backend.standard.{StandardExpressionFunctions, StandardExpressionFunctionsParams}
 import cromwell.core.path.PathImplicits._
-import wdl4s.expression.{PureStandardLibraryFunctionsLike, WdlStandardLibraryFunctions}
-import wdl4s.values._
-
-import scala.util.{Success, Try}
-
-class JesExpressionFunctions(override val pathBuilders: List[PathBuilder], context: CallContext)
-  extends WdlStandardLibraryFunctions with PureStandardLibraryFunctionsLike with ReadLikeFunctions with WriteFunctions with GlobFunctions {
-
-  def callContext: CallContext = context
-
-  override def writeTempFile(path: String, prefix: String, suffix: String, content: String): String = super[WriteFunctions].writeTempFile(path, prefix, suffix, content)
-
-  override def preMapping(str: String): String = if (!GcsPathBuilder.isValidGcsUrl(str)) {
-    context.root.resolve(str.stripPrefix("/")).toRealString
-  } else str
+import cromwell.filesystems.gcs.GcsPathBuilder
 
-  override def stdout(params: Seq[Try[WdlValue]]) = Success(WdlFile(context.stdout))
-  override def stderr(params: Seq[Try[WdlValue]]) = Success(WdlFile(context.stderr))
+class JesExpressionFunctions(standardParams: StandardExpressionFunctionsParams)
+  extends StandardExpressionFunctions(standardParams) {
 
-  override val writeDirectory: Path = context.root
+  override def preMapping(str: String): String =
+    if (!GcsPathBuilder.isValidGcsUrl(str)) {
+      callContext.root.resolve(str.stripPrefix("/")).toRealString
+    } else {
+      str
+    }
 }
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesFinalizationActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesFinalizationActor.scala
index 0cf19bc2d..b5729dbbc 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesFinalizationActor.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesFinalizationActor.scala
@@ -1,15 +1,11 @@
 package cromwell.backend.impl.jes
 
-import java.nio.file.Path
-
 import better.files._
 import cats.instances.future._
 import cats.syntax.functor._
 import cromwell.backend._
 import cromwell.backend.standard.{StandardFinalizationActor, StandardFinalizationActorParams}
 import cromwell.core.CallOutputs
-import cromwell.core.Dispatcher.IoDispatcher
-import cromwell.core.path.PathCopier
 import wdl4s.TaskCall
 
 import scala.concurrent.Future
@@ -27,68 +23,23 @@ case class JesFinalizationActorParams
   override val configurationDescriptor: BackendConfigurationDescriptor = jesConfiguration.configurationDescriptor
 }
 
-class JesFinalizationActor(jesParams: JesFinalizationActorParams)
-  extends StandardFinalizationActor {
-
-  override val standardParams: StandardFinalizationActorParams = jesParams
+class JesFinalizationActor(val jesParams: JesFinalizationActorParams)
+  extends StandardFinalizationActor(jesParams) {
 
   lazy val jesConfiguration: JesConfiguration = jesParams.jesConfiguration
 
-  private val workflowPaths = initializationDataOption.map {
-    _.asInstanceOf[JesBackendInitializationData].workflowPaths
-  }
-
-  private val iOExecutionContext = context.system.dispatchers.lookup(IoDispatcher)
-
   override def afterAll(): Future[Unit] = {
     for {
       // NOTE: These are currently in series, not in parallel. Not sure how many threads to throw at finalization
       _ <- deleteAuthenticationFile()
-      _ <- copyCallOutputs()
+      _ <- super.afterAll()
     } yield ()
   }
 
   private def deleteAuthenticationFile(): Future[Unit] = {
     (jesConfiguration.needAuthFileUpload, workflowPaths) match {
-      case (true, Some(paths)) => Future { File(paths.gcsAuthFilePath).delete(false) } void
-      case _ => Future.successful(())
-    }
-  }
-
-  private def copyCallOutputs(): Future[Unit] = {
-    /*
-    NOTE: Only using one thread pool slot here to upload all the files for all the calls.
-    Using the io-dispatcher defined in application.conf because this might take a while.
-    One could also use Future.sequence to flood the dispatcher, or even create a separate jes final call specific thread
-    pool for parallel uploads.
-
-    Measure and optimize as necessary. Will likely need retry code at some level as well.
-     */
-    workflowPaths match {
-      case Some(paths) => Future(paths.finalCallLogsPath foreach copyCallOutputs)(iOExecutionContext)
+      case (true, Some(paths: JesWorkflowPaths)) => Future { File(paths.gcsAuthFilePath).delete(false) } void
       case _ => Future.successful(())
     }
   }
-
-  private def copyCallOutputs(callLogsPath: Path): Unit = {
-    copyLogs(callLogsPath, logPaths)
-  }
-
-  private lazy val logPaths: Seq[Path] = {
-    val allCallPaths = jobExecutionMap flatMap {
-      case (backendJobDescriptor, keys) =>
-        keys map { JesJobPaths(_, backendJobDescriptor, jesConfiguration)(context.system) }
-    }
-
-    allCallPaths.toSeq flatMap { callPaths =>
-      Seq(callPaths.stdout, callPaths.stderr, callPaths.jesLogPath)
-    }
-  }
-
-  private def copyLogs(callLogsDirPath: Path, logPaths: Seq[Path]): Unit = {
-    workflowPaths match {
-      case Some(paths) => logPaths.foreach(PathCopier.copy(paths.executionRoot, _, callLogsDirPath))
-      case None =>
-    }
-  }
 }
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesInitializationActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesInitializationActor.scala
index 3ebaa570d..d9aaede16 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesInitializationActor.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesInitializationActor.scala
@@ -2,32 +2,17 @@ package cromwell.backend.impl.jes
 
 import java.io.IOException
 
-import akka.actor.{ActorRef, Props}
-import cats.instances.future._
-import cats.syntax.functor._
-import com.google.api.services.genomics.Genomics
+import akka.actor.ActorRef
 import cromwell.backend.impl.jes.authentication.{GcsLocalizing, JesAuthInformation}
 import cromwell.backend.impl.jes.io._
 import cromwell.backend.standard.{StandardInitializationActor, StandardInitializationActorParams, StandardValidatedRuntimeAttributesBuilder}
 import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendWorkflowDescriptor}
-import cromwell.core.Dispatcher.BackendDispatcher
 import cromwell.filesystems.gcs.auth.{ClientSecrets, GoogleAuthMode}
 import spray.json.JsObject
 import wdl4s.TaskCall
 
 import scala.concurrent.Future
-import scala.language.postfixOps
-
-object JesInitializationActor {
-  /* NOTE: Only used by tests */
-  def props(workflowDescriptor: BackendWorkflowDescriptor,
-            calls: Set[TaskCall],
-            jesConfiguration: JesConfiguration,
-            serviceRegistryActor: ActorRef): Props = {
-    val params = JesInitializationActorParams(workflowDescriptor, calls, jesConfiguration, serviceRegistryActor)
-    Props(new JesInitializationActor(params)).withDispatcher(BackendDispatcher)
-  }
-}
+import scala.util.Try
 
 case class JesInitializationActorParams
 (
@@ -40,9 +25,7 @@ case class JesInitializationActorParams
 }
 
 class JesInitializationActor(jesParams: JesInitializationActorParams)
-  extends StandardInitializationActor {
-
-  override val standardParams: StandardInitializationActorParams = jesParams
+  extends StandardInitializationActor(jesParams) {
 
   private val jesConfiguration = jesParams.jesConfiguration
 
@@ -56,31 +39,31 @@ class JesInitializationActor(jesParams: JesInitializationActorParams)
     } yield GcsLocalizing(clientSecrets, token)
   }
 
-  /**
-    * A call which happens before anything else runs
-    */
-  override def beforeAll(): Future[Option[BackendInitializationData]] = {
+  private lazy val genomics = jesConfiguration.genomicsFactory.withOptions(workflowDescriptor.workflowOptions)
 
-    def buildGenomics: Future[Genomics] = Future {
-      jesConfiguration.genomicsFactory.withOptions(workflowDescriptor.workflowOptions)
-    }
+  override lazy val workflowPaths: JesWorkflowPaths =
+    new JesWorkflowPaths(workflowDescriptor, jesConfiguration)(context.system)
 
-    for {
-      genomics <- buildGenomics
-      workflowPaths = new JesWorkflowPaths(workflowDescriptor, jesConfiguration)(context.system)
-      _ <- if (jesConfiguration.needAuthFileUpload) writeAuthenticationFile(workflowPaths) else Future.successful(())
-      _ = publishWorkflowRoot(workflowPaths.workflowRoot.toString)
-    } yield Option(JesBackendInitializationData(workflowPaths, runtimeAttributesBuilder, jesConfiguration, genomics))
-  }
 
-  private def writeAuthenticationFile(workflowPath: JesWorkflowPaths): Future[Unit] = {
-    generateAuthJson(jesConfiguration.dockerCredentials, refreshTokenAuth) map { content =>
+  override lazy val initializationData: JesBackendInitializationData =
+    JesBackendInitializationData(workflowPaths, runtimeAttributesBuilder, jesConfiguration, genomics)
+
+  override def beforeAll(): Future[Option[BackendInitializationData]] = Future.fromTry(Try {
+    if (jesConfiguration.needAuthFileUpload) writeAuthenticationFile(workflowPaths)
+    publishWorkflowRoot(workflowPaths.workflowRoot.toString)
+    Option(initializationData)
+  })
+
+  private def writeAuthenticationFile(workflowPath: JesWorkflowPaths): Unit = {
+    generateAuthJson(jesConfiguration.dockerCredentials, refreshTokenAuth) foreach { content =>
       val path = workflowPath.gcsAuthFilePath
       workflowLogger.info(s"Creating authentication file for workflow ${workflowDescriptor.id} at \n ${path.toUri}")
-      Future(path.writeAsJson(content)).void.recoverWith {
-        case failure => Future.failed(new IOException("Failed to upload authentication file", failure))
-      } void
-    } getOrElse Future.successful(())
+      try {
+        path.writeAsJson(content)
+      } catch {
+        case exception: Exception => throw new IOException("Failed to upload authentication file", exception)
+      }
+    }
   }
 
   def generateAuthJson(authInformation: Option[JesAuthInformation]*): Option[String] = {
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobPaths.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobPaths.scala
index fdbdb1dc7..b1932b7c8 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobPaths.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobPaths.scala
@@ -5,7 +5,6 @@ import java.nio.file.Path
 import akka.actor.ActorSystem
 import cromwell.backend.io.JobPaths
 import cromwell.backend.{BackendJobDescriptorKey, BackendWorkflowDescriptor}
-import cromwell.core.CallContext
 import cromwell.services.metadata.CallMetadataKeys
 
 object JesJobPaths {
@@ -35,8 +34,6 @@ class JesJobPaths(val jobKey: BackendJobDescriptorKey, workflowDescriptor: Backe
   val jesLogFilename: String = s"$jesLogBasename.log"
   lazy val jesLogPath: Path = callExecutionRoot.resolve(jesLogFilename)
   
-  lazy val callContext = CallContext(callExecutionRoot, stdoutFilename, stderrFilename)
-
   /*
   TODO: Move various monitoring files path generation here.
 
@@ -57,4 +54,8 @@ class JesJobPaths(val jobKey: BackendJobDescriptorKey, workflowDescriptor: Backe
     JesJobPaths.GcsExecPathKey -> script,
     JesJobPaths.JesLogPathKey -> jesLogPath
   )
+
+  override lazy val customLogPaths: Map[String, Path] = Map(
+    JesJobPaths.JesLogPathKey -> jesLogPath
+  )
 }
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesRuntimeAttributes.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesRuntimeAttributes.scala
index b62aa445b..f64716d9b 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesRuntimeAttributes.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesRuntimeAttributes.scala
@@ -9,7 +9,6 @@ import cromwell.backend.impl.jes.io.{JesAttachedDisk, JesWorkingDisk}
 import cromwell.backend.standard.StandardValidatedRuntimeAttributesBuilder
 import cromwell.backend.validation.{BooleanRuntimeAttributesValidation, _}
 import lenthall.validation.ErrorOr._
-import org.slf4j.Logger
 import wdl4s.types._
 import wdl4s.values._
 
@@ -105,14 +104,6 @@ object JesRuntimeAttributes {
       noAddress
     )
   }
-
-  // NOTE: Currently only used by test specs
-  private[jes] def apply(attrs: Map[String, WdlValue], logger: Logger,
-                         jesConfiguration: JesConfiguration): JesRuntimeAttributes = {
-    val runtimeAttributesBuilder = JesRuntimeAttributes.runtimeAttributesBuilder(jesConfiguration)
-    val validatedRuntimeAttributes = runtimeAttributesBuilder.build(attrs, logger)
-    apply(validatedRuntimeAttributes)
-  }
 }
 
 object ZonesValidation extends RuntimeAttributesValidation[Vector[String]] {
diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala
index 9d33c195b..3ddc1bc8d 100644
--- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala
+++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala
@@ -14,10 +14,12 @@ import cromwell.backend.impl.jes.JesAsyncBackendJobExecutionActor.JesPendingExec
 import cromwell.backend.impl.jes.RunStatus.Failed
 import cromwell.backend.impl.jes.io.{DiskType, JesWorkingDisk}
 import cromwell.backend.impl.jes.statuspolling.JesApiQueryManager.DoPoll
-import cromwell.backend.standard.{DefaultStandardAsyncExecutionActorParams, StandardAsyncExecutionActorParams, StandardAsyncJob}
+import cromwell.backend.standard.{DefaultStandardAsyncExecutionActorParams, StandardAsyncExecutionActorParams, StandardAsyncJob, StandardExpressionFunctionsParams}
+import cromwell.backend.wdl.WdlFileMapper
 import cromwell.core._
 import cromwell.core.labels.Labels
 import cromwell.core.logging.JobLogger
+import cromwell.core.path.PathBuilder
 import cromwell.core.path.PathImplicits._
 import cromwell.filesystems.gcs.auth.GoogleAuthMode.NoAuthMode
 import cromwell.filesystems.gcs.{GcsPathBuilder, GcsPathBuilderFactory}
@@ -71,8 +73,13 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
 
   lazy val TestableCallContext = CallContext(mockPathBuilder.build("gs://root").get, "out", "err")
 
+  lazy val TestableStandardExpressionFunctionsParams = new StandardExpressionFunctionsParams {
+    override lazy val pathBuilders: List[PathBuilder] = List(mockPathBuilder)
+    override lazy val callContext: CallContext = TestableCallContext
+  }
+
   lazy val TestableJesExpressionFunctions: JesExpressionFunctions = {
-    new JesExpressionFunctions(List(mockPathBuilder), TestableCallContext)
+    new JesExpressionFunctions(TestableStandardExpressionFunctionsParams)
   }
 
   private def buildInitializationData(jobDescriptor: BackendJobDescriptor, configuration: JesConfiguration) = {
@@ -216,10 +223,9 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
     val actorRef = buildPreemptibleTestActorRef(2, 1)
     val jesBackend = actorRef.underlyingActor
     val handle = mock[JesPendingExecutionHandle]
-    implicit val ec = system.dispatcher
 
     val failedStatus = Failed(10, Option("14: VM XXX shut down unexpectedly."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance"))
-    val executionResult = Await.result(jesBackend.executionResult(failedStatus, handle), 2.seconds)
+    val executionResult = jesBackend.handleExecutionResult(failedStatus, handle)
     executionResult.isInstanceOf[FailedNonRetryableExecutionHandle] shouldBe true
     val failedHandle = executionResult.asInstanceOf[FailedNonRetryableExecutionHandle]
     failedHandle.returnCode shouldBe None
@@ -229,10 +235,9 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
     val actorRef = buildPreemptibleTestActorRef(1, 1)
     val jesBackend = actorRef.underlyingActor
     val handle = mock[JesPendingExecutionHandle]
-    implicit val ec = system.dispatcher
 
     val failedStatus = Failed(10, Option("14: VM XXX shut down unexpectedly."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance"))
-    val executionResult = Await.result(jesBackend.executionResult(failedStatus, handle), 2.seconds)
+    val executionResult = jesBackend.handleExecutionResult(failedStatus, handle)
     executionResult.isInstanceOf[FailedRetryableExecutionHandle] shouldBe true
     val retryableHandle = executionResult.asInstanceOf[FailedRetryableExecutionHandle]
     retryableHandle.throwable.isInstanceOf[PreemptedException] shouldBe true
@@ -245,10 +250,9 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
     val actorRef = buildPreemptibleTestActorRef(1, 2)
     val jesBackend = actorRef.underlyingActor
     val handle = mock[JesPendingExecutionHandle]
-    implicit val ec = system.dispatcher
 
     val failedStatus = Failed(10, Option("14: VM XXX shut down unexpectedly."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance"))
-    val executionResult = Await.result(jesBackend.executionResult(failedStatus, handle), 2.seconds)
+    val executionResult = jesBackend.handleExecutionResult(failedStatus, handle)
     executionResult.isInstanceOf[FailedRetryableExecutionHandle] shouldBe true
     val retryableHandle = executionResult.asInstanceOf[FailedRetryableExecutionHandle]
     retryableHandle.throwable.isInstanceOf[PreemptedException] shouldBe true
@@ -261,26 +265,21 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
     val actorRef = buildPreemptibleTestActorRef(1, 1)
     val jesBackend = actorRef.underlyingActor
     val handle = mock[JesPendingExecutionHandle]
-    implicit val ec = system.dispatcher
-
-    Await.result(jesBackend.executionResult(
-      Failed(10, Option("15: Other type of error."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")), handle), 2.seconds
-    ).isInstanceOf[FailedNonRetryableExecutionHandle] shouldBe true
-    Await.result(jesBackend.executionResult(
-      Failed(11, Option("14: Wrong errorCode."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")), handle), 2.seconds
-    ).isInstanceOf[FailedNonRetryableExecutionHandle] shouldBe true
-    Await.result(jesBackend.executionResult(
-      Failed(10, Option("Weird error message."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")), handle), 2.seconds
-    ).isInstanceOf[FailedNonRetryableExecutionHandle] shouldBe true
-    Await.result(jesBackend.executionResult(
-      Failed(10, Option("UnparsableInt: Even weirder error message."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")), handle), 2.seconds
-    ).isInstanceOf[FailedNonRetryableExecutionHandle] shouldBe true
-    Await.result(jesBackend.executionResult(
-      Failed(10, Option.empty, Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")), handle), 2.seconds
-    ).isInstanceOf[FailedNonRetryableExecutionHandle] shouldBe true
-    Await.result(jesBackend.executionResult(
-      Failed(10, Option("Operation canceled at"), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance")), handle), 2.seconds
-    ) shouldBe AbortedExecutionHandle
+
+    def checkFailedResult(errorCode: Int, errorMessage: Option[String]): ExecutionHandle = {
+      val failed =
+        Failed(errorCode, errorMessage, Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance"))
+      jesBackend.handleExecutionResult(failed, handle)
+    }
+
+    checkFailedResult(10, Option("15: Other type of error."))
+      .isInstanceOf[FailedNonRetryableExecutionHandle] shouldBe true
+    checkFailedResult(11, Option("14: Wrong errorCode.")).isInstanceOf[FailedNonRetryableExecutionHandle] shouldBe true
+    checkFailedResult(10, Option("Weird error message.")).isInstanceOf[FailedNonRetryableExecutionHandle] shouldBe true
+    checkFailedResult(10, Option("UnparsableInt: Even weirder error message."))
+      .isInstanceOf[FailedNonRetryableExecutionHandle] shouldBe true
+    checkFailedResult(10, None).isInstanceOf[FailedNonRetryableExecutionHandle] shouldBe true
+    checkFailedResult(10, Option("Operation canceled at")) shouldBe AbortedExecutionHandle
 
     actorRef.stop()
   }
@@ -316,7 +315,12 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
     val testActorRef = TestActorRef[TestableJesJobExecutionActor](
       props, s"TestableJesJobExecutionActor-${jobDescriptor.workflowDescriptor.id}")
 
-    val mappedInputs = jobDescriptor.fullyQualifiedInputs mapValues testActorRef.underlyingActor.gcsPathToLocal
+
+    def gcsPathToLocal(wdlValue: WdlValue): WdlValue = {
+      WdlFileMapper.mapWdlFiles(testActorRef.underlyingActor.mapCommandLineWdlFile)(wdlValue).get
+    }
+
+    val mappedInputs = jobDescriptor.fullyQualifiedInputs mapValues gcsPathToLocal
 
     mappedInputs(stringKey) match {
       case WdlString(v) => assert(v.equalsIgnoreCase(stringVal.value))
@@ -432,8 +436,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
       "strs" -> WdlArray(WdlArrayType(WdlStringType), Seq("A", "B", "C").map(WdlString))
     )
 
-    class TestJesExpressionFunctions extends JesExpressionFunctions(
-      List(mockPathBuilder), TestableCallContext) {
+    class TestJesExpressionFunctions extends JesExpressionFunctions(TestableStandardExpressionFunctionsParams) {
       override def write_lines(params: Seq[Try[WdlValue]]): Try[WdlFile] = {
         Success(WdlFile(s"gs://some/path/file.txt"))
       }
@@ -547,7 +550,11 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
     val testActorRef = TestActorRef[TestableJesJobExecutionActor](
       props, s"TestableJesJobExecutionActor-${jobDescriptor.workflowDescriptor.id}")
 
-    val result = outputValues map testActorRef.underlyingActor.wdlValueToGcsPath(jesOutputs)
+    def wdlValueToGcsPath(jesOutputs: Set[JesFileOutput])(wdlValue: WdlValue): WdlValue = {
+      WdlFileMapper.mapWdlFiles(testActorRef.underlyingActor.wdlFileToGcsPath(jesOutputs))(wdlValue).get
+    }
+
+    val result = outputValues map wdlValueToGcsPath(jesOutputs)
     result should have size 3
     result should contain(WdlFile("gs://path/to/file1"))
     result should contain(WdlArray(WdlArrayType(WdlFileType),
diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesCallPathsSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesCallPathsSpec.scala
index 1abf3f22f..2c508523b 100644
--- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesCallPathsSpec.scala
+++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesCallPathsSpec.scala
@@ -58,8 +58,10 @@ class JesCallPathsSpec extends TestKitSuite with FlatSpecLike with Matchers with
     val callPaths = JesJobPaths(jobDescriptorKey, workflowDescriptor, jesConfiguration)
     callPaths.callContext.root.toRealString should
       be(s"gs://my-cromwell-workflows-bucket/wf_hello/${workflowDescriptor.id}/call-hello")
-    callPaths.callContext.stdout should be("hello-stdout.log")
-    callPaths.callContext.stderr should be("hello-stderr.log")
+    callPaths.callContext.stdout should
+      be(s"gs://my-cromwell-workflows-bucket/wf_hello/${workflowDescriptor.id}/call-hello/hello-stdout.log")
+    callPaths.callContext.stderr should
+      be(s"gs://my-cromwell-workflows-bucket/wf_hello/${workflowDescriptor.id}/call-hello/hello-stderr.log")
   }
 
 }
diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala
index d810e7f8a..2fa9f6bd2 100644
--- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala
+++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala
@@ -2,12 +2,14 @@ package cromwell.backend.impl.jes
 
 import java.util.UUID
 
+import akka.actor.Props
 import akka.testkit._
 import com.typesafe.config.{Config, ConfigFactory}
 import cromwell.backend.BackendWorkflowInitializationActor.{InitializationFailed, InitializationSuccess, Initialize}
 import cromwell.backend.async.RuntimeAttributeValidationFailures
 import cromwell.backend.impl.jes.authentication.GcsLocalizing
 import cromwell.backend.{BackendConfigurationDescriptor, BackendSpec, BackendWorkflowDescriptor}
+import cromwell.core.Dispatcher.BackendDispatcher
 import cromwell.core.Tags.IntegrationTest
 import cromwell.core.logging.LoggingTest._
 import cromwell.core.{TestKitSuite, WorkflowOptions}
@@ -139,8 +141,16 @@ class JesInitializationActorSpec extends TestKitSuite("JesInitializationActorSpe
 
   val refreshTokenConfig: Config = ConfigFactory.parseString(refreshTokenConfigTemplate)
 
+  private def getJesBackendProps(workflowDescriptor: BackendWorkflowDescriptor,
+                                 calls: Set[TaskCall],
+                                 jesConfiguration: JesConfiguration): Props = {
+    val params = JesInitializationActorParams(workflowDescriptor, calls, jesConfiguration, emptyActor)
+    Props(new JesInitializationActor(params)).withDispatcher(BackendDispatcher)
+  }
+
   private def getJesBackend(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[TaskCall], conf: BackendConfigurationDescriptor) = {
-    system.actorOf(JesInitializationActor.props(workflowDescriptor, calls, new JesConfiguration(conf), emptyActor))
+    val props = getJesBackendProps(workflowDescriptor, calls, new JesConfiguration(conf))
+    system.actorOf(props, "TestableJesInitializationActor-" + UUID.randomUUID)
   }
 
   behavior of "JesInitializationActor"
@@ -192,7 +202,7 @@ class JesInitializationActorSpec extends TestKitSuite("JesInitializationActorSpe
     val jesConfiguration = new JesConfiguration(backendConfigurationDescriptor)
 
     val actorRef = TestActorRef[JesInitializationActor](
-      JesInitializationActor.props(workflowDescriptor, calls, jesConfiguration, emptyActor),
+      getJesBackendProps(workflowDescriptor, calls, jesConfiguration),
       "TestableJesInitializationActor-" + UUID.randomUUID)
     TestingBits(actorRef, jesConfiguration)
   }
diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesRuntimeAttributesSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesRuntimeAttributesSpec.scala
index cb18aa907..23ab89ed1 100644
--- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesRuntimeAttributesSpec.scala
+++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesRuntimeAttributesSpec.scala
@@ -195,11 +195,8 @@ class JesRuntimeAttributesSpec extends WordSpecLike with Matchers with Mockito {
                                                            expectedRuntimeAttributes: JesRuntimeAttributes,
                                                            workflowOptions: WorkflowOptions = emptyWorkflowOptions,
                                                            defaultZones: NonEmptyList[String] = defaultZones): Unit = {
-    val withDefaults = RuntimeAttributeDefinition.addDefaultsToAttributes(
-      staticRuntimeAttributeDefinitions, workflowOptions) _
     try {
-      val actualRuntimeAttributes =
-        JesRuntimeAttributes(withDefaults(runtimeAttributes), NOPLogger.NOP_LOGGER, jesConfiguration)
+      val actualRuntimeAttributes = toJesRuntimeAttributes(runtimeAttributes, workflowOptions, jesConfiguration)
       assert(actualRuntimeAttributes == expectedRuntimeAttributes)
     } catch {
       case ex: RuntimeException => fail(s"Exception was not expected but received: ${ex.getMessage}")
@@ -208,17 +205,25 @@ class JesRuntimeAttributesSpec extends WordSpecLike with Matchers with Mockito {
   }
 
   private def assertJesRuntimeAttributesFailedCreation(runtimeAttributes: Map[String, WdlValue], exMsg: String, workflowOptions: WorkflowOptions = emptyWorkflowOptions): Unit = {
-    val withDefaults = RuntimeAttributeDefinition.addDefaultsToAttributes(
-      staticRuntimeAttributeDefinitions, workflowOptions) _
     try {
-      JesRuntimeAttributes(withDefaults(runtimeAttributes), NOPLogger.NOP_LOGGER, jesConfiguration)
-      fail("A RuntimeException was expected.")
+      toJesRuntimeAttributes(runtimeAttributes, workflowOptions, jesConfiguration)
+      fail(s"A RuntimeException was expected with message: $exMsg")
     } catch {
       case ex: RuntimeException => assert(ex.getMessage.contains(exMsg))
     }
     ()
   }
 
+  private def toJesRuntimeAttributes(runtimeAttributes: Map[String, WdlValue],
+                                     workflowOptions: WorkflowOptions,
+                                     jesConfiguration: JesConfiguration): JesRuntimeAttributes = {
+    val runtimeAttributesBuilder = JesRuntimeAttributes.runtimeAttributesBuilder(jesConfiguration)
+    val defaultedAttributes = RuntimeAttributeDefinition.addDefaultsToAttributes(
+      staticRuntimeAttributeDefinitions, workflowOptions)(runtimeAttributes)
+    val validatedRuntimeAttributes = runtimeAttributesBuilder.build(defaultedAttributes, NOPLogger.NOP_LOGGER)
+    JesRuntimeAttributes(validatedRuntimeAttributes)
+  }
+
   private val emptyWorkflowOptions = WorkflowOptions.fromMap(Map.empty).get
   private val defaultZones = NonEmptyList.of("us-central1-b", "us-central1-a")
   private val jesConfiguration = {
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendLifecycleActorFactory.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendLifecycleActorFactory.scala
index 78060285d..ca17199d6 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendLifecycleActorFactory.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendLifecycleActorFactory.scala
@@ -22,9 +22,9 @@ class ConfigBackendLifecycleActorFactory(name: String, val configurationDescript
     configurationDescriptor.backendConfig.as[Option[Config]]("filesystems.local.caching") map ConfigHashingStrategy.apply getOrElse ConfigHashingStrategy.defaultStrategy
   }
 
-  override def initializationActorClass: Class[ConfigInitializationActor] = classOf[ConfigInitializationActor]
+  override lazy val initializationActorClass: Class[ConfigInitializationActor] = classOf[ConfigInitializationActor]
 
-  override def asyncExecutionActorClass: Class[_ <: ConfigAsyncJobExecutionActor] = {
+  override lazy val asyncExecutionActorClass: Class[_ <: ConfigAsyncJobExecutionActor] = {
     val runInBackground = configurationDescriptor.backendConfig.as[Option[Boolean]](RunInBackgroundConfig).getOrElse(false)
     if (runInBackground)
       classOf[BackgroundConfigAsyncJobExecutionActor]
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigInitializationActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigInitializationActor.scala
index c78013582..34f93b0ff 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigInitializationActor.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigInitializationActor.scala
@@ -2,7 +2,7 @@ package cromwell.backend.impl.sfs.config
 
 import cromwell.backend.io.WorkflowPaths
 import cromwell.backend.sfs._
-import cromwell.backend.standard.{DefaultInitializationActorParams, StandardInitializationData, StandardValidatedRuntimeAttributesBuilder}
+import cromwell.backend.standard.{StandardInitializationActorParams, StandardInitializationData, StandardValidatedRuntimeAttributesBuilder}
 import wdl4s.WdlNamespace
 
 /**
@@ -21,7 +21,8 @@ class ConfigInitializationData
   runtimeAttributesBuilder: StandardValidatedRuntimeAttributesBuilder,
   val declarationValidations: Seq[DeclarationValidation],
   val wdlNamespace: WdlNamespace)
-  extends StandardInitializationData(workflowPaths, runtimeAttributesBuilder)
+  extends StandardInitializationData(workflowPaths, runtimeAttributesBuilder,
+    classOf[SharedFileSystemExpressionFunctions])
 
 /**
   * Extends the SharedFileSystemInitializationActor to create an instance of the ConfigInitializationData.
@@ -30,7 +31,7 @@ class ConfigInitializationData
   *
   * @param params Parameters to create an initialization actor.
   */
-class ConfigInitializationActor(params: DefaultInitializationActorParams)
+class ConfigInitializationActor(params: StandardInitializationActorParams)
   extends SharedFileSystemInitializationActor(params) {
 
   lazy val configWdlNamespace = new ConfigWdlNamespace(params.configurationDescriptor.backendConfig)
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/BackgroundAsyncJobExecutionActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/BackgroundAsyncJobExecutionActor.scala
index 8367ffb2a..5d5ff1eb9 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/BackgroundAsyncJobExecutionActor.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/BackgroundAsyncJobExecutionActor.scala
@@ -8,48 +8,34 @@ import cromwell.core.path.PathFactory._
 
 trait BackgroundAsyncJobExecutionActor extends SharedFileSystemAsyncJobExecutionActor {
 
-  override def makeProcessRunner(): ProcessRunner = {
-    val backgroundScript = pathPlusSuffix(jobPaths.script, "background")
-    writeBackgroundScript(backgroundScript, processArgs.argv.mkString("'", "' '", "'"))
-    val stdout = pathPlusSuffix(jobPaths.stdout, "background")
-    val stderr = pathPlusSuffix(jobPaths.stderr, "background")
-    val argv = Seq("/bin/bash", backgroundScript)
-    new ProcessRunner(argv, stdout.path, stderr.path)
-  }
-
-  private def writeBackgroundScript(backgroundScript: File, backgroundCommand: String): Unit = {
-    /*
-    Run the `backgroundCommand` in the background. Redirect the stdout and stderr to the appropriate files. While not
-    necessary, mark the job as not receiving any stdin by pointing it at /dev/null.
+  lazy val backgroundScript: File = pathPlusSuffix(jobPaths.script, "background")
 
-    If the `backgroundCommand` errors for some reason, put a "-1" into the rc file.
-
-    Finally, run all of the above in the bash background, and return the PID of the backgrounded command.
+  override def writeScriptContents(): Unit = {
+    super.writeScriptContents()
+    writeBackgroundScriptContents()
+  }
 
-    bashism | english
-    --------|--------------------------------------------------------------------------
-       >    | redirect stdout to <file>
-       2>   | redirect stderr to <file>
-       <    | redirect stdin from <file>
-       ||   | if the previous command fails, then run the following command
-       >    | redirect stdout to <file>
-       &    | send the entire compound command, including the || to the background
-       $!   | a variable containing the previous background command's process id (PID)
-     */
+  /**
+    * Run the command via bash in the background, and echo the PID.
+    */
+  private def writeBackgroundScriptContents(): Unit = {
+    val backgroundCommand = redirectOutputs(processArgs.argv.mkString("'", "' '", "'"))
+    // $! contains the previous background command's process id (PID)
     backgroundScript.write(
       s"""|#!/bin/bash
-          |$backgroundCommand \\
-          |  > ${jobPaths.stdout} \\
-          |  2> ${jobPaths.stderr} \\
-          |  < /dev/null \\
-          |  || echo -1 \\
-          |  > ${jobPaths.returnCode} \\
-          |  &
+          |BACKGROUND_COMMAND &
           |echo $$!
-          |""".stripMargin)
+          |""".stripMargin.replace("BACKGROUND_COMMAND", backgroundCommand))
     ()
   }
 
+  override def makeProcessRunner(): ProcessRunner = {
+    val stdout = pathPlusSuffix(jobPaths.stdout, "background")
+    val stderr = pathPlusSuffix(jobPaths.stderr, "background")
+    val argv = Seq("/bin/bash", backgroundScript)
+    new ProcessRunner(argv, stdout.path, stderr.path)
+  }
+
   override def getJob(exitValue: Int, stdout: Path, stderr: Path): StandardAsyncJob = {
     val pid = File(stdout).contentAsString.stripLineEnd
     StandardAsyncJob(pid)
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala
index e72c201ae..f785872a7 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala
@@ -8,13 +8,13 @@ import cats.syntax.functor._
 import com.typesafe.config.Config
 import com.typesafe.scalalogging.StrictLogging
 import cromwell.backend.io.JobPaths
+import cromwell.backend.wdl.WdlFileMapper
 import cromwell.core.CromwellFatalExceptionMarker
 import cromwell.core.path.FileImplicits._
 import cromwell.core.path.PathFactory
 import cromwell.core.path.PathFactory._
 import lenthall.util.TryUtil
 import wdl4s.EvaluatedTaskInputs
-import wdl4s.types.{WdlArrayType, WdlMapType}
 import wdl4s.values._
 
 import scala.collection.JavaConverters._
@@ -138,18 +138,15 @@ trait SharedFileSystem extends PathFactory {
   }
 
   def outputMapper(job: JobPaths)(wdlValue: WdlValue): Try[WdlValue] = {
-    wdlValue match {
+    WdlFileMapper.mapWdlFiles(mapJobWdlFile(job))(wdlValue)
+  }
+
+  def mapJobWdlFile(job: JobPaths)(wdlFile: WdlFile): WdlFile = {
+    wdlFile match {
       case fileNotFound: WdlFile if !hostAbsoluteFilePath(job.callExecutionRoot, fileNotFound.valueString).exists =>
-        Failure(new RuntimeException("Could not process output, file not found: " +
-          s"${hostAbsoluteFilePath(job.callExecutionRoot, fileNotFound.valueString).pathAsString}"))
-      case file: WdlFile => Try(WdlFile(hostAbsoluteFilePath(job.callExecutionRoot, file.valueString).pathAsString))
-      case array: WdlArray =>
-        val mappedArray = array.value map outputMapper(job)
-        TryUtil.sequence(mappedArray) map { WdlArray(array.wdlType, _) }
-      case map: WdlMap =>
-        val mappedMap = map.value mapValues outputMapper(job)
-        TryUtil.sequenceMap(mappedMap) map { WdlMap(map.wdlType, _) }
-      case other => Success(other)
+        throw new RuntimeException("Could not process output, file not found: " +
+          s"${hostAbsoluteFilePath(job.callExecutionRoot, fileNotFound.valueString).pathAsString}")
+      case _ => WdlFile(hostAbsoluteFilePath(job.callExecutionRoot, wdlFile.valueString).pathAsString)
     }
   }
 
@@ -162,6 +159,15 @@ trait SharedFileSystem extends PathFactory {
    * been performed for this `Backend` implementation.
    */
   def localizeInputs(inputsRoot: Path, docker: Boolean)(inputs: EvaluatedTaskInputs): Try[EvaluatedTaskInputs] = {
+    TryUtil.sequenceMap(
+      inputs mapValues WdlFileMapper.mapWdlFiles(localizeWdlFile(inputsRoot, docker)),
+      "Failures during localization"
+    ) recoverWith {
+      case e => Failure(new IOException(e.getMessage) with CromwellFatalExceptionMarker)
+    }
+  }
+
+  def localizeWdlFile(inputsRoot: Path, docker: Boolean)(value: WdlFile): WdlFile = {
     val strategies = if (docker) DockerLocalizers else Localizers
 
     // Use URI to identify protocol scheme and strip it out
@@ -192,71 +198,23 @@ trait SharedFileSystem extends PathFactory {
     }
 
     // Optional function to adjust the path to "docker path" if the call runs in docker
-    val localizeFunction = localizeWdlValue(toCallPath, strategies.toStream) _
-    val localizedValues = inputs.toSeq map {
-      case (declaration, value) => localizeFunction(value) map { declaration -> _ }
-    }
-
-    TryUtil.sequence(localizedValues, "Failures during localization").map(_.toMap) recoverWith {
-      case e => Failure(new IOException(e.getMessage) with CromwellFatalExceptionMarker)
-    }
+    localizeWdlFile(toCallPath _, strategies.toStream)(value)
   }
 
   /**
-   * Try to localize a WdlValue if it is or contains one or more WdlFiles.
-   *
-   * @param toDestPath function specifying how to generate the destination path from the source path
-   * @param strategies strategies to use for localization
-   * @param wdlValue WdlValue to localize
-   * @return localized wdlValue
-   */
-  private def localizeWdlValue(toDestPath: (String => Try[PairOfFiles]), strategies: Stream[DuplicationStrategy])
-                              (wdlValue: WdlValue): Try[WdlValue] = {
-
-    def adjustArray(t: WdlArrayType, inputArray: Seq[WdlValue]): Try[WdlArray] = {
-      val tryAdjust = inputArray map localizeWdlValue(toDestPath, strategies)
-
-      TryUtil.sequence(tryAdjust, s"Failed to localize files in input Array ${wdlValue.valueString}") map { adjusted =>
-        new WdlArray(t, adjusted)
-      }
-    }
-
-    def adjustMap(t: WdlMapType, inputMap: Map[WdlValue, WdlValue]): Try[WdlMap] = {
-      val tryAdjust = inputMap mapValues { localizeWdlValue(toDestPath, strategies) }
-
-      TryUtil.sequenceMap(tryAdjust, s"Failed to localize files in input Map ${wdlValue.valueString}") map { adjusted =>
-        new WdlMap(t, adjusted)
-      }
-    }
-
-    def adjustPair(p: WdlPair): Try[WdlPair] = {
-      for {
-        l <- localizeWdlValue(toDestPath, strategies)(p.left)
-        r <- localizeWdlValue(toDestPath, strategies)(p.right)
-      } yield WdlPair(l, r)
-    }
-
-    def adjustOptional(v: WdlOptionalValue): Try[WdlOptionalValue] = {
-      v.value map localizeWdlValue(toDestPath, strategies) match {
-        case Some(Success(adjusted)) => Success(WdlOptionalValue(v.innerType, Option(adjusted)))
-        case Some(Failure(t)) => Failure(t)
-        case None => Success(v)
-      }
-    }
-
-    def adjustFile(path: String) = {
-      toDestPath(path) flatMap {
-        case PairOfFiles(src, dst) => duplicate("localize", src, dst, strategies) map { _ => WdlFile(dst.toString) }
-      }
-    }
-
-    wdlValue match {
-      case wdlFile: WdlFile => adjustFile(wdlFile.value)
-      case WdlArray(t, values) => adjustArray(t, values)
-      case WdlMap(t, values) => adjustMap(t, values)
-      case pair: WdlPair => adjustPair(pair)
-      case opt: WdlOptionalValue => adjustOptional(opt)
-      case x => Success(x)
+    * Try to localize a WdlFile.
+    *
+    * @param toDestPath function specifying how to generate the destination path from the source path
+    * @param strategies strategies to use for localization
+    * @param wdlFile WdlFile to localize
+    * @return localized wdl file
+    */
+  private def localizeWdlFile(toDestPath: (String => Try[PairOfFiles]), strategies: Stream[DuplicationStrategy])
+                             (wdlFile: WdlFile): WdlFile = {
+    val path = wdlFile.value
+    val result = toDestPath(path) flatMap {
+      case PairOfFiles(src, dst) => duplicate("localize", src, dst, strategies) map { _ => WdlFile(dst.toString) }
     }
+    result.get
   }
 }
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala
index 5cdf0e0a7..db97805cd 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala
@@ -4,20 +4,17 @@ import java.nio.file.{FileAlreadyExistsException, Path}
 
 import better.files._
 import cromwell.backend._
-import cromwell.backend.async.{ExecutionHandle, FailedNonRetryableExecutionHandle, PendingExecutionHandle, SuccessfulExecutionHandle}
+import cromwell.backend.async.{ExecutionHandle, FailedNonRetryableExecutionHandle, PendingExecutionHandle}
 import cromwell.backend.io.JobPathsWithDocker
-import cromwell.backend.standard.{StandardAsyncExecutionActor, StandardAsyncJob, StandardInitializationData}
+import cromwell.backend.standard.{StandardAsyncExecutionActor, StandardAsyncJob}
 import cromwell.backend.validation._
-import cromwell.backend.wdl.OutputEvaluator
+import cromwell.core.path.DefaultPathBuilder
 import cromwell.core.path.FileImplicits._
 import cromwell.core.path.PathFactory._
-import cromwell.core.path.{DefaultPathBuilder, PathBuilder}
 import cromwell.core.retry.SimpleExponentialBackoff
-import wdl4s.EvaluatedTaskInputs
-import wdl4s.values.{WdlArray, WdlFile, WdlGlobFile, WdlMap, WdlOptionalValue, WdlPair, WdlValue}
+import wdl4s.values.WdlFile
 
 import scala.concurrent.duration._
-import scala.util.{Failure, Success, Try}
 
 case class SharedFileSystemRunStatus(returnCodeFileExists: Boolean) {
   override def toString: String = if (returnCodeFileExists) "Done" else "WaitingForReturnCodeFile"
@@ -100,42 +97,33 @@ trait SharedFileSystemAsyncJobExecutionActor
 
   lazy val jobPathsWithDocker: JobPathsWithDocker = jobPaths.asInstanceOf[JobPathsWithDocker]
 
-  def toUnixPath(docker: Boolean)(path: WdlValue): WdlValue = {
-    val toUnixPathFunc: WdlValue => WdlValue = toUnixPath(docker) _
-    path match {
-      case _: WdlFile =>
-        val cleanPath = DefaultPathBuilder.build(path.valueString).get
-        WdlFile(if (docker) jobPathsWithDocker.toDockerPath(cleanPath).toString else cleanPath.toString)
-      case array: WdlArray => WdlArray(array.wdlType, array.value map toUnixPathFunc)
-      case map: WdlMap => WdlMap(map.wdlType, map.value mapValues toUnixPathFunc)
-      case pair: WdlPair => WdlPair(toUnixPathFunc(pair.left), toUnixPathFunc(pair.right))
-      case opt: WdlOptionalValue => WdlOptionalValue(opt.innerType, opt.value map toUnixPathFunc)
-      case wdlValue => wdlValue
-    }
-  }
-
   def jobName: String = s"cromwell_${jobDescriptor.workflowDescriptor.id.shortString}_${jobDescriptor.call.unqualifiedName}"
 
-  lazy val pathBuilders: List[PathBuilder] = StandardInitializationData.pathBuilders(backendInitializationDataOption)
-  lazy val backendEngineFunctions = SharedFileSystemExpressionFunctions(jobPaths, pathBuilders)
-
   lazy val isDockerRun: Boolean = RuntimeAttributesValidation.extractOption(
     DockerValidation.instance, validatedRuntimeAttributes).isDefined
 
-  override lazy val commandLineFunctions: SharedFileSystemExpressionFunctions = backendEngineFunctions
+  /**
+    * Localizes the file, run outside of docker.
+    */
+  override def preProcessWdlFile(wdlFile: WdlFile): WdlFile = {
+    sharedFileSystem.localizeWdlFile(jobPathsWithDocker.callInputsRoot, isDockerRun)(wdlFile)
+  }
 
-  override lazy val commandLinePreProcessor: (EvaluatedTaskInputs) => Try[EvaluatedTaskInputs] =
-    sharedFileSystem.localizeInputs(jobPathsWithDocker.callInputsRoot, isDockerRun)
+  /**
+    * Returns the paths to the file, inside of docker.
+    */
+  override def mapCommandLineWdlFile(wdlFile: WdlFile): WdlFile = {
+    val cleanPath = DefaultPathBuilder.build(wdlFile.valueString).get
+    WdlFile(if (isDockerRun) jobPathsWithDocker.toDockerPath(cleanPath).toString else cleanPath.toString)
+  }
 
-  override lazy val commandLineValueMapper: (WdlValue) => WdlValue = toUnixPath(isDockerRun)
+  override lazy val commandDirectory: Path = {
+    if (isDockerRun) jobPathsWithDocker.callExecutionDockerRoot else jobPaths.callExecutionRoot
+  }
 
   override def execute(): ExecutionHandle = {
-    val script = instantiatedCommand
-    jobLogger.info(s"`$script`")
     File(jobPaths.callExecutionRoot).createPermissionedDirectories()
-    val cwd = if (isDockerRun) jobPathsWithDocker.callExecutionDockerRoot else jobPaths.callExecutionRoot
-    writeScript(script, cwd, backendEngineFunctions.findGlobOutputs(call, jobDescriptor))
-    jobLogger.info(s"command: $processArgs")
+    writeScriptContents()
     val runner = makeProcessRunner()
     val exitValue = runner.run()
     if (exitValue != 0) {
@@ -147,6 +135,11 @@ trait SharedFileSystemAsyncJobExecutionActor
     }
   }
 
+  def writeScriptContents(): Unit = {
+    File(jobPaths.script).write(commandScriptContents)
+    ()
+  }
+
   /**
     * Creates a script to submit the script for asynchronous processing. The default implementation assumes the
     * processArgs already runs the script asynchronously. If not, mix in the `BackgroundAsyncJobExecutionActor` that
@@ -160,47 +153,6 @@ trait SharedFileSystemAsyncJobExecutionActor
     new ProcessRunner(processArgs.argv, stdout.path, stderr.path)
   }
 
-  /**
-    * Writes the script file containing the user's command from the WDL as well
-    * as some extra shell code for monitoring jobs
-    */
-  private def writeScript(instantiatedCommand: String, cwd: Path, globFiles: Set[WdlGlobFile]) = {
-    val rcPath = if (isDockerRun) jobPathsWithDocker.toDockerPath(jobPaths.returnCode) else jobPaths.returnCode
-    val rcTmpPath = pathPlusSuffix(rcPath, "tmp").path
-
-    def globManipulation(globFile: WdlGlobFile) = {
-
-      // TODO: Move glob list and directory generation into trait GlobFunctions? There is already a globPath using callContext
-      val globDir = backendEngineFunctions.globName(globFile.value)
-      val globDirectory = File(cwd)./(globDir)
-      val globList = File(cwd)./(s"$globDir.list")
-
-      s"""|mkdir $globDirectory
-          |( ln -L ${globFile.value} $globDirectory 2> /dev/null ) || ( ln ${globFile.value} $globDirectory )
-          |ls -1 $globDirectory > $globList
-          |""".stripMargin
-    }
-
-    val globManipulations = globFiles.map(globManipulation).mkString("\n")
-
-    val scriptBody =
-      s"""|#!/bin/sh
-          |umask 0000
-          |(
-          |cd $cwd
-          |INSTANTIATED_COMMAND
-          |)
-          |echo $$? > $rcTmpPath
-          |(
-          |cd $cwd
-          |$globManipulations
-          |)
-          |mv $rcTmpPath $rcPath
-          |""".stripMargin.replace("INSTANTIATED_COMMAND", instantiatedCommand)
-
-    File(jobPaths.script).write(scriptBody)
-  }
-
   override def recover(job: StandardAsyncJob): ExecutionHandle = {
     // To avoid race conditions, check for the rc file after checking if the job is alive.
     if (isAlive(job) || File(jobPaths.returnCode).exists) {
@@ -249,14 +201,7 @@ trait SharedFileSystemAsyncJobExecutionActor
     runStatus.returnCodeFileExists
   }
 
-  override def handleExecutionSuccess(runStatus: StandardAsyncRunStatus, handle: StandardAsyncPendingExecutionHandle,
-                                      returnCode: Int): ExecutionHandle = {
-    val outputsTry =
-      OutputEvaluator.evaluateOutputs(jobDescriptor, backendEngineFunctions, sharedFileSystem.outputMapper(jobPaths))
-    outputsTry match {
-      case Success(outputs) => SuccessfulExecutionHandle(outputs, returnCode, jobPaths.detritusPaths, Seq.empty)
-      case Failure(throwable) => FailedNonRetryableExecutionHandle(throwable, Option(returnCode))
-    }
+  override def mapOutputWdlFile(wdlFile: WdlFile): WdlFile = {
+    sharedFileSystem.mapJobWdlFile(jobPaths)(wdlFile)
   }
-
 }
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemBackendLifecycleActorFactory.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemBackendLifecycleActorFactory.scala
index 150f8b199..85bd0f1d9 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemBackendLifecycleActorFactory.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemBackendLifecycleActorFactory.scala
@@ -1,8 +1,6 @@
 package cromwell.backend.sfs
 
-import cromwell.backend._
 import cromwell.backend.standard._
-import wdl4s.expression.WdlStandardLibraryFunctions
 
 /**
   * A factory that can be extended for any shared file system implementation.
@@ -13,14 +11,7 @@ trait SharedFileSystemBackendLifecycleActorFactory extends StandardLifecycleActo
 
   override def jobIdKey: String = SharedFileSystemAsyncJobExecutionActor.JobIdKey
 
-  override lazy val standardCacheHitCopyingActorOption: Option[Class[_ <: StandardCacheHitCopyingActor]] = {
+  override lazy val cacheHitCopyingActorClassOption: Option[Class[_ <: StandardCacheHitCopyingActor]] = {
     Option(classOf[SharedFileSystemCacheHitCopyingActor])
   }
-
-  override def expressionLanguageFunctions(workflowDescriptor: BackendWorkflowDescriptor,
-                                           jobKey: BackendJobDescriptorKey,
-                                           initializationData: Option[BackendInitializationData]):
-  WdlStandardLibraryFunctions = {
-    SharedFileSystemExpressionFunctions(workflowDescriptor, configurationDescriptor, jobKey, initializationData)
-  }
 }
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemCacheHitCopyingActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemCacheHitCopyingActor.scala
index f6a8ad65c..7c6ba612f 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemCacheHitCopyingActor.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemCacheHitCopyingActor.scala
@@ -4,8 +4,8 @@ import java.nio.file.Path
 
 import cromwell.backend.standard.{StandardCacheHitCopyingActor, StandardCacheHitCopyingActorParams}
 
-class SharedFileSystemCacheHitCopyingActor(override val standardParams: StandardCacheHitCopyingActorParams)
-  extends StandardCacheHitCopyingActor with SharedFileSystemJobCachingActorHelper {
+class SharedFileSystemCacheHitCopyingActor(standardParams: StandardCacheHitCopyingActorParams)
+  extends StandardCacheHitCopyingActor(standardParams) with SharedFileSystemJobCachingActorHelper {
   override protected def duplicate(source: Path, destination: Path): Unit = {
     // -Ywarn-value-discard
     sharedFileSystem.cacheCopy(source, destination)
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemExpressionFunctions.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemExpressionFunctions.scala
index 0250c7a10..b3513bd97 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemExpressionFunctions.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemExpressionFunctions.scala
@@ -3,15 +3,9 @@ package cromwell.backend.sfs
 import java.nio.file.Path
 
 import cromwell.backend.io._
-import cromwell.backend.standard.StandardInitializationData
-import cromwell.backend.wdl._
-import cromwell.backend._
+import cromwell.backend.standard.{DefaultStandardExpressionFunctionsParams, StandardExpressionFunctions, StandardExpressionFunctionsParams}
 import cromwell.core.CallContext
 import cromwell.core.path.PathBuilder
-import wdl4s.expression.PureStandardLibraryFunctionsLike
-import wdl4s.values._
-
-import scala.util.{Success, Try}
 
 object SharedFileSystemExpressionFunctions {
   private val LocalFileSystemScheme = "file"
@@ -19,57 +13,21 @@ object SharedFileSystemExpressionFunctions {
   def isLocalPath(path: Path): Boolean =
     path.toUri.getScheme == SharedFileSystemExpressionFunctions.LocalFileSystemScheme
 
-  def apply(workflowDescriptor: BackendWorkflowDescriptor,
-            jobKey: BackendJobDescriptorKey,
-            configurationDescriptor: BackendConfigurationDescriptor,
-            pathBuilders: List[PathBuilder]): SharedFileSystemExpressionFunctions = {
-    val jobPaths = new JobPathsWithDocker(jobKey, workflowDescriptor, configurationDescriptor.backendConfig)
-    val callContext = CallContext(
-      jobPaths.callExecutionRoot,
-      jobPaths.stdout.toString,
-      jobPaths.stderr.toString
-    )
-    new SharedFileSystemExpressionFunctions(pathBuilders, callContext)
-  }
-
   def apply(jobPaths: JobPaths, pathBuilders: List[PathBuilder]): SharedFileSystemExpressionFunctions = {
-    val callContext = CallContext(
-      jobPaths.callExecutionRoot,
-      jobPaths.stdout.toString,
-      jobPaths.stderr.toString
-    )
-    new SharedFileSystemExpressionFunctions(pathBuilders, callContext)
-  }
-
-  def apply(workflowDescriptor: BackendWorkflowDescriptor,
-            configurationDescriptor: BackendConfigurationDescriptor,
-            jobKey: BackendJobDescriptorKey,
-            initializationData: Option[BackendInitializationData]): SharedFileSystemExpressionFunctions = {
-    val jobPaths = new JobPathsWithDocker(jobKey, workflowDescriptor, configurationDescriptor.backendConfig)
-    val callContext = CallContext(
-      jobPaths.callExecutionRoot,
-      jobPaths.stdout.toString,
-      jobPaths.stderr.toString
-    )
-
-    new SharedFileSystemExpressionFunctions(StandardInitializationData.pathBuilders(initializationData), callContext)
+    new SharedFileSystemExpressionFunctions(pathBuilders, jobPaths.callContext)
   }
 }
 
-class SharedFileSystemExpressionFunctions(override val pathBuilders: List[PathBuilder],
-                                          context: CallContext
-                                 ) extends PureStandardLibraryFunctionsLike with ReadLikeFunctions with WriteFunctions with GlobFunctions {
-  import SharedFileSystemExpressionFunctions._
-
-  def callContext: CallContext = context
-
-  override def writeTempFile(path: String, prefix: String, suffix: String, content: String): String = super[WriteFunctions].writeTempFile(path, prefix, suffix, content)
+class SharedFileSystemExpressionFunctions(standardParams: StandardExpressionFunctionsParams)
+  extends StandardExpressionFunctions(standardParams) {
 
-  override val writeDirectory: Path = context.root
-
-  override def stdout(params: Seq[Try[WdlValue]]) = Success(WdlFile(context.stdout))
-  override def stderr(params: Seq[Try[WdlValue]]) = Success(WdlFile(context.stderr))
+  def this(pathBuilders: List[PathBuilder], callContext: CallContext) = {
+    this(DefaultStandardExpressionFunctionsParams(pathBuilders, callContext))
+  }
 
   override def postMapping(path: Path): Path =
-    if (!path.isAbsolute && isLocalPath(path)) context.root.resolve(path) else path
+    if (!path.isAbsolute && SharedFileSystemExpressionFunctions.isLocalPath(path))
+      callContext.root.resolve(path)
+    else
+      path
 }
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemInitializationActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemInitializationActor.scala
index 4249c7bb4..ba0c74fb4 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemInitializationActor.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemInitializationActor.scala
@@ -4,8 +4,9 @@ import better.files._
 import cats.data.Validated.{Invalid, Valid}
 import cromwell.backend.BackendInitializationData
 import cromwell.backend.io.WorkflowPaths
-import cromwell.backend.standard.{DefaultInitializationActorParams, StandardInitializationActor, StandardInitializationActorParams, StandardInitializationData}
+import cromwell.backend.standard.{StandardExpressionFunctions, StandardInitializationActor, StandardInitializationActorParams}
 import cromwell.backend.wfs.WorkflowPathBuilder
+import cromwell.core.path.FileImplicits._
 import cromwell.core.path.{DefaultPathBuilderFactory, PathBuilder, PathBuilderFactory}
 import cromwell.filesystems.gcs.{GcsPathBuilderFactory, GoogleConfiguration}
 import lenthall.exception.MessageAggregation
@@ -13,16 +14,9 @@ import net.ceedubs.ficus.Ficus._
 
 import scala.concurrent.Future
 import scala.util.Try
-import cromwell.core.path.FileImplicits._
-/**
-  * Initializes a shared file system actor factory and creates initialization data to pass to the execution actors.
-  *
-  * @param params Initialization parameters.
-  */
-class SharedFileSystemInitializationActor(params: DefaultInitializationActorParams)
-  extends StandardInitializationActor {
 
-  override val standardParams: StandardInitializationActorParams = params
+class SharedFileSystemInitializationActor(standardParams: StandardInitializationActorParams)
+  extends StandardInitializationActor(standardParams) {
 
   /**
     * If the backend sets a gcs authentication mode, try to create a PathBuilderFactory with it.
@@ -43,15 +37,14 @@ class SharedFileSystemInitializationActor(params: DefaultInitializationActorPara
   lazy val pathBuilderFactories: List[PathBuilderFactory] =
     List(gcsPathBuilderFactory, Option(DefaultPathBuilderFactory)).flatten
 
-  lazy val pathBuilders: List[PathBuilder] =
+  override lazy val pathBuilders: List[PathBuilder] =
     pathBuilderFactories map { _.withOptions(workflowDescriptor.workflowOptions)(context.system) }
 
-  val workflowPaths: WorkflowPaths =
+  override lazy val workflowPaths: WorkflowPaths =
     WorkflowPathBuilder.workflowPaths(configurationDescriptor, workflowDescriptor, pathBuilders)
 
-  def initializationData: StandardInitializationData = {
-    new StandardInitializationData(workflowPaths, runtimeAttributesBuilder)
-  }
+  override lazy val expressionFunctions: Class[_ <: StandardExpressionFunctions] =
+    classOf[SharedFileSystemExpressionFunctions]
 
   override def beforeAll(): Future[Option[BackendInitializationData]] = {
     Future.fromTry(Try {
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemJobCachingActorHelper.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemJobCachingActorHelper.scala
index ff78494d3..e43a6bf41 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemJobCachingActorHelper.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemJobCachingActorHelper.scala
@@ -2,7 +2,7 @@ package cromwell.backend.sfs
 
 import akka.actor.Actor
 import com.typesafe.config.{Config, ConfigFactory}
-import cromwell.backend.standard.{StandardCachingActorHelper, StandardInitializationData}
+import cromwell.backend.standard.StandardCachingActorHelper
 import cromwell.core.logging.JobLogging
 import cromwell.core.path.PathBuilder
 import net.ceedubs.ficus.Ficus._
@@ -11,9 +11,8 @@ trait SharedFileSystemJobCachingActorHelper extends StandardCachingActorHelper {
   this: Actor with JobLogging =>
 
   lazy val sharedFileSystem = new SharedFileSystem {
-    override val pathBuilders: List[PathBuilder] = {
-      StandardInitializationData.pathBuilders(backendInitializationDataOption)
-    }
+    override lazy val pathBuilders: List[PathBuilder] = standardInitializationData.workflowPaths.pathBuilders
+
     override lazy val sharedFileSystemConfig: Config = {
       configurationDescriptor.backendConfig.as[Option[Config]]("filesystems.local").getOrElse(ConfigFactory.empty())
     }
diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/TestLocalAsyncJobExecutionActor.scala b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/TestLocalAsyncJobExecutionActor.scala
index 2c2b9ab20..f57aa093d 100644
--- a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/TestLocalAsyncJobExecutionActor.scala
+++ b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/TestLocalAsyncJobExecutionActor.scala
@@ -34,7 +34,8 @@ object TestLocalAsyncJobExecutionActor {
     val emptyActor = system.actorOf(Props.empty)
     val workflowPaths = new WorkflowPathsWithDocker(jobDescriptor.workflowDescriptor, configurationDescriptor.backendConfig)
     val initializationData = new StandardInitializationData(workflowPaths,
-      StandardValidatedRuntimeAttributesBuilder.default.withValidation(DockerValidation.optional))
+      StandardValidatedRuntimeAttributesBuilder.default.withValidation(DockerValidation.optional),
+      classOf[SharedFileSystemExpressionFunctions])
     val asyncClass = classOf[TestLocalAsyncJobExecutionActor]
 
     val params = DefaultStandardSyncExecutionActorParams(SharedFileSystemAsyncJobExecutionActor.JobIdKey, emptyActor,

From ecc6f3739db1cf11c5be24dd564d0eb0925aa5ef Mon Sep 17 00:00:00 2001
From: Khalid Shakir <kshakir@broadinstitute.org>
Date: Mon, 6 Feb 2017 11:29:02 -0500
Subject: [PATCH 46/84] Wrapped java nio paths with our own paths. Updated path
 builder usage and added tests. NOTE: GcsPathBuilder.build().get.toString now
 returns "gs://bucket/path" instead of "/path".

---
 CHANGELOG.md                                       |   2 +
 .../cromwell/backend/BackendJobBreadCrumb.scala    |   3 +-
 .../backend/BackendJobExecutionActor.scala         |   3 +-
 .../backend/BackendLifecycleActorFactory.scala     |   3 +-
 .../cromwell/backend/async/ExecutionHandle.scala   |   5 +-
 .../cromwell/backend/async/ExecutionResult.scala   |   5 +-
 .../backend/async/KnownJobFailureException.scala   |   3 +-
 .../backend/callcaching/CacheHitDuplicating.scala  |   7 +-
 .../scala/cromwell/backend/io/GlobFunctions.scala  |  24 +-
 .../main/scala/cromwell/backend/io/JobPaths.scala  |   6 +-
 .../cromwell/backend/io/JobPathsWithDocker.scala   |   8 +-
 .../scala/cromwell/backend/io/WorkflowPaths.scala  |   4 +-
 .../backend/io/WorkflowPathsWithDocker.scala       |   6 +-
 .../standard/StandardAsyncExecutionActor.scala     |  26 +-
 .../standard/StandardCacheHitCopyingActor.scala    |   4 +-
 .../standard/StandardCachingActorHelper.scala      |   3 +-
 .../standard/StandardExpressionFunctions.scala     |   4 +-
 .../standard/StandardFinalizationActor.scala       |   4 +-
 .../standard/StandardLifecycleActorFactory.scala   |   5 +-
 .../cromwell/backend/wdl/ReadLikeFunctions.scala   |   9 +-
 .../scala/cromwell/backend/wdl/WdlFileMapper.scala |  17 +-
 .../cromwell/backend/wdl/WriteFunctions.scala      |  10 +-
 .../scala/cromwell/backend/io/JobPathsSpec.scala   |  63 +--
 .../cromwell/backend/io/WorkflowPathsSpec.scala    |  15 +-
 core/src/main/scala/cromwell/core/core.scala       |   3 +-
 .../cromwell/core/logging/WorkflowLogger.scala     |  16 +-
 .../cromwell/core/path/BetterFileMethods.scala     | 588 +++++++++++++++++++++
 .../cromwell/core/path/DefaultPathBuilder.scala    |  45 +-
 .../cromwell/core/path/EvenBetterPathMethods.scala |  74 +++
 .../scala/cromwell/core/path/FileImplicits.scala   |  36 --
 .../cromwell/core/path/MappedPathBuilder.scala     |  31 ++
 .../scala/cromwell/core/path/NioPathMethods.scala  |  69 +++
 .../main/scala/cromwell/core/path/Obsolete.scala   |  68 +++
 .../scala/cromwell/core/path/PathBuilder.scala     | 114 +++-
 .../main/scala/cromwell/core/path/PathCopier.scala |  31 +-
 .../scala/cromwell/core/path/PathFactory.scala     |  26 +-
 .../scala/cromwell/core/path/PathImplicits.scala   |  17 -
 .../cromwell/core/path/PathObjectMethods.scala     |  19 +
 .../main/scala/cromwell/core/path/PathWriter.scala |   7 +-
 .../main/scala/cromwell/core/path/package.scala    |   5 +
 core/src/main/scala/cromwell/util/FileUtil.scala   |  39 --
 .../core/path/DefaultPathBuilderSpec.scala         | 433 +++++++++++++++
 .../cromwell/core/path/MappedPathBuilderSpec.scala | 101 ++++
 .../cromwell/core/path/PathBuilderSpecUtils.scala  | 126 +++++
 .../{ => proxy}/RetryableFileSystemProxySpec.scala |   6 +-
 .../test/scala/cromwell/util/FileUtilSpec.scala    |  16 -
 core/src/test/scala/cromwell/util/SampleWdl.scala  |  59 ++-
 .../test/scala/cromwell/util/TestFileUtil.scala    |  16 +-
 .../workflow/SingleWorkflowRunnerActor.scala       |  13 +-
 .../engine/workflow/WorkflowManagerActor.scala     |   9 +-
 .../workflow/lifecycle/CopyWorkflowLogsActor.scala |  14 +-
 .../lifecycle/CopyWorkflowOutputsActor.scala       |   6 +-
 .../MaterializeWorkflowDescriptorActor.scala       |  46 +-
 .../execution/callcaching/CallCache.scala          |   6 +-
 .../test/scala/cromwell/ArrayWorkflowSpec.scala    |  17 +-
 .../scala/cromwell/CopyWorkflowOutputsSpec.scala   |  17 +-
 .../test/scala/cromwell/CromwellTestKitSpec.scala  |  23 +-
 .../src/test/scala/cromwell/MapWorkflowSpec.scala  |  18 +-
 .../cromwell/engine/EngineFunctionsSpec.scala      |  10 +-
 .../workflow/SingleWorkflowRunnerActorSpec.scala   |  15 +-
 .../engine/workflow/WorkflowActorSpec.scala        |   9 +-
 .../cromwell/filesystems/gcs/GcsPathBuilder.scala  |  45 +-
 .../filesystems/gcs/GcsPathBuilderSpec.scala       | 359 ++++++++++++-
 .../filesystems/gcs/auth/GoogleAuthModeSpec.scala  |  22 +-
 project/Dependencies.scala                         |   2 +-
 .../cromwell/services/metadata/MetadataQuery.scala |   3 -
 src/main/scala/cromwell/CromwellCommandLine.scala  |  31 +-
 .../scala/cromwell/CromwellCommandLineSpec.scala   |  21 +-
 .../impl/htcondor/HtCondorJobExecutionActor.scala  |   2 +-
 .../backend/impl/htcondor/HtCondorWrapper.scala    |   7 +-
 .../localization/CachedResultLocalization.scala    |   8 +-
 .../impl/htcondor/HtCondorCommandSpec.scala        |   3 +-
 .../htcondor/HtCondorJobExecutionActorSpec.scala   |   5 +-
 .../CachedResultLocalizationSpec.scala             |   5 +-
 .../jes/JesAsyncBackendJobExecutionActor.scala     |  54 +-
 .../backend/impl/jes/JesExpressionFunctions.scala  |   5 +-
 .../backend/impl/jes/JesFinalizationActor.scala    |   3 +-
 .../backend/impl/jes/JesInitializationActor.scala  |   4 +-
 .../impl/jes/JesJobCachingActorHelper.scala        |   9 +-
 .../cromwell/backend/impl/jes/JesJobPaths.scala    |   3 +-
 .../cromwell/backend/impl/jes/JesParameters.scala  |   7 +-
 .../backend/impl/jes/JesWorkflowPaths.scala        |   7 +-
 .../cromwell/backend/impl/jes/RunStatus.scala      |   3 +-
 .../backend/impl/jes/errors/JesError.scala         |   3 +-
 .../impl/jes/errors/JesKnownJobFailure.scala       |   9 +-
 .../backend/impl/jes/io/JesAttachedDisk.scala      |  13 +-
 .../cromwell/backend/impl/jes/io/package.scala     |  12 +-
 .../jes/JesAsyncBackendJobExecutionActorSpec.scala |  73 ++-
 .../backend/impl/jes/JesAttachedDiskSpec.scala     |   7 +-
 .../backend/impl/jes/JesCallPathsSpec.scala        |  11 +-
 .../backend/impl/jes/JesConfigurationSpec.scala    |   7 +-
 .../backend/impl/jes/JesWorkflowPathsSpec.scala    |   7 +-
 .../sfs/config/ConfigAsyncJobExecutionActor.scala  |  32 +-
 .../impl/sfs/config/ConfigBackendFileHashing.scala |   3 +-
 .../impl/sfs/config/ConfigHashingStrategy.scala    |  16 +-
 .../sfs/BackgroundAsyncJobExecutionActor.scala     |  19 +-
 .../scala/cromwell/backend/sfs/ProcessRunner.scala |   2 +-
 .../cromwell/backend/sfs/SharedFileSystem.scala    |  56 +-
 .../SharedFileSystemAsyncJobExecutionActor.scala   |  37 +-
 .../sfs/SharedFileSystemCacheHitCopyingActor.scala |   3 +-
 .../sfs/SharedFileSystemExpressionFunctions.scala  |  20 +-
 .../sfs/SharedFileSystemInitializationActor.scala  |   6 +-
 .../sfs/config/ConfigHashingStrategySpec.scala     |  15 +-
 .../SharedFileSystemJobExecutionActorSpec.scala    |  40 +-
 .../backend/sfs/SharedFileSystemSpec.scala         |  22 +-
 .../sfs/TestLocalAsyncJobExecutionActor.scala      |   6 +-
 .../backend/impl/spark/SparkClusterProcess.scala   |  15 +-
 .../impl/spark/SparkJobExecutionActor.scala        |   2 +-
 .../cromwell/backend/impl/spark/SparkProcess.scala |   7 +-
 .../impl/spark/SparkClusterProcessSpec.scala       |  17 +-
 .../impl/spark/SparkJobExecutionActorSpec.scala    |   5 +-
 111 files changed, 2597 insertions(+), 863 deletions(-)
 create mode 100644 core/src/main/scala/cromwell/core/path/BetterFileMethods.scala
 create mode 100644 core/src/main/scala/cromwell/core/path/EvenBetterPathMethods.scala
 delete mode 100644 core/src/main/scala/cromwell/core/path/FileImplicits.scala
 create mode 100644 core/src/main/scala/cromwell/core/path/MappedPathBuilder.scala
 create mode 100644 core/src/main/scala/cromwell/core/path/NioPathMethods.scala
 create mode 100644 core/src/main/scala/cromwell/core/path/Obsolete.scala
 delete mode 100644 core/src/main/scala/cromwell/core/path/PathImplicits.scala
 create mode 100644 core/src/main/scala/cromwell/core/path/PathObjectMethods.scala
 create mode 100644 core/src/main/scala/cromwell/core/path/package.scala
 delete mode 100644 core/src/main/scala/cromwell/util/FileUtil.scala
 create mode 100644 core/src/test/scala/cromwell/core/path/DefaultPathBuilderSpec.scala
 create mode 100644 core/src/test/scala/cromwell/core/path/MappedPathBuilderSpec.scala
 create mode 100644 core/src/test/scala/cromwell/core/path/PathBuilderSpecUtils.scala
 rename core/src/test/scala/cromwell/core/path/{ => proxy}/RetryableFileSystemProxySpec.scala (99%)
 delete mode 100644 core/src/test/scala/cromwell/util/FileUtilSpec.scala

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7ec14e5cf..b2efbc431 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -18,6 +18,8 @@
   The non-root user could either be the default user for a given Docker image (e.g. specified in a Dockerfile via a `USER` directive),
   or the Config backend could pass an optional `"-u username"` as part of the `submit-docker` command.
 * For MySQL users, a massive scalability improvement via batched DB writing of internal metadata events. Note that one must add `rewriteBatchedStatements=true` to their JDBC URL in their config in order to take advantage of this
+* In some cases the SFS backend, used for Local, SGE, etc., coerced `WdlFile` to `WdlString` by using `.toUri`. This
+resulted in strings prepended with `file:///path/to/file`. Now absolute file paths will not contain the uri scheme.
 
 ### Database schema changes
 * Added CUSTOM_LABELS as a field of WORKFLOW_STORE_ENTRY, to store workflow store entries.
diff --git a/backend/src/main/scala/cromwell/backend/BackendJobBreadCrumb.scala b/backend/src/main/scala/cromwell/backend/BackendJobBreadCrumb.scala
index 1dbc9ca50..3fa6e9c19 100644
--- a/backend/src/main/scala/cromwell/backend/BackendJobBreadCrumb.scala
+++ b/backend/src/main/scala/cromwell/backend/BackendJobBreadCrumb.scala
@@ -1,8 +1,7 @@
 package cromwell.backend
 
-import java.nio.file.Path
-
 import cromwell.backend.io.JobPaths
+import cromwell.core.path.Path
 import cromwell.core.{JobKey, WorkflowId}
 import wdl4s.Workflow
 
diff --git a/backend/src/main/scala/cromwell/backend/BackendJobExecutionActor.scala b/backend/src/main/scala/cromwell/backend/BackendJobExecutionActor.scala
index 816f43da9..ffeccd7cd 100644
--- a/backend/src/main/scala/cromwell/backend/BackendJobExecutionActor.scala
+++ b/backend/src/main/scala/cromwell/backend/BackendJobExecutionActor.scala
@@ -1,12 +1,11 @@
 package cromwell.backend
 
-import java.nio.file.Path
-
 import akka.actor.ActorLogging
 import akka.event.LoggingReceive
 import cromwell.backend.BackendJobExecutionActor._
 import cromwell.backend.BackendLifecycleActor._
 import cromwell.backend.wdl.OutputEvaluator
+import cromwell.core.path.Path
 import cromwell.core.{CallOutputs, ExecutionEvent, JobKey}
 import wdl4s.expression.WdlStandardLibraryFunctions
 import wdl4s.values.WdlValue
diff --git a/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala b/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala
index cb625a78b..b6ae61917 100644
--- a/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala
+++ b/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala
@@ -1,7 +1,5 @@
 package cromwell.backend
 
-import java.nio.file.Path
-
 import akka.actor.{ActorRef, Props}
 import com.typesafe.config.Config
 import cromwell.backend.callcaching.FileHashingActor
@@ -9,6 +7,7 @@ import cromwell.backend.callcaching.FileHashingActor.FileHashingFunction
 import cromwell.backend.io.WorkflowPathsWithDocker
 import cromwell.core.CallOutputs
 import cromwell.core.JobExecutionToken.JobExecutionTokenType
+import cromwell.core.path.Path
 import wdl4s.TaskCall
 import wdl4s.expression.{PureStandardLibraryFunctions, WdlStandardLibraryFunctions}
 
diff --git a/backend/src/main/scala/cromwell/backend/async/ExecutionHandle.scala b/backend/src/main/scala/cromwell/backend/async/ExecutionHandle.scala
index 62618190c..1a0262d36 100644
--- a/backend/src/main/scala/cromwell/backend/async/ExecutionHandle.scala
+++ b/backend/src/main/scala/cromwell/backend/async/ExecutionHandle.scala
@@ -1,10 +1,9 @@
 package cromwell.backend.async
 
-import java.nio.file.Path
-
 import cromwell.backend.BackendJobDescriptor
 import cromwell.backend.async.AsyncBackendJobExecutionActor.JobId
-import cromwell.core.{ExecutionEvent, CallOutputs}
+import cromwell.core.path.Path
+import cromwell.core.{CallOutputs, ExecutionEvent}
 
 /**
  * Trait to encapsulate whether an execution is complete and if so provide a result.  Useful in conjunction
diff --git a/backend/src/main/scala/cromwell/backend/async/ExecutionResult.scala b/backend/src/main/scala/cromwell/backend/async/ExecutionResult.scala
index ff9722337..fc8f95e04 100644
--- a/backend/src/main/scala/cromwell/backend/async/ExecutionResult.scala
+++ b/backend/src/main/scala/cromwell/backend/async/ExecutionResult.scala
@@ -1,9 +1,8 @@
 package cromwell.backend.async
 
-import java.nio.file.Path
-
 import cromwell.backend.BackendJobDescriptor
-import cromwell.core.{ExecutionEvent, CallOutputs}
+import cromwell.core.path.Path
+import cromwell.core.{CallOutputs, ExecutionEvent}
 
 /**
  * ADT representing the result of an execution of a BackendCall.
diff --git a/backend/src/main/scala/cromwell/backend/async/KnownJobFailureException.scala b/backend/src/main/scala/cromwell/backend/async/KnownJobFailureException.scala
index 53130ada9..79503c927 100644
--- a/backend/src/main/scala/cromwell/backend/async/KnownJobFailureException.scala
+++ b/backend/src/main/scala/cromwell/backend/async/KnownJobFailureException.scala
@@ -1,7 +1,6 @@
 package cromwell.backend.async
 
-import java.nio.file.Path
-
+import cromwell.core.path.Path
 import lenthall.exception.ThrowableAggregation
 import wdl4s.values.WdlValue
 
diff --git a/backend/src/main/scala/cromwell/backend/callcaching/CacheHitDuplicating.scala b/backend/src/main/scala/cromwell/backend/callcaching/CacheHitDuplicating.scala
index bfde60986..979166f08 100644
--- a/backend/src/main/scala/cromwell/backend/callcaching/CacheHitDuplicating.scala
+++ b/backend/src/main/scala/cromwell/backend/callcaching/CacheHitDuplicating.scala
@@ -1,13 +1,10 @@
 package cromwell.backend.callcaching
 
-import java.nio.file.Path
-
 import akka.actor.ActorRef
 import cromwell.backend.BackendCacheHitCopyingActor
 import cromwell.backend.BackendJobExecutionActor.{BackendJobExecutionResponse, JobSucceededResponse}
 import cromwell.backend.io.JobPaths
-import cromwell.core.path.PathCopier
-import cromwell.core.path.PathImplicits._
+import cromwell.core.path.{Path, PathCopier}
 import cromwell.core.simpleton.{WdlValueBuilder, WdlValueSimpleton}
 import wdl4s.values.WdlFile
 
@@ -67,7 +64,7 @@ trait CacheHitDuplicating {
         val sourcePath = getPath(wdlFile.value).get
         val destinationPath = PathCopier.getDestinationFilePath(sourceCallRootPath, sourcePath, destinationCallRootPath)
         duplicate(sourcePath, destinationPath)
-        WdlValueSimpleton(key, WdlFile(destinationPath.toRealString))
+        WdlValueSimpleton(key, WdlFile(destinationPath.pathAsString))
       case wdlValueSimpleton => wdlValueSimpleton
     }
   }
diff --git a/backend/src/main/scala/cromwell/backend/io/GlobFunctions.scala b/backend/src/main/scala/cromwell/backend/io/GlobFunctions.scala
index c4fce1353..9ece33cdd 100644
--- a/backend/src/main/scala/cromwell/backend/io/GlobFunctions.scala
+++ b/backend/src/main/scala/cromwell/backend/io/GlobFunctions.scala
@@ -1,14 +1,10 @@
 package cromwell.backend.io
 
-import java.nio.file.Files
-
 import cromwell.backend.BackendJobDescriptor
 import cromwell.core.CallContext
 import wdl4s.TaskCall
 import wdl4s.expression.{NoFunctions, PureStandardLibraryFunctionsLike}
 import wdl4s.values._
-import scala.collection.JavaConverters._
-import cromwell.core.path.PathImplicits._
 
 trait GlobFunctions extends PureStandardLibraryFunctionsLike {
 
@@ -26,35 +22,29 @@ trait GlobFunctions extends PureStandardLibraryFunctionsLike {
   def globName(glob: String) = s"glob-${glob.md5Sum}"
 
   /**
-    * Returns a path to the glob using toRealString.
-    *
-    * NOTE: Due to use of toRealString, returned paths must be passed to PathBuilders.buildPath, and will not work with
-    * Paths.get.
+    * Returns a path to the glob.
     *
     * This path is usually passed back into the glob() method below.
     *
     * @param glob The glob. This is the same "pattern" passed to glob() below.
-    * @return The path converted using .toRealString.
+    * @return The path.
     */
-  override def globPath(glob: String): String = callContext.root.resolve(globDirectory(glob)).toRealString
+  override def globPath(glob: String): String = callContext.root.resolve(globDirectory(glob)).pathAsString
 
   /**
-    * Returns a list of path from the glob, each path converted to a string using toRealString.
+    * Returns a list of path from the glob.
     *
     * The paths are currently read from a list file based on the pattern, and the path parameter is not used.
     *
-    * NOTE: Due to use of toRealString, returned paths must be passed to PathBuilders.buildPath, and will not work with
-    * Paths.get.
-    *
     * @param path    The path string returned by globPath. This isn't currently used.
     * @param pattern The pattern of the glob. This is the same "glob" passed to globPath().
-    * @return The paths that match the pattern, each path converted using .toRealString.
+    * @return The paths that match the pattern.
     */
   override def glob(path: String, pattern: String): Seq[String] = {
     val name = globName(pattern)
     val listFile = callContext.root.resolve(s"$name.list").toRealPath()
-    Files.readAllLines(listFile).asScala map { fileName =>
-      callContext.root.resolve(name).resolve(fileName).toRealString
+    listFile.lines.toSeq map { fileName =>
+      callContext.root.resolve(name).resolve(fileName).pathAsString
     }
   }
 }
diff --git a/backend/src/main/scala/cromwell/backend/io/JobPaths.scala b/backend/src/main/scala/cromwell/backend/io/JobPaths.scala
index 24339d86c..2e0dfab43 100644
--- a/backend/src/main/scala/cromwell/backend/io/JobPaths.scala
+++ b/backend/src/main/scala/cromwell/backend/io/JobPaths.scala
@@ -1,7 +1,6 @@
 package cromwell.backend.io
 
-import java.nio.file.Path
-
+import cromwell.core.path.Path
 import cromwell.core.{CallContext, JobKey}
 import cromwell.services.metadata.CallMetadataKeys
 
@@ -27,7 +26,6 @@ object JobPaths {
 
 trait JobPaths { this: WorkflowPaths =>
   import JobPaths._
-  import cromwell.core.path.PathImplicits._
 
   def returnCodeFilename: String = "rc"
   def stdoutFilename: String = "stdout"
@@ -69,5 +67,5 @@ trait JobPaths { this: WorkflowPaths =>
   lazy val detritusPaths = commonDetritusPaths ++ customDetritusPaths
   lazy val logPaths = commonLogPaths ++ customLogPaths
 
-  lazy val callContext = CallContext(callExecutionRoot, stdout.toRealString, stderr.toRealString)
+  lazy val callContext = CallContext(callExecutionRoot, stdout.pathAsString, stderr.pathAsString)
 }
diff --git a/backend/src/main/scala/cromwell/backend/io/JobPathsWithDocker.scala b/backend/src/main/scala/cromwell/backend/io/JobPathsWithDocker.scala
index 49c748e88..9524e8231 100644
--- a/backend/src/main/scala/cromwell/backend/io/JobPathsWithDocker.scala
+++ b/backend/src/main/scala/cromwell/backend/io/JobPathsWithDocker.scala
@@ -1,10 +1,8 @@
 package cromwell.backend.io
 
-import java.nio.file.Path
-
 import com.typesafe.config.Config
-import cromwell.backend.{BackendWorkflowDescriptor, BackendJobDescriptorKey}
-import cromwell.core.path.PathBuilder
+import cromwell.backend.{BackendJobDescriptorKey, BackendWorkflowDescriptor}
+import cromwell.core.path.{Path, PathBuilder}
 
 class JobPathsWithDocker(val jobKey: BackendJobDescriptorKey,
                          workflowDescriptor: BackendWorkflowDescriptor,
@@ -36,4 +34,4 @@ class JobPathsWithDocker(val jobKey: BackendJobDescriptorKey,
         WorkflowPathsWithDocker.DockerRoot.resolve(subpath)
     }
   }
-}
\ No newline at end of file
+}
diff --git a/backend/src/main/scala/cromwell/backend/io/WorkflowPaths.scala b/backend/src/main/scala/cromwell/backend/io/WorkflowPaths.scala
index b1955c9ec..8a784a1ae 100644
--- a/backend/src/main/scala/cromwell/backend/io/WorkflowPaths.scala
+++ b/backend/src/main/scala/cromwell/backend/io/WorkflowPaths.scala
@@ -1,11 +1,9 @@
 package cromwell.backend.io
 
-import java.nio.file.Path
-
 import com.typesafe.config.Config
 import cromwell.backend.{BackendJobDescriptor, BackendJobDescriptorKey, BackendWorkflowDescriptor}
 import cromwell.core.WorkflowOptions.FinalCallLogsDir
-import cromwell.core.path.{DefaultPathBuilder, PathFactory}
+import cromwell.core.path.{DefaultPathBuilder, Path, PathFactory}
 import net.ceedubs.ficus.Ficus._
 
 import scala.util.Try
diff --git a/backend/src/main/scala/cromwell/backend/io/WorkflowPathsWithDocker.scala b/backend/src/main/scala/cromwell/backend/io/WorkflowPathsWithDocker.scala
index 9c1706d96..4e7811feb 100644
--- a/backend/src/main/scala/cromwell/backend/io/WorkflowPathsWithDocker.scala
+++ b/backend/src/main/scala/cromwell/backend/io/WorkflowPathsWithDocker.scala
@@ -1,13 +1,11 @@
 package cromwell.backend.io
 
-import java.nio.file.{Path, Paths}
-
 import com.typesafe.config.Config
 import cromwell.backend.{BackendJobDescriptorKey, BackendWorkflowDescriptor}
-import cromwell.core.path.PathBuilder
+import cromwell.core.path.{DefaultPathBuilder, Path, PathBuilder}
 
 object WorkflowPathsWithDocker {
-  val DockerRoot: Path = Paths.get("/cromwell-executions")
+  val DockerRoot: Path = DefaultPathBuilder.get("/cromwell-executions")
 }
 
 class WorkflowPathsWithDocker(val workflowDescriptor: BackendWorkflowDescriptor, val config: Config, val pathBuilders: List[PathBuilder] = WorkflowPaths.DefaultPathBuilders) extends WorkflowPaths {
diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala b/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala
index 0adb0dc38..5c2416009 100644
--- a/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala
+++ b/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala
@@ -1,10 +1,7 @@
 package cromwell.backend.standard
 
-import java.nio.file.Path
-
 import akka.actor.{Actor, ActorLogging, ActorRef}
 import akka.event.LoggingReceive
-import better.files.File
 import cromwell.backend.BackendJobExecutionActor.{AbortedResponse, BackendJobExecutionResponse}
 import cromwell.backend.BackendLifecycleActor.AbortJobCommand
 import cromwell.backend.async.AsyncBackendJobExecutionActor.{ExecutionMode, JobId, Recover}
@@ -12,7 +9,7 @@ import cromwell.backend.async.{AbortedExecutionHandle, AsyncBackendJobExecutionA
 import cromwell.backend.validation._
 import cromwell.backend.wdl.{Command, OutputEvaluator, WdlFileMapper}
 import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendJobDescriptor, BackendJobLifecycleActor}
-import cromwell.core.path.PathFactory.pathPlusSuffix
+import cromwell.core.path.Path
 import cromwell.core.{CallOutputs, CromwellAggregatedException, ExecutionEvent}
 import cromwell.services.keyvalue.KeyValueServiceActor._
 import cromwell.services.metadata.CallMetadataKeys
@@ -107,13 +104,13 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
   }
 
   /**
-    * Maps WdlFile to a local path, often maping Path.toRealString to Path.toString, for use in the
-    * commandLineValueMapper.
+    * Maps WdlFile to a local path, for use in the commandLineValueMapper.
     *
     * @param wdlFile The wdlFile.
     * @return The updated wdlFile.
     */
-  def mapCommandLineWdlFile(wdlFile: WdlFile): WdlFile = WdlSingleFile(workflowPaths.buildPath(wdlFile.value).toString)
+  def mapCommandLineWdlFile(wdlFile: WdlFile): WdlFile =
+    WdlSingleFile(workflowPaths.buildPath(wdlFile.value).pathAsString)
 
   /** @see [[Command.instantiate]] */
   final lazy val commandLineValueMapper: WdlValue => WdlValue = {
@@ -152,8 +149,8 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
   def globManipulation(globFile: WdlGlobFile): String = {
     val parentDirectory = globParentDirectory(globFile)
     val globDir = backendEngineFunctions.globName(globFile.value)
-    val globDirectory = File(parentDirectory)./(globDir)
-    val globList = File(parentDirectory)./(s"$globDir.list")
+    val globDirectory = parentDirectory./(globDir)
+    val globList = parentDirectory./(s"$globDir.list")
 
     s"""|mkdir $globDirectory
         |( ln -L ${globFile.value} $globDirectory 2> /dev/null ) || ( ln ${globFile.value} $globDirectory )
@@ -169,9 +166,9 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
     jobLogger.info(s"`$instantiatedCommand`")
 
     val cwd = commandDirectory
-    val tmpDir = File(cwd)./("tmp").path
-    val rcPath = File(cwd)./(jobPaths.returnCodeFilename).path
-    val rcTmpPath = pathPlusSuffix(rcPath, "tmp").path
+    val tmpDir = cwd./("tmp")
+    val rcPath = cwd./(jobPaths.returnCodeFilename)
+    val rcTmpPath = rcPath.plusExt("tmp")
 
     val globFiles = backendEngineFunctions.findGlobOutputs(call, jobDescriptor)
 
@@ -623,12 +620,12 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
                             oldHandle: StandardAsyncPendingExecutionHandle): ExecutionHandle = {
     try {
 
-      lazy val returnCodeAsString: Try[String] = Try(File(jobPaths.returnCode).contentAsString)
+      lazy val returnCodeAsString: Try[String] = Try(jobPaths.returnCode.contentAsString)
       lazy val returnCodeAsInt: Try[Int] = returnCodeAsString.map(_.trim.toInt)
       lazy val stderrAsOption: Option[Path] = Option(jobPaths.stderr)
 
       if (isSuccess(status)) {
-        lazy val stderrLength: Try[Long] = Try(File(jobPaths.stderr).size)
+        lazy val stderrLength: Try[Long] = Try(jobPaths.stderr.size)
         (stderrLength, returnCodeAsString, returnCodeAsInt) match {
             // Failed to get stderr size -> Retry
           case (Failure(exception), _, _) =>
@@ -660,7 +657,6 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
     } catch {
       case exception: Exception if isFatal(exception) =>
         jobLogger.warn("Caught fatal exception processing job result", exception)
-        // Return the original handle to try again.
         FailedNonRetryableExecutionHandle(exception)
       case exception: Exception =>
         jobLogger.warn("Caught exception processing job result, retrying", exception)
diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardCacheHitCopyingActor.scala b/backend/src/main/scala/cromwell/backend/standard/StandardCacheHitCopyingActor.scala
index 41d1b9118..5f0ee8982 100644
--- a/backend/src/main/scala/cromwell/backend/standard/StandardCacheHitCopyingActor.scala
+++ b/backend/src/main/scala/cromwell/backend/standard/StandardCacheHitCopyingActor.scala
@@ -1,11 +1,9 @@
 package cromwell.backend.standard
 
-import java.nio.file.Path
-
 import akka.actor.ActorRef
 import cromwell.backend.callcaching.CacheHitDuplicating
 import cromwell.backend.{BackendCacheHitCopyingActor, BackendConfigurationDescriptor, BackendInitializationData, BackendJobDescriptor}
-import cromwell.core.path.PathCopier
+import cromwell.core.path.{Path, PathCopier}
 
 /**
   * Trait of parameters passed to a StandardCacheHitCopyingActor.
diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardCachingActorHelper.scala b/backend/src/main/scala/cromwell/backend/standard/StandardCachingActorHelper.scala
index db93cc1b5..f2102ab58 100644
--- a/backend/src/main/scala/cromwell/backend/standard/StandardCachingActorHelper.scala
+++ b/backend/src/main/scala/cromwell/backend/standard/StandardCachingActorHelper.scala
@@ -1,13 +1,12 @@
 package cromwell.backend.standard
 
-import java.nio.file.Path
-
 import akka.actor.{Actor, ActorRef}
 import cromwell.backend._
 import cromwell.backend.callcaching.JobCachingActorHelper
 import cromwell.backend.io.{JobPaths, WorkflowPaths}
 import cromwell.backend.validation.{RuntimeAttributesValidation, ValidatedRuntimeAttributes}
 import cromwell.core.logging.JobLogging
+import cromwell.core.path.Path
 import wdl4s.TaskCall
 
 import scala.util.Try
diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardExpressionFunctions.scala b/backend/src/main/scala/cromwell/backend/standard/StandardExpressionFunctions.scala
index 3f0ea002f..3b29b796d 100644
--- a/backend/src/main/scala/cromwell/backend/standard/StandardExpressionFunctions.scala
+++ b/backend/src/main/scala/cromwell/backend/standard/StandardExpressionFunctions.scala
@@ -1,11 +1,9 @@
 package cromwell.backend.standard
 
-import java.nio.file.Path
-
 import cromwell.backend.io.GlobFunctions
 import cromwell.backend.wdl.{ReadLikeFunctions, WriteFunctions}
 import cromwell.core.CallContext
-import cromwell.core.path.PathBuilder
+import cromwell.core.path.{Path, PathBuilder}
 import wdl4s.expression.PureStandardLibraryFunctionsLike
 import wdl4s.values.{WdlFile, WdlValue}
 
diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardFinalizationActor.scala b/backend/src/main/scala/cromwell/backend/standard/StandardFinalizationActor.scala
index 588697a16..6acbe6cb7 100644
--- a/backend/src/main/scala/cromwell/backend/standard/StandardFinalizationActor.scala
+++ b/backend/src/main/scala/cromwell/backend/standard/StandardFinalizationActor.scala
@@ -1,13 +1,11 @@
 package cromwell.backend.standard
 
-import java.nio.file.Path
-
 import akka.dispatch.MessageDispatcher
 import cromwell.backend._
 import cromwell.backend.io.WorkflowPaths
 import cromwell.core.CallOutputs
 import cromwell.core.Dispatcher.IoDispatcher
-import cromwell.core.path.PathCopier
+import cromwell.core.path.{Path, PathCopier}
 import wdl4s.TaskCall
 
 import scala.concurrent.Future
diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardLifecycleActorFactory.scala b/backend/src/main/scala/cromwell/backend/standard/StandardLifecycleActorFactory.scala
index 61d980e15..ab11ccd82 100644
--- a/backend/src/main/scala/cromwell/backend/standard/StandardLifecycleActorFactory.scala
+++ b/backend/src/main/scala/cromwell/backend/standard/StandardLifecycleActorFactory.scala
@@ -1,12 +1,11 @@
 package cromwell.backend.standard
 
-import java.nio.file.Path
-
 import akka.actor.{ActorRef, Props}
 import com.typesafe.config.Config
 import cromwell.backend._
-import cromwell.core.{CallOutputs, Dispatcher}
 import cromwell.core.Dispatcher.BackendDispatcher
+import cromwell.core.path.Path
+import cromwell.core.{CallOutputs, Dispatcher}
 import wdl4s.TaskCall
 import wdl4s.expression.WdlStandardLibraryFunctions
 
diff --git a/backend/src/main/scala/cromwell/backend/wdl/ReadLikeFunctions.scala b/backend/src/main/scala/cromwell/backend/wdl/ReadLikeFunctions.scala
index 89f3ea00d..5a45cef96 100644
--- a/backend/src/main/scala/cromwell/backend/wdl/ReadLikeFunctions.scala
+++ b/backend/src/main/scala/cromwell/backend/wdl/ReadLikeFunctions.scala
@@ -10,7 +10,6 @@ import wdl4s.values._
 import scala.util.{Failure, Success, Try}
 
 trait ReadLikeFunctions extends PathFactory { this: WdlStandardLibraryFunctions =>
-  import better.files._
 
   /**
     * Asserts that the parameter list contains a single parameter which will be interpreted
@@ -28,7 +27,7 @@ trait ReadLikeFunctions extends PathFactory { this: WdlStandardLibraryFunctions
     wdlObjects <- WdlObject.fromTsv(contents)
   } yield wdlObjects
 
-  override def readFile(path: String): String = File(buildPath(path)).contentAsString
+  override def readFile(path: String): String = buildPath(path).contentAsString
 
   /**
     * Read all lines from the file referenced by the first parameter and return an Array[String]
@@ -94,12 +93,12 @@ trait ReadLikeFunctions extends PathFactory { this: WdlStandardLibraryFunctions
       for {
         value <- wdlValue
         unit <- convertTo
-      } yield MemorySize(File(buildPath(value.valueString)).size.toDouble, MemoryUnit.Bytes).to(unit).amount
+      } yield MemorySize(buildPath(value.valueString).size.toDouble, MemoryUnit.Bytes).to(unit).amount
     }
 
     params match {
-      case oneArg if params.length == 1 => fileSize(params.head) map WdlFloat.apply
-      case twoArgs if params.length == 2 => fileSize(params.head, toUnit(params(1))) map WdlFloat.apply
+      case _ if params.length == 1 => fileSize(params.head) map WdlFloat.apply
+      case _ if params.length == 2 => fileSize(params.head, toUnit(params.tail.head)) map WdlFloat.apply
       case _ => Failure(new UnsupportedOperationException(s"Expected one or two parameters but got ${params.length} instead."))
     }
   }
diff --git a/backend/src/main/scala/cromwell/backend/wdl/WdlFileMapper.scala b/backend/src/main/scala/cromwell/backend/wdl/WdlFileMapper.scala
index b079f8cc5..1c39c4d89 100644
--- a/backend/src/main/scala/cromwell/backend/wdl/WdlFileMapper.scala
+++ b/backend/src/main/scala/cromwell/backend/wdl/WdlFileMapper.scala
@@ -1,7 +1,7 @@
 package cromwell.backend.wdl
 
 import lenthall.util.TryUtil
-import wdl4s.values.{WdlArray, WdlFile, WdlMap, WdlValue}
+import wdl4s.values.{WdlArray, WdlFile, WdlMap, WdlOptionalValue, WdlPair, WdlValue}
 
 import scala.util.{Success, Try}
 
@@ -16,12 +16,21 @@ object WdlFileMapper {
         }
       case map: WdlMap =>
         val mappedMap = map.value map {
-          // TODO: TryUtil.sequenceMap doesn't currently handle keys of Try[WdlValue], so for now calling .get here
-          case (key, value) => mapWdlFiles(mapper)(key).get -> mapWdlFiles(mapper)(value)
+          case (key, value) => mapWdlFiles(mapper)(key) -> mapWdlFiles(mapper)(value)
         }
-        TryUtil.sequenceMap(mappedMap) map {
+        TryUtil.sequenceKeyValues(mappedMap) map {
           WdlMap(map.wdlType, _)
         }
+      case pair: WdlPair =>
+        val mappedPair: (Try[WdlValue], Try[WdlValue]) = (mapWdlFiles(mapper)(pair.left), mapWdlFiles(mapper)(pair.right))
+        TryUtil.sequenceTuple(mappedPair) map {
+          (WdlPair.apply _).tupled
+        }
+      case optionalValue: WdlOptionalValue =>
+        val mappedOptional: Option[Try[WdlValue]] = optionalValue.value.map(mapWdlFiles(mapper))
+        TryUtil.sequenceOption(mappedOptional) map {
+          WdlOptionalValue(optionalValue.innerType, _)
+        }
       case other => Success(other)
     }
   }
diff --git a/backend/src/main/scala/cromwell/backend/wdl/WriteFunctions.scala b/backend/src/main/scala/cromwell/backend/wdl/WriteFunctions.scala
index 293a5f05f..dd467bcf6 100644
--- a/backend/src/main/scala/cromwell/backend/wdl/WriteFunctions.scala
+++ b/backend/src/main/scala/cromwell/backend/wdl/WriteFunctions.scala
@@ -1,25 +1,21 @@
 package cromwell.backend.wdl
 
-import java.nio.file.Path
-
-import cromwell.core.path.FileImplicits._
+import cromwell.core.path.Path
 import wdl4s.TsvSerializable
 import wdl4s.expression.WdlStandardLibraryFunctions
 import wdl4s.types._
 import wdl4s.values._
 
-import scala.language.existentials
 import scala.util.{Failure, Try}
 
 trait WriteFunctions { this: WdlStandardLibraryFunctions =>
-  import better.files._
 
   /**
     * Directory that will be used to write files.
     */
   def writeDirectory: Path
 
-  private lazy val _writeDirectory = File(writeDirectory).createPermissionedDirectories()
+  private lazy val _writeDirectory = writeDirectory.createPermissionedDirectories()
 
   def writeTempFile(path: String,prefix: String,suffix: String,content: String): String = throw new NotImplementedError("This method is not used anywhere and should be removed")
 
@@ -29,7 +25,7 @@ trait WriteFunctions { this: WdlStandardLibraryFunctions =>
     Try {
       if (tmpFile.notExists) tmpFile.write(content)
     } map { _ =>
-      WdlFile(tmpFile.uri.toString)
+      WdlFile(tmpFile.pathAsString)
     }
   }
 
diff --git a/backend/src/test/scala/cromwell/backend/io/JobPathsSpec.scala b/backend/src/test/scala/cromwell/backend/io/JobPathsSpec.scala
index 94073dd52..0a8cb5936 100644
--- a/backend/src/test/scala/cromwell/backend/io/JobPathsSpec.scala
+++ b/backend/src/test/scala/cromwell/backend/io/JobPathsSpec.scala
@@ -1,10 +1,8 @@
 package cromwell.backend.io
 
-import java.nio.file.Paths
-
-import better.files._
 import com.typesafe.config.ConfigFactory
 import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptorKey, BackendSpec}
+import cromwell.core.path.DefaultPathBuilder
 import org.scalatest.{FlatSpec, Matchers}
 import wdl4s.TaskCall
 
@@ -36,42 +34,45 @@ class JobPathsSpec extends FlatSpec with Matchers with BackendSpec {
     val jobKey = BackendJobDescriptorKey(call, None, 1)
     val jobPaths = new JobPathsWithDocker(jobKey, wd, backendConfig)
     val id = wd.id
-    jobPaths.callRoot.toString shouldBe
-      File(s"local-cromwell-executions/wf_hello/$id/call-hello").pathAsString
-    jobPaths.callExecutionRoot.toString shouldBe
-      File(s"local-cromwell-executions/wf_hello/$id/call-hello/execution").pathAsString
-    jobPaths.returnCode.toString shouldBe
-      File(s"local-cromwell-executions/wf_hello/$id/call-hello/execution/rc").pathAsString
-    jobPaths.script.toString shouldBe
-      File(s"local-cromwell-executions/wf_hello/$id/call-hello/execution/script").pathAsString
-    jobPaths.stderr.toString shouldBe
-      File(s"local-cromwell-executions/wf_hello/$id/call-hello/execution/stderr").pathAsString
-    jobPaths.stdout.toString shouldBe
-      File(s"local-cromwell-executions/wf_hello/$id/call-hello/execution/stdout").pathAsString
-    jobPaths.callExecutionRoot.toString shouldBe
-      File(s"local-cromwell-executions/wf_hello/$id/call-hello/execution").pathAsString
-    jobPaths.callDockerRoot.toString shouldBe
-      File(s"/cromwell-executions/wf_hello/$id/call-hello").pathAsString
-    jobPaths.callExecutionDockerRoot.toString shouldBe
-      File(s"/cromwell-executions/wf_hello/$id/call-hello/execution").pathAsString
-    jobPaths.toDockerPath(Paths.get(s"local-cromwell-executions/wf_hello/$id/call-hello/execution/stdout")).toString shouldBe
-      File(s"/cromwell-executions/wf_hello/$id/call-hello/execution/stdout").pathAsString
-    jobPaths.toDockerPath(Paths.get("/cromwell-executions/dock/path")).toString shouldBe
-      File("/cromwell-executions/dock/path").pathAsString
+    jobPaths.callRoot.pathAsString shouldBe
+      fullPath(s"local-cromwell-executions/wf_hello/$id/call-hello")
+    jobPaths.callExecutionRoot.pathAsString shouldBe
+      fullPath(s"local-cromwell-executions/wf_hello/$id/call-hello/execution")
+    jobPaths.returnCode.pathAsString shouldBe
+      fullPath(s"local-cromwell-executions/wf_hello/$id/call-hello/execution/rc")
+    jobPaths.script.pathAsString shouldBe
+      fullPath(s"local-cromwell-executions/wf_hello/$id/call-hello/execution/script")
+    jobPaths.stderr.pathAsString shouldBe
+      fullPath(s"local-cromwell-executions/wf_hello/$id/call-hello/execution/stderr")
+    jobPaths.stdout.pathAsString shouldBe
+      fullPath(s"local-cromwell-executions/wf_hello/$id/call-hello/execution/stdout")
+    jobPaths.callExecutionRoot.pathAsString shouldBe
+      fullPath(s"local-cromwell-executions/wf_hello/$id/call-hello/execution")
+    jobPaths.callDockerRoot.pathAsString shouldBe
+      fullPath(s"/cromwell-executions/wf_hello/$id/call-hello")
+    jobPaths.callExecutionDockerRoot.pathAsString shouldBe
+      fullPath(s"/cromwell-executions/wf_hello/$id/call-hello/execution")
+    jobPaths.toDockerPath(DefaultPathBuilder.get(
+      s"local-cromwell-executions/wf_hello/$id/call-hello/execution/stdout")).pathAsString shouldBe
+      fullPath(s"/cromwell-executions/wf_hello/$id/call-hello/execution/stdout")
+    jobPaths.toDockerPath(DefaultPathBuilder.get("/cromwell-executions/dock/path")).pathAsString shouldBe
+      fullPath("/cromwell-executions/dock/path")
 
     val jobKeySharded = BackendJobDescriptorKey(call, Option(0), 1)
     val jobPathsSharded = new JobPathsWithDocker(jobKeySharded, wd, backendConfig)
-    jobPathsSharded.callExecutionRoot.toString shouldBe
-      File(s"local-cromwell-executions/wf_hello/$id/call-hello/shard-0/execution").pathAsString
+    jobPathsSharded.callExecutionRoot.pathAsString shouldBe
+      fullPath(s"local-cromwell-executions/wf_hello/$id/call-hello/shard-0/execution")
 
     val jobKeyAttempt = BackendJobDescriptorKey(call, None, 2)
     val jobPathsAttempt = new JobPathsWithDocker(jobKeyAttempt, wd, backendConfig)
-    jobPathsAttempt.callExecutionRoot.toString shouldBe
-      File(s"local-cromwell-executions/wf_hello/$id/call-hello/attempt-2/execution").pathAsString
+    jobPathsAttempt.callExecutionRoot.pathAsString shouldBe
+      fullPath(s"local-cromwell-executions/wf_hello/$id/call-hello/attempt-2/execution")
 
     val jobKeyShardedAttempt = BackendJobDescriptorKey(call, Option(0), 2)
     val jobPathsShardedAttempt = new JobPathsWithDocker(jobKeyShardedAttempt, wd, backendConfig)
-    jobPathsShardedAttempt.callExecutionRoot.toString shouldBe
-      File(s"local-cromwell-executions/wf_hello/$id/call-hello/shard-0/attempt-2/execution").pathAsString
+    jobPathsShardedAttempt.callExecutionRoot.pathAsString shouldBe
+      fullPath(s"local-cromwell-executions/wf_hello/$id/call-hello/shard-0/attempt-2/execution")
   }
+
+  private def fullPath(path: String): String = DefaultPathBuilder.get(path).toAbsolutePath.pathAsString
 }
diff --git a/backend/src/test/scala/cromwell/backend/io/WorkflowPathsSpec.scala b/backend/src/test/scala/cromwell/backend/io/WorkflowPathsSpec.scala
index 5cdec5500..c16348da9 100644
--- a/backend/src/test/scala/cromwell/backend/io/WorkflowPathsSpec.scala
+++ b/backend/src/test/scala/cromwell/backend/io/WorkflowPathsSpec.scala
@@ -1,8 +1,8 @@
 package cromwell.backend.io
 
-import better.files._
 import com.typesafe.config.Config
 import cromwell.backend.{BackendJobBreadCrumb, BackendSpec, BackendWorkflowDescriptor}
+import cromwell.core.path.DefaultPathBuilder
 import cromwell.core.{JobKey, WorkflowId}
 import org.mockito.Mockito._
 import org.scalatest.{FlatSpec, Matchers}
@@ -18,9 +18,9 @@ class WorkflowPathsSpec extends FlatSpec with Matchers with BackendSpec {
     val wd = buildWorkflowDescriptor(TestWorkflows.HelloWorld)
     val workflowPaths = new WorkflowPathsWithDocker(wd, backendConfig)
     val id = wd.id
-    workflowPaths.workflowRoot.toString shouldBe
-      File(s"local-cromwell-executions/wf_hello/$id").pathAsString
-    workflowPaths.dockerWorkflowRoot.toString shouldBe
+    workflowPaths.workflowRoot.pathAsString shouldBe
+      DefaultPathBuilder.get(s"local-cromwell-executions/wf_hello/$id").toAbsolutePath.pathAsString
+    workflowPaths.dockerWorkflowRoot.pathAsString shouldBe
       s"/cromwell-executions/wf_hello/$id"
   }
 
@@ -58,7 +58,10 @@ class WorkflowPathsSpec extends FlatSpec with Matchers with BackendSpec {
     subWd.id returns subWorkflowId
     
     val workflowPaths = new WorkflowPathsWithDocker(subWd, backendConfig)
-    workflowPaths.workflowRoot.toString shouldBe File(s"local-cromwell-executions/rootWorkflow/$rootWorkflowId/call-call1/shard-1/attempt-2/subWorkflow/$subWorkflowId").pathAsString
-    workflowPaths.dockerWorkflowRoot.toString shouldBe s"/cromwell-executions/rootWorkflow/$rootWorkflowId/call-call1/shard-1/attempt-2/subWorkflow/$subWorkflowId"
+    workflowPaths.workflowRoot.pathAsString shouldBe
+      DefaultPathBuilder.get(
+        s"local-cromwell-executions/rootWorkflow/$rootWorkflowId/call-call1/shard-1/attempt-2/subWorkflow/$subWorkflowId"
+      ).toAbsolutePath.pathAsString
+    workflowPaths.dockerWorkflowRoot.pathAsString shouldBe s"/cromwell-executions/rootWorkflow/$rootWorkflowId/call-call1/shard-1/attempt-2/subWorkflow/$subWorkflowId"
   }
 }
diff --git a/core/src/main/scala/cromwell/core/core.scala b/core/src/main/scala/cromwell/core/core.scala
index bf7bd72d1..8cfe7c2f1 100644
--- a/core/src/main/scala/cromwell/core/core.scala
+++ b/core/src/main/scala/cromwell/core/core.scala
@@ -1,7 +1,6 @@
 package cromwell.core
 
-import java.nio.file.Path
-
+import cromwell.core.path.Path
 import lenthall.exception.ThrowableAggregation
 import wdl4s.values.WdlValue
 
diff --git a/core/src/main/scala/cromwell/core/logging/WorkflowLogger.scala b/core/src/main/scala/cromwell/core/logging/WorkflowLogger.scala
index 289dd901f..a27e2b4a1 100644
--- a/core/src/main/scala/cromwell/core/logging/WorkflowLogger.scala
+++ b/core/src/main/scala/cromwell/core/logging/WorkflowLogger.scala
@@ -1,7 +1,5 @@
 package cromwell.core.logging
 
-import java.nio.file.{Path, Paths}
-
 import akka.actor.{Actor, ActorLogging}
 import akka.event.LoggingAdapter
 import ch.qos.logback.classic.encoder.PatternLayoutEncoder
@@ -10,7 +8,7 @@ import ch.qos.logback.classic.{Level, LoggerContext}
 import ch.qos.logback.core.FileAppender
 import com.typesafe.config.{Config, ConfigFactory}
 import cromwell.core.WorkflowId
-import cromwell.core.path.FileImplicits._
+import cromwell.core.path.{DefaultPathBuilder, Path}
 import net.ceedubs.ficus.Ficus._
 import org.slf4j.helpers.NOPLogger
 import org.slf4j.{Logger, LoggerFactory}
@@ -27,7 +25,7 @@ object WorkflowLogger {
   def makeFileLogger(path: Path, level: Level): Logger = {
     // see below
     val ctx = LoggerFactory.getILoggerFactory.asInstanceOf[LoggerContext]
-    val name = path.getFileName.toString
+    val name = path.name
 
     Option(ctx.exists(name)) match {
       case Some(existingLogger) => existingLogger
@@ -54,7 +52,7 @@ object WorkflowLogger {
         encoder.start()
 
         val appender = new FileAppender[ILoggingEvent]()
-        appender.setFile(path.toString)
+        appender.setFile(path.pathAsString)
         appender.setEncoder(encoder)
         appender.setName(name)
         appender.setContext(ctx)
@@ -77,7 +75,7 @@ object WorkflowLogger {
       workflowConfig <- conf.as[Option[Config]]("workflow-options")
       dir <- workflowConfig.as[Option[String]]("workflow-log-dir") if !dir.isEmpty
       temporary <- workflowConfig.as[Option[Boolean]]("workflow-log-temporary") orElse Option(true)
-    } yield WorkflowLogConfiguration(Paths.get(dir).toAbsolutePath, temporary)
+    } yield WorkflowLogConfiguration(DefaultPathBuilder.get(dir).toAbsolutePath, temporary)
   }
 
   val isEnabled = workflowLogConfiguration.isDefined
@@ -108,16 +106,14 @@ class WorkflowLogger(loggerName: String,
                      otherLoggers: Set[Logger] = Set.empty[Logger])
   extends LoggerWrapper {
 
-  import better.files._
-
   override def getName = loggerName
 
-  def deleteLogFile() = Try { workflowLogPath foreach { File(_).delete(swallowIOExceptions = false) } }
+  def deleteLogFile() = Try { workflowLogPath foreach { _.delete() } }
 
   import WorkflowLogger._
 
   lazy val workflowLogPath = workflowLogConfiguration.map(workflowLogConfigurationActual =>
-    File(workflowLogConfigurationActual.dir).createPermissionedDirectories() / s"workflow.$workflowId.log").map(_.path)
+    workflowLogConfigurationActual.dir.createPermissionedDirectories() / s"workflow.$workflowId.log")
 
   lazy val fileLogger = workflowLogPath match {
     case Some(path) => makeFileLogger(path, Level.toLevel(sys.props.getOrElse("LOG_LEVEL", "debug")))
diff --git a/core/src/main/scala/cromwell/core/path/BetterFileMethods.scala b/core/src/main/scala/cromwell/core/path/BetterFileMethods.scala
new file mode 100644
index 000000000..b6c09bf7a
--- /dev/null
+++ b/core/src/main/scala/cromwell/core/path/BetterFileMethods.scala
@@ -0,0 +1,588 @@
+package cromwell.core.path
+
+import java.io._
+import java.net.URI
+import java.nio.channels.{AsynchronousFileChannel, FileChannel}
+import java.nio.file.attribute._
+import java.nio.file.{FileSystem, PathMatcher}
+import java.time.Instant
+import java.util.zip.Deflater
+
+import scala.collection.JavaConverters._
+import scala.io.{BufferedSource, Codec}
+
+/**
+  * Implements methods with the same names and signatures as better.files.File.
+  *
+  * @see [[cromwell.core.path.Path]]
+  * @see [[cromwell.core.path.NioPathMethods]]
+  * @see [[cromwell.core.path.EvenBetterPathMethods]]
+  */
+trait BetterFileMethods {
+  self: Path =>
+
+  import BetterFileMethods._
+
+  private final def newPath(file: better.files.File): Path = newPath(file.path)
+
+  private final def newPathOrNull(file: better.files.File): Path = Option(file).map(newPath).orNull
+
+  final def toJava: JFile = betterFile.toJava
+
+  final def name: String = betterFile.name
+
+  final def nameOption: Option[String] = betterFile.nameOption
+
+  // NOTE: Check for null before it reaches better files' ==> root: File = [implicit File.apply]path.getRoot
+  final def root: Path = Option(betterFile.path.getRoot).map(_ => newPathOrNull(betterFile.root)).orNull
+
+  final def nameWithoutExtension: String = betterFile.nameWithoutExtension
+
+  final def extension: Option[String] = betterFile.extension
+
+  final def extension(includeDot: Boolean = true, includeAll: Boolean = false,
+                      toLowerCase: Boolean = true): Option[String] =
+    betterFile.extension(includeDot, includeAll, toLowerCase)
+
+  final def hasExtension: Boolean = betterFile.hasExtension
+
+  final def changeExtensionTo(extension: String): Path = newPath(betterFile.changeExtensionTo(extension))
+
+  final def contentType: Option[String] = betterFile.contentType
+
+  final def parent: Path = parentOption.orNull
+
+  final def parentOption: Option[Path] = betterFile.parentOption.map(newPath)
+
+  final def /(child: String): Path = newPath(betterFile./(child))
+
+  final def createChild(child: String, asDirectory: Boolean = false)
+                       (implicit attributes: Attributes = Attributes.default,
+                        linkOptions: LinkOptions = LinkOptions.default): Path =
+    newPath(betterFile.createChild(child, asDirectory)(attributes, linkOptions))
+
+  final def createIfNotExists(asDirectory: Boolean = false, createParents: Boolean = false)
+                             (implicit attributes: Attributes = Attributes.default,
+                              linkOptions: LinkOptions = LinkOptions.default): Path = {
+    newPath(betterFile.createIfNotExists(asDirectory, createParents)(attributes, linkOptions))
+  }
+
+  final def exists(implicit linkOptions: LinkOptions = LinkOptions.default): Boolean = betterFile.exists(linkOptions)
+
+  final def notExists(implicit linkOptions: LinkOptions = LinkOptions.default): Boolean =
+    betterFile.notExists(linkOptions)
+
+  final def sibling(name: String): Path = newPath(betterFile.sibling(name))
+
+  final def isSiblingOf(sibling: Path): Boolean = betterFile.isSiblingOf(sibling.betterFile)
+
+  final def siblings: Paths = betterFile.siblings.map(newPath)
+
+  final def isChildOf(parent: Path): Boolean = betterFile.isChildOf(parent.betterFile)
+
+  final def contains(file: Path): Boolean = betterFile.contains(file.betterFile)
+
+  final def isParentOf(child: Path): Boolean = contains(child)
+
+  final def bytes: Iterator[Byte] = betterFile.bytes
+
+  final def loadBytes: Array[Byte] = betterFile.loadBytes
+
+  final def byteArray: Array[Byte] = betterFile.byteArray
+
+  final def createDirectory()(implicit attributes: Attributes = Attributes.default): this.type = {
+    betterFile.createDirectory()(attributes)
+    this
+  }
+
+  final def createDirectories()(implicit attributes: Attributes = Attributes.default): this.type = {
+    betterFile.createDirectories()(attributes)
+    this
+  }
+
+  final def chars(implicit codec: Codec): Iterator[Char] = betterFile.chars(codec)
+
+  final def lines(implicit codec: Codec): Traversable[String] = betterFile.lines(codec)
+
+  final def lineIterator(implicit codec: Codec): Iterator[String] = betterFile.lineIterator(codec)
+
+  final def tokens(implicit config: Scanner.Config = Scanner.Config.default, codec: Codec): Traversable[String] =
+    betterFile.tokens(config, codec)
+
+  final def contentAsString(implicit codec: Codec): String = betterFile.contentAsString(codec)
+
+  final def `!`(implicit codec: Codec): String = betterFile.`!`(codec)
+
+  final def printLines(lines: Iterator[Any])(implicit openOptions: OpenOptions = OpenOptions.append): this.type = {
+    betterFile.printLines(lines)(openOptions)
+    this
+  }
+
+  final def appendLines(lines: String*)(implicit openOptions: OpenOptions = OpenOptions.append,
+                                        codec: Codec): this.type = {
+    betterFile.appendLines(lines: _*)(openOptions, codec)
+    this
+  }
+
+  final def <<(line: String)(implicit openOptions: OpenOptions = OpenOptions.append, codec: Codec): this.type = {
+    betterFile.<<(line)(openOptions, codec)
+    this
+  }
+
+  final def >>:(line: String)(implicit openOptions: OpenOptions = OpenOptions.append, codec: Codec): this.type = {
+    betterFile.>>:(line)(openOptions, codec)
+    this
+  }
+
+  final def appendLine(line: String = "")(implicit openOptions: OpenOptions = OpenOptions.append,
+                                          codec: Codec): this.type = {
+    betterFile.appendLine(line)(openOptions, codec)
+    this
+  }
+
+  final def append(text: String)(implicit openOptions: OpenOptions = OpenOptions.append, codec: Codec): this.type = {
+    betterFile.append(text)(openOptions, codec)
+    this
+  }
+
+  final def append(bytes: Array[Byte])(implicit openOptions: OpenOptions): this.type = {
+    betterFile.append(bytes)(openOptions)
+    this
+  }
+
+  final def appendBytes(bytes: Iterator[Byte])(implicit openOptions: OpenOptions = OpenOptions.append): this.type = {
+    betterFile.appendBytes(bytes)(openOptions)
+    this
+  }
+
+  final def write(bytes: Array[Byte])(implicit openOptions: OpenOptions): this.type = {
+    betterFile.write(bytes)(openOptions)
+    this
+  }
+
+  final def writeBytes(bytes: Iterator[Byte])(implicit openOptions: OpenOptions = OpenOptions.append): this.type = {
+    betterFile.writeBytes(bytes)(openOptions)
+    this
+  }
+
+  final def write(text: String)(implicit openOptions: OpenOptions = OpenOptions.default, codec: Codec): this.type = {
+    betterFile.write(text)(openOptions, codec)
+    this
+  }
+
+  final def overwrite(text: String)(implicit openOptions: OpenOptions = OpenOptions.default,
+                                    codec: Codec): this.type = {
+    betterFile.overwrite(text)(openOptions, codec)
+    this
+  }
+
+  final def <(text: String)(implicit openOptions: OpenOptions = OpenOptions.default, codec: Codec): this.type = {
+    betterFile.<(text)(openOptions, codec)
+    this
+  }
+
+  final def `>:`(text: String)(implicit openOptions: OpenOptions = OpenOptions.default, codec: Codec): this.type = {
+    betterFile.`>:`(text)(openOptions, codec)
+    this
+  }
+
+  final def newBufferedSource(implicit codec: Codec): BufferedSource = betterFile.newBufferedSource(codec)
+
+  final def bufferedSource(implicit codec: Codec): ManagedResource[BufferedSource] =
+    betterFile.bufferedSource(codec)
+
+  final def newRandomAccess(mode: RandomAccessMode = RandomAccessMode.read): RandomAccessFile =
+    betterFile.newRandomAccess(mode)
+
+  final def randomAccess(mode: RandomAccessMode = RandomAccessMode.read): ManagedResource[RandomAccessFile] =
+    betterFile.randomAccess(mode)
+
+  final def newBufferedReader(implicit codec: Codec): BufferedReader =
+    betterFile.newBufferedReader(codec)
+
+  final def bufferedReader(implicit codec: Codec): ManagedResource[BufferedReader] =
+    betterFile.bufferedReader(codec)
+
+  final def newBufferedWriter(implicit codec: Codec, openOptions: OpenOptions = OpenOptions.default): BufferedWriter =
+    betterFile.newBufferedWriter(codec)
+
+  final def bufferedWriter(implicit codec: Codec,
+                           openOptions: OpenOptions = OpenOptions.default): ManagedResource[BufferedWriter] =
+    betterFile.bufferedWriter(codec)
+
+  final def newFileReader: FileReader = betterFile.newFileReader
+
+  final def fileReader: ManagedResource[FileReader] = betterFile.fileReader
+
+  final def newFileWriter(append: Boolean = false): FileWriter = betterFile.newFileWriter(append)
+
+  final def fileWriter(append: Boolean = false): ManagedResource[FileWriter] = betterFile.fileWriter(append)
+
+  final def newPrintWriter(autoFlush: Boolean = false)
+                          (implicit openOptions: OpenOptions = OpenOptions.default): PrintWriter =
+    betterFile.newPrintWriter(autoFlush)
+
+  final def printWriter(autoFlush: Boolean = false)
+                       (implicit openOptions: OpenOptions = OpenOptions.default): ManagedResource[PrintWriter] =
+    betterFile.printWriter(autoFlush)
+
+  final def newInputStream(implicit openOptions: OpenOptions = OpenOptions.default): InputStream =
+    betterFile.newInputStream(openOptions)
+
+  final def inputStream(implicit openOptions: OpenOptions = OpenOptions.default): ManagedResource[InputStream] =
+    betterFile.inputStream(openOptions)
+
+  final def newScanner(implicit config: Scanner.Config = Scanner.Config.default): Scanner =
+    betterFile.newScanner(config)
+
+  final def scanner(implicit config: Scanner.Config = Scanner.Config.default): ManagedResource[Scanner] =
+    betterFile.scanner(config)
+
+  final def newOutputStream(implicit openOptions: OpenOptions = OpenOptions.default): OutputStream =
+    betterFile.newOutputStream(openOptions)
+
+  final def outputStream(implicit openOptions: OpenOptions = OpenOptions.default): ManagedResource[OutputStream] =
+    betterFile.outputStream(openOptions)
+
+  final def newFileChannel(implicit openOptions: OpenOptions = OpenOptions.default,
+                           attributes: Attributes = Attributes.default): FileChannel =
+    betterFile.newFileChannel(openOptions, attributes)
+
+  final def fileChannel(implicit openOptions: OpenOptions = OpenOptions.default,
+                        attributes: Attributes = Attributes.default): ManagedResource[FileChannel] =
+    betterFile.fileChannel(openOptions, attributes)
+
+  final def newAsynchronousFileChannel(implicit openOptions: OpenOptions = OpenOptions.default):
+  AsynchronousFileChannel = betterFile.newAsynchronousFileChannel(openOptions)
+
+  final def asynchronousFileChannel(implicit openOptions: OpenOptions = OpenOptions.default):
+  ManagedResource[AsynchronousFileChannel] = betterFile.asynchronousFileChannel(openOptions)
+
+  final def digest(algorithmName: String): Array[Byte] = betterFile.digest(algorithmName)
+
+  final def checksum(algorithm: String): String = betterFile.checksum(algorithm)
+
+  final def md5: String = betterFile.md5
+
+  final def sha1: String = betterFile.sha1
+
+  final def sha256: String = betterFile.sha256
+
+  final def sha512: String = betterFile.sha512
+
+  final def symbolicLink: Option[Path] = betterFile.symbolicLink.map(newPath)
+
+  final def isDirectory: Boolean = betterFile.isDirectory
+
+  final def isRegularFile: Boolean = betterFile.isRegularFile
+
+  final def isSymbolicLink: Boolean = betterFile.isSymbolicLink
+
+  final def isHidden: Boolean = betterFile.isHidden
+
+  final def isLocked(mode: RandomAccessMode, position: Long = 0L, size: Long = Long.MaxValue,
+                     isShared: Boolean = false): Boolean = betterFile.isLocked(mode, position, size, isShared)
+
+  final def isReadLocked(position: Long = 0L, size: Long = Long.MaxValue, isShared: Boolean = false): Boolean =
+    betterFile.isReadLocked(position, size, isShared)
+
+  final def isWriteLocked(position: Long = 0L, size: Long = Long.MaxValue, isShared: Boolean = false): Boolean =
+    betterFile.isWriteLocked(position, size, isShared)
+
+  final def list: Paths = betterFile.list.map(newPath)
+
+  final def children: Paths = betterFile.children.map(newPath)
+
+  final def entries: Paths = betterFile.entries.map(newPath)
+
+  final def listRecursively: Paths = betterFile.listRecursively.map(newPath)
+
+  final def walk(maxDepth: Int = Int.MaxValue): Paths = betterFile.walk(maxDepth).map(newPath)
+
+  final def pathMatcher(syntax: PathMatcherSyntax)(pattern: String): PathMatcher =
+    betterFile.pathMatcher(syntax)(pattern)
+
+  final def glob(pattern: String): Paths = betterFile.glob(pattern).map(newPath)
+
+  final def collectChildren(matchFilter: Path => Boolean): Paths = walk().filter(matchFilter(_))
+
+  final def fileSystem: FileSystem = betterFile.fileSystem
+
+  final def uri: URI = betterFile.uri
+
+  final def size: Long = betterFile.size
+
+  final def permissions: Set[PosixFilePermission] = betterFile.permissions
+
+  final def permissionsAsString: String = betterFile.permissionsAsString
+
+  final def setPermissions(permissions: Set[PosixFilePermission]): this.type = {
+    betterFile.setPermissions(permissions)
+    this
+  }
+
+  final def addPermission(permission: PosixFilePermission): this.type = {
+    betterFile.addPermission(permission)
+    this
+  }
+
+  final def removePermission(permission: PosixFilePermission): this.type = {
+    betterFile.removePermission(permission)
+    this
+  }
+
+  // Conflicts with the legacy cromwell.core.path.Obsolete.PathMethodAliases.getFileName(). Uncomment when that's gone.
+  //final def apply(permission: PosixFilePermission): Boolean = betterFile.apply(permission)
+
+  final def isOwnerReadable: Boolean = betterFile.isOwnerReadable
+
+  final def isOwnerWritable: Boolean = betterFile.isOwnerWritable
+
+  final def isOwnerExecutable: Boolean = betterFile.isOwnerExecutable
+
+  final def isGroupReadable: Boolean = betterFile.isGroupReadable
+
+  final def isGroupWritable: Boolean = betterFile.isGroupWritable
+
+  final def isGroupExecutable: Boolean = betterFile.isGroupExecutable
+
+  final def isOtherReadable: Boolean = betterFile.isOtherReadable
+
+  final def isOtherWritable: Boolean = betterFile.isOtherWritable
+
+  final def isOtherExecutable: Boolean = betterFile.isOtherExecutable
+
+  final def isReadable: Boolean = betterFile.isReadable
+
+  final def isWriteable: Boolean = betterFile.isWriteable
+
+  final def isExecutable: Boolean = betterFile.isExecutable
+
+  final def attributes(implicit linkOptions: LinkOptions = LinkOptions.default): BasicFileAttributes =
+    betterFile.attributes(linkOptions)
+
+  final def posixAttributes(implicit linkOptions: LinkOptions = LinkOptions.default): PosixFileAttributes =
+    betterFile.posixAttributes(linkOptions)
+
+  final def dosAttributes(implicit linkOptions: LinkOptions = LinkOptions.default): DosFileAttributes =
+    betterFile.dosAttributes(linkOptions)
+
+  final def owner(implicit linkOptions: LinkOptions = LinkOptions.default): UserPrincipal =
+    betterFile.owner(linkOptions)
+
+  final def ownerName(implicit linkOptions: LinkOptions = LinkOptions.default): String =
+    betterFile.ownerName(linkOptions)
+
+  final def group(implicit linkOptions: LinkOptions = LinkOptions.default): GroupPrincipal =
+    betterFile.group(linkOptions)
+
+  final def groupName(implicit linkOptions: LinkOptions = LinkOptions.default): String =
+    betterFile.groupName(linkOptions)
+
+  final def setOwner(owner: String): this.type = {
+    betterFile.setOwner(owner)
+    this
+  }
+
+  final def setGroup(group: String): this.type = {
+    betterFile.setGroup(group)
+    this
+  }
+
+  final def touch(time: Instant = Instant.now())
+                 (implicit attributes: Attributes = Attributes.default,
+                  linkOptions: LinkOptions = LinkOptions.default): this.type = {
+    betterFile.touch(time)(attributes, linkOptions)
+    this
+  }
+
+  final def lastModifiedTime(implicit linkOptions: LinkOptions = LinkOptions.default): Instant =
+    betterFile.lastModifiedTime(linkOptions)
+
+  final def delete(swallowIOExceptions: Boolean = false): this.type = {
+    betterFile.delete(swallowIOExceptions)
+    this
+  }
+
+  final def renameTo(newName: String): Path = newPath(betterFile.renameTo(newName))
+
+  final def moveTo(destination: Path, overwrite: Boolean = false): destination.type = {
+    betterFile.moveTo(destination.betterFile, overwrite)
+    destination
+  }
+
+  final def copyTo(destination: Path, overwrite: Boolean = false): destination.type = {
+    betterFile.copyTo(destination.betterFile, overwrite)
+    destination
+  }
+
+  final def symbolicLinkTo(destination: Path)
+                          (implicit attributes: Attributes = Attributes.default): destination.type = {
+    betterFile.symbolicLinkTo(destination.betterFile)(attributes)
+    destination
+  }
+
+  final def linkTo(destination: Path, symbolic: Boolean = false)
+                  (implicit attributes: Attributes = Attributes.default): destination.type = {
+    betterFile.linkTo(destination.betterFile, symbolic)(attributes)
+    destination
+  }
+
+  final def listRelativePaths(implicit visitOptions: VisitOptions = VisitOptions.default): Paths =
+    betterFile.listRelativePaths(visitOptions).map(newPath)
+
+  final def isSamePathAs(that: Path): Boolean = betterFile.isSamePathAs(that.betterFile)
+
+  final def isSameFileAs(that: Path): Boolean = betterFile.isSameFileAs(that.betterFile)
+
+  final def isSameContentAs(that: Path): Boolean = betterFile.isSameContentAs(that.betterFile)
+
+  final def `===`(that: Path): Boolean = betterFile.isSimilarContentAs(that.betterFile)
+
+  final def isSimilarContentAs(that: Path): Boolean = betterFile.isSimilarContentAs(that.betterFile)
+
+  final def =!=(that: Path): Boolean = betterFile.=!=(that.betterFile)
+
+  final def isEmpty: Boolean = betterFile.isEmpty
+
+  final def clear(): this.type = {
+    betterFile.clear()
+    this
+  }
+
+  final def zipTo(destination: Path, compressionLevel: Int = Deflater.DEFAULT_COMPRESSION)
+                 (implicit codec: Codec): destination.type = {
+    betterFile.zipTo(destination.betterFile, compressionLevel)(codec)
+    destination
+  }
+
+  final def zip(compressionLevel: Int = Deflater.DEFAULT_COMPRESSION)(implicit codec: Codec): Path =
+    newPath(betterFile.zip(compressionLevel)(codec))
+
+  final def unzipTo(destination: Path)(implicit codec: Codec): destination.type = {
+    betterFile.unzipTo(destination.betterFile)(codec)
+    destination
+  }
+
+  final def unzip()(implicit codec: Codec): Path = newPath(betterFile.unzip()(codec))
+}
+
+object BetterFileMethods {
+
+  def roots: Iterable[Path] = better.files.File.roots.map(file => DefaultPath(file.path))
+
+  def root: Path = DefaultPath(better.files.File.root.path)
+
+  def home: Path = DefaultPath(better.files.File.home.path)
+
+  def temp: Path = DefaultPath(better.files.File.temp.path)
+
+  def currentWorkingDirectory: Path = DefaultPath(better.files.File.currentWorkingDirectory.path)
+
+  object Cmds {
+    def ~ : Path = BetterFileMethods.home
+
+    def pwd: Path = BetterFileMethods.currentWorkingDirectory
+
+    def cwd: Path = pwd
+
+    val `..`: Path => Path = _.parent
+
+    val `.`: Path => Path = identity
+
+    implicit class FileDsl(file: Path) {
+      def /(f: Path => Path): Path = f(file)
+    }
+
+    def cp(file1: Path, file2: Path): Path = file1.copyTo(file2, overwrite = true)
+
+    def mv(file1: Path, file2: Path): Path = file1.moveTo(file2, overwrite = true)
+
+    def rm(file: Path): Path = file.delete(swallowIOExceptions = true)
+
+    def del(file: Path): Path = rm(file)
+
+    def ln(file1: Path, file2: Path): Path = file1 linkTo file2
+
+    def ln_s(file1: Path, file2: Path): Path = file1 symbolicLinkTo file2
+
+    def cat(files: Path*): Seq[Iterator[Byte]] = files.map(_.bytes)
+
+    def ls(file: Path): Paths = file.list
+
+    def dir(file: Path): Paths = ls(file)
+
+    def ls_r(file: Path): Paths = file.listRecursively
+
+    def touch(file: Path): Path = file.touch()
+
+    def mkdir(file: Path): Path = file.createDirectory()
+
+    def md5(file: Path): String = file.md5
+
+    def sha1(file: Path): String = file.sha1
+
+    def sha256(file: Path): String = file.sha256
+
+    def sha512(file: Path): String = file.sha512
+
+    def mkdirs(file: Path): Path = file.createDirectories()
+
+    def chown(owner: String, file: Path): Path = file.setOwner(owner)
+
+    def chgrp(group: String, file: Path): Path = file.setGroup(group)
+
+    def chmod(permissions: String, file: Path): Path = file.setPermissions(PosixFilePermissions.fromString(permissions).asScala.toSet)
+
+    def chmod_+(permission: PosixFilePermission, file: Path): Path = file.addPermission(permission)
+
+    def chmod_-(permission: PosixFilePermission, file: Path): Path = file.removePermission(permission)
+
+    def stat(file: Path): PosixFileAttributes = file.posixAttributes
+
+    def unzip(zipFile: Path)(destination: Path)(implicit codec: Codec): destination.type =
+      zipFile.unzipTo(destination)(codec)
+
+    def zip(files: Path*)(destination: Path, compressionLevel: Int = Deflater.DEFAULT_COMPRESSION)
+           (implicit codec: Codec): destination.type = {
+      better.files.Cmds.zip(files.map(_.betterFile): _*)(destination.betterFile, compressionLevel)
+      destination
+    }
+  }
+
+  type PathMatcherSyntax = better.files.File.PathMatcherSyntax
+  val PathMatcherSyntax = better.files.File.PathMatcherSyntax
+
+  type RandomAccessMode = better.files.File.RandomAccessMode
+  val RandomAccessMode = better.files.File.RandomAccessMode
+
+  type Attributes = better.files.File.Attributes
+  val Attributes = better.files.File.Attributes
+
+  type CopyOptions = better.files.File.CopyOptions
+  val CopyOptions = better.files.File.CopyOptions
+
+  type Events = better.files.File.Events
+  val Events = better.files.File.Events
+
+  type OpenOptions = better.files.File.OpenOptions
+  val OpenOptions = better.files.File.OpenOptions
+
+  type LinkOptions = better.files.File.LinkOptions
+  val LinkOptions = better.files.File.LinkOptions
+
+  type VisitOptions = better.files.File.VisitOptions
+  val VisitOptions = better.files.File.VisitOptions
+
+  type Order = better.files.File.Order
+  val Order = better.files.File.Order
+
+  type Scanner = better.files.Scanner
+  val Scanner = better.files.Scanner
+
+  type JFile = java.io.File
+  type Paths = Iterator[Path]
+
+  type Closable = better.files.Closeable
+  type ManagedResource[A <: Closable] = better.files.ManagedResource[A]
+}
diff --git a/core/src/main/scala/cromwell/core/path/DefaultPathBuilder.scala b/core/src/main/scala/cromwell/core/path/DefaultPathBuilder.scala
index 1db86bb35..09a9eed84 100644
--- a/core/src/main/scala/cromwell/core/path/DefaultPathBuilder.scala
+++ b/core/src/main/scala/cromwell/core/path/DefaultPathBuilder.scala
@@ -1,7 +1,7 @@
 package cromwell.core.path
 
 import java.net.URI
-import java.nio.file.{FileSystems, Path}
+import java.nio.file.FileSystems
 
 import com.google.common.net.UrlEscapers
 
@@ -12,13 +12,48 @@ import scala.util.Try
   */
 case object DefaultPathBuilder extends PathBuilder {
   override def name = "Default"
-  override def build(pathAsString: String): Try[Path] = Try {
+
+  override def build(pathAsString: String): Try[DefaultPath] = Try {
     val uri = URI.create(UrlEscapers.urlFragmentEscaper().escape(pathAsString))
-    val host = Option(uri.getHost) getOrElse ""
-    val path = Option(uri.getPath) getOrElse ""
     Option(uri.getScheme) match {
-      case Some("file") | None => FileSystems.getDefault.getPath(host, path)
+      case Some("file") | None =>
+
+        if (pathAsString.startsWith("file://")) {
+          // NOTE: Legacy support for old paths generated as URIs by the old .toRealString
+          val host = Option(uri.getHost) getOrElse ""
+          val path = Option(uri.getPath) getOrElse ""
+          DefaultPath(FileSystems.getDefault.getPath(host, path))
+        } else {
+          DefaultPath(FileSystems.getDefault.getPath(pathAsString))
+        }
+
       case _ => throw new RuntimeException(s"Cannot build a local path from $pathAsString")
     }
   }
+
+  /**
+    * Bypasses build()'s URI scheme detection and attempts to directly create a path, providing behavior similar to
+    * java.nio.Paths.get.
+    *
+    * Works even for bad paths like "gs:myfilename.txt", generated by:
+    * centaur 546c001, wf sub_function, call sub, input swappedFile, only when run on JES
+    */
+  def get(first: String, more: String*): Path = DefaultPath(FileSystems.getDefault.getPath(first, more: _*))
+
+  def createTempDirectory(prefix: String): DefaultPath = DefaultPath(java.nio.file.Files.createTempDirectory(prefix))
+
+  def createTempFile(prefix: String = "", suffix: String = "", parent: Option[Path] = None): Path = {
+    parent match {
+      case Some(dir) => dir.createTempFile(prefix, suffix)
+      case _ => DefaultPath(java.nio.file.Files.createTempFile(prefix, suffix))
+    }
+  }
+}
+
+case class DefaultPath private[path](nioPath: NioPath) extends Path {
+  override protected def newPath(nioPath: NioPath): DefaultPath = DefaultPath(nioPath)
+
+  override def pathAsString: String = nioPath.toString
+
+  override def pathWithoutScheme: String = nioPath.toAbsolutePath.toString
 }
diff --git a/core/src/main/scala/cromwell/core/path/EvenBetterPathMethods.scala b/core/src/main/scala/cromwell/core/path/EvenBetterPathMethods.scala
new file mode 100644
index 000000000..8ca7ba12c
--- /dev/null
+++ b/core/src/main/scala/cromwell/core/path/EvenBetterPathMethods.scala
@@ -0,0 +1,74 @@
+package cromwell.core.path
+
+import java.nio.file.FileAlreadyExistsException
+import java.nio.file.attribute.{PosixFilePermission, PosixFilePermissions}
+
+import scala.annotation.tailrec
+import scala.collection.JavaConverters._
+
+/**
+  * Implements methods beyond those implemented in NioPathMethods and BetterFileMethods
+  *
+  * @see [[cromwell.core.path.Path]]
+  * @see [[cromwell.core.path.NioPathMethods]]
+  * @see [[cromwell.core.path.BetterFileMethods]]
+  */
+trait EvenBetterPathMethods {
+  self: Path =>
+
+  final def getAttribute(attribute: String): Any = java.nio.file.Files.getAttribute(nioPathPrivate, attribute)
+
+  final def plusExt(ext: String): Path = plusSuffix(s".$ext")
+
+  final def swapExt(oldExt: String, newExt: String): Path = swapSuffix(s".$oldExt", s".$newExt")
+
+  final def plusSuffix(suffix: String): Path = swapSuffix("", suffix)
+
+  final def swapSuffix(oldSuffix: String, newSuffix: String): Path = {
+    sibling(s"${name.stripSuffix(oldSuffix)}$newSuffix")
+  }
+
+  final def createTempFile(prefix: String = "", suffix: String = ""): Path = {
+    newPath(java.nio.file.Files.createTempFile(nioPathPrivate, prefix, suffix))
+  }
+
+  def chmod(permissions: String): this.type = {
+    setPermissions(PosixFilePermissions.fromString(permissions).asScala.toSet)
+    this
+  }
+
+  @tailrec
+  final def followSymbolicLinks: Path = symbolicLink match {
+    case Some(target) => target.followSymbolicLinks
+    case None => this
+  }
+
+  final def createPermissionedDirectories(): this.type = {
+    if (!exists) {
+      parent.createPermissionedDirectories()
+      try {
+        createDirectories()
+        // When using PosixFilePermissions/FileAttributes with createDirectories, the umask Cromwell happens to be using
+        // affects the resulting directory permissions.  This is not the desired behavior, these directories should be
+        // world readable/writable/executable irrespective of the umask.
+        addPermission(PosixFilePermission.OTHERS_READ)
+        addPermission(PosixFilePermission.OTHERS_WRITE)
+        addPermission(PosixFilePermission.OTHERS_EXECUTE)
+      }
+      catch {
+        // Race condition that's particularly likely with scatters.  Ignore.
+        case _: FileAlreadyExistsException =>
+        // The GCS filesystem does not support setting permissions and will throw an `UnsupportedOperationException`.
+        // Evaluating expressions like `write_lines` in a command block will cause the above permission-manipulating
+        // code to run against a GCS Path. Fortunately creating directories in GCS is also unnecessary, so this
+        // exception type is just ignored.
+        case _: UnsupportedOperationException =>
+      }
+    }
+    this
+  }
+
+  final def untailed = UntailedWriter(this)
+
+  final def tailed(tailedSize: Int) = TailedWriter(this, tailedSize)
+}
diff --git a/core/src/main/scala/cromwell/core/path/FileImplicits.scala b/core/src/main/scala/cromwell/core/path/FileImplicits.scala
deleted file mode 100644
index 0857402da..000000000
--- a/core/src/main/scala/cromwell/core/path/FileImplicits.scala
+++ /dev/null
@@ -1,36 +0,0 @@
-package cromwell.core.path
-
-import java.nio.file.FileAlreadyExistsException
-import java.nio.file.attribute.PosixFilePermission
-
-import better.files.File
-
-object FileImplicits {
-
-  implicit class EnhancedFile(val directory: File) extends AnyVal {
-    def createPermissionedDirectories(): directory.type = {
-      if (!directory.exists) {
-        directory.parent.createPermissionedDirectories()
-        try {
-          directory.createDirectory()
-          // When using PosixFilePermissions/FileAttributes with createDirectories, the umask Cromwell happens to be using
-          // affects the resulting directory permissions.  This is not the desired behavior, these directories should be
-          // world readable/writable/executable irrespective of the umask.
-          directory.addPermission(PosixFilePermission.OTHERS_READ)
-          directory.addPermission(PosixFilePermission.OTHERS_WRITE)
-          directory.addPermission(PosixFilePermission.OTHERS_EXECUTE)
-        }
-        catch {
-          // Race condition that's particularly likely with scatters.  Ignore.
-          case _: FileAlreadyExistsException =>
-          // The GCS filesystem does not support setting permissions and will throw an `UnsupportedOperationException`.
-          // Evaluating expressions like `write_lines` in a command block will cause the above permission-manipulating
-          // code to run against a GCS Path. Fortunately creating directories in GCS is also unnecessary, so this
-          // exception type is just ignored.
-          case _: UnsupportedOperationException =>
-        }
-      }
-      directory
-    }
-  }
-}
diff --git a/core/src/main/scala/cromwell/core/path/MappedPathBuilder.scala b/core/src/main/scala/cromwell/core/path/MappedPathBuilder.scala
new file mode 100644
index 000000000..ade950cb8
--- /dev/null
+++ b/core/src/main/scala/cromwell/core/path/MappedPathBuilder.scala
@@ -0,0 +1,31 @@
+package cromwell.core.path
+
+import java.nio.file.FileSystems
+
+import scala.util.Try
+
+class MappedPathBuilder(prefix: String, mappedRoot: String) extends PathBuilder {
+  override def name: String = s"$prefix mapped to $mappedRoot"
+
+  override def build(pathAsString: String): Try[Path] = Try {
+    if (pathAsString.startsWith(prefix)) {
+      val fileSystem = FileSystems.getDefault
+      val root = fileSystem.getPath(mappedRoot)
+      val path = fileSystem.getPath(mappedRoot, pathAsString.stripPrefix(prefix))
+      MappedPath(prefix, root, path)
+
+    } else {
+      throw new IllegalArgumentException(s"$pathAsString must start with $prefix")
+    }
+  }
+}
+
+case class MappedPath(prefix: String, mappedRoot: NioPath, nioPath: NioPath) extends Path {
+  override protected def newPath(nioPath: NioPath): Path = MappedPath(prefix, mappedRoot, nioPath)
+
+  override def pathAsString: String = nioPath.toString
+
+  override def pathWithoutScheme: String = mappedRoot.relativize(nioPath).toString
+
+  def prefixedPathAsString: String = prefix + pathWithoutScheme
+}
diff --git a/core/src/main/scala/cromwell/core/path/NioPathMethods.scala b/core/src/main/scala/cromwell/core/path/NioPathMethods.scala
new file mode 100644
index 000000000..607ed945d
--- /dev/null
+++ b/core/src/main/scala/cromwell/core/path/NioPathMethods.scala
@@ -0,0 +1,69 @@
+package cromwell.core.path
+
+import java.net.URI
+import java.nio.file.WatchEvent.{Kind, Modifier}
+import java.nio.file.{LinkOption, WatchKey, WatchService}
+
+import scala.collection.JavaConverters._
+
+/**
+  * Implements methods with the same names and signatures as java.nio.Path
+  *
+  * @see [[cromwell.core.path.Path]]
+  * @see [[cromwell.core.path.BetterFileMethods]]
+  * @see [[cromwell.core.path.EvenBetterPathMethods]]
+  */
+trait NioPathMethods {
+  self: Path =>
+
+  final def subpath(beginIndex: Int, endIndex: Int): Path = newPath(nioPathPrivate.subpath(beginIndex, endIndex))
+
+  final def toFile: java.io.File = nioPathPrivate.toFile
+
+  final def resolveSibling(other: Path): Path = newPath(nioPathPrivate.resolveSibling(other.nioPathPrivate))
+
+  final def resolveSibling(other: String): Path = newPath(nioPathPrivate.resolveSibling(other))
+
+  final def isAbsolute: Boolean = nioPathPrivate.isAbsolute
+
+  final def getName(index: Int): Path = newPath(nioPathPrivate.getName(index))
+
+  final def getParent: Path = newPathOrNull(nioPathPrivate.getParent)
+
+  final def toAbsolutePath: Path = newPath(nioPathPrivate.toAbsolutePath)
+
+  final def relativize(other: Path): Path = newPath(nioPathPrivate.relativize(other.nioPathPrivate))
+
+  final def getNameCount: Int = nioPathPrivate.getNameCount
+
+  final def toUri: URI = nioPathPrivate.toUri
+
+  final def compareTo(other: Path): Int = nioPathPrivate.compareTo(other.nioPathPrivate)
+
+  final def register(watcher: WatchService, events: Array[Kind[_]], modifiers: Modifier*): WatchKey =
+    nioPathPrivate.register(watcher, events, modifiers: _*)
+
+  final def register(watcher: WatchService, events: Kind[_]*): WatchKey = nioPathPrivate.register(watcher, events: _*)
+
+  final def getFileName: Path = newPathOrNull(nioPathPrivate.getFileName)
+
+  final def getRoot: Path = newPathOrNull(nioPathPrivate.getRoot)
+
+  final def iterator(): java.util.Iterator[Path] = nioPathPrivate.iterator().asScala.map(newPath).asJava
+
+  final def normalize(): Path = newPath(nioPathPrivate.normalize())
+
+  final def endsWith(other: Path): Boolean = nioPathPrivate.endsWith(other.nioPathPrivate)
+
+  final def endsWith(other: String): Boolean = nioPathPrivate.endsWith(other)
+
+  final def resolve(other: Path): Path = newPath(nioPathPrivate.resolve(other.nioPathPrivate))
+
+  final def resolve(other: String): Path = newPath(nioPathPrivate.resolve(other))
+
+  final def startsWith(other: Path): Boolean = nioPathPrivate.startsWith(other.nioPathPrivate)
+
+  final def startsWith(other: String): Boolean = nioPathPrivate.startsWith(other)
+
+  final def toRealPath(options: LinkOption*): Path = newPath(nioPathPrivate.toRealPath(options: _*))
+}
diff --git a/core/src/main/scala/cromwell/core/path/Obsolete.scala b/core/src/main/scala/cromwell/core/path/Obsolete.scala
new file mode 100644
index 000000000..77fd32b54
--- /dev/null
+++ b/core/src/main/scala/cromwell/core/path/Obsolete.scala
@@ -0,0 +1,68 @@
+package cromwell.core.path
+
+/**
+  * Implements bridges for code that hasn't been updated to use cromwell.core.path.Path.
+  *
+  * Before adding methods here, recommend checking out and trying [[cromwell.core.path.Path]].
+  */
+object Obsolete {
+  def rm(path: Path) = path.delete(true)
+
+  implicit class PathMethodAliases(val originalPath: Path) extends AnyVal {
+    // Instead of `myPath.path`, just use `myPath`.
+    def path: Path = originalPath
+
+    // Instead of `myPath.getFileName()`, just use `myPath.getFileName`
+    def getFileName(unused: Unit = ()): Path = originalPath.getFileName
+  }
+
+  implicit class StringToPath(val path: String) extends AnyVal {
+    def toFile: Path = DefaultPathBuilder.get(path)
+  }
+
+  object Files {
+    def createTempDirectory(prefix: String): DefaultPath = DefaultPathBuilder.createTempDirectory(prefix)
+
+    def exists(path: Path): Boolean = path.exists
+
+    def isSymbolicLink(path: Path): Boolean = path.isSymbolicLink
+
+    // NOTE: When upgrading, symbolicLinkTo returns the target, not the link.
+    def createSymbolicLink(link: Path, target: Path): link.type = {
+      link.symbolicLinkTo(target)
+      link
+    }
+
+    def delete(path: Path): Unit = {
+      path.delete()
+      ()
+    }
+  }
+
+  object Paths {
+    def get(first: String, more: String*): Path = DefaultPathBuilder.get(first, more: _*)
+  }
+
+  type File = Path
+  val File = ObsoleteFile
+
+  object ObsoleteFile {
+    def newTemporaryDirectory(prefix: String = ""): DefaultPath = {
+      DefaultPath(better.files.File.newTemporaryDirectory(prefix).path)
+    }
+
+    def newTemporaryFile(prefix: String = "", suffix: String = "", parent: Option[Path] = None): Path = {
+      parent match {
+        case Some(dir) => dir.createTempFile(prefix, suffix)
+        case _ => DefaultPathBuilder.createTempFile(prefix, suffix)
+      }
+    }
+
+    def apply(path: String, fragments: String*) = DefaultPath(better.files.File(path, fragments: _*).path)
+
+    def apply(path: NioPath) = DefaultPath(better.files.File(path).path)
+
+    def apply(path: Path) = path
+  }
+
+}
diff --git a/core/src/main/scala/cromwell/core/path/PathBuilder.scala b/core/src/main/scala/cromwell/core/path/PathBuilder.scala
index c21310192..5d412a10e 100644
--- a/core/src/main/scala/cromwell/core/path/PathBuilder.scala
+++ b/core/src/main/scala/cromwell/core/path/PathBuilder.scala
@@ -1,10 +1,120 @@
 package cromwell.core.path
 
-import java.nio.file.Path
-
 import scala.util.Try
 
 trait PathBuilder {
   def name: String
+
   def build(pathAsString: String): Try[Path]
 }
+
+/**
+  * A path that was built by a PathBuilder.
+  *
+  * Encapsulates an underlying java nio path adding utility methods.
+  *
+  * @see [[cromwell.core.path.NioPathMethods]]
+  * @see [[cromwell.core.path.BetterFileMethods]]
+  * @see [[cromwell.core.path.EvenBetterPathMethods]]
+  */
+trait Path extends PathObjectMethods with NioPathMethods with BetterFileMethods with EvenBetterPathMethods {
+  /**
+    * A reference to the underlying nioPath, used to create new java.nio.Path's that will then be sent to newPath
+    * for wrapping.
+    *
+    * @return The internal nioPath.
+    */
+  protected def nioPath: NioPath
+
+  /**
+    * Wraps the nioPath in our container type.
+    *
+    * The nioPath should be from the same filesystem.
+    *
+    * @param nioPath The nioPath to be wrapped.
+    * @return A new Path.
+    */
+  protected def newPath(nioPath: NioPath): Path
+
+  /**
+    * Returns the path as a string. This path must be usable by this path builder or another in the list to build paths,
+    * and when isAbsolute returns true, should be usable by external users for accessing their files.
+    *
+    * Examples:
+    * {{{
+    * path                          path.pathAsString
+    * ---------------------------   ---------------------------
+    * gs://bucket/path/to/file      gs://bucket/path/to/file
+    * gs://bucket/path/to/my file   gs://bucket/path/to/my file
+    * /mount/path/to/file           /mount/path/to/file
+    * /mount/path/to/my file        /mount/path/to/my file
+    * }}}
+    *
+    * If isAbsolute returns false, the behavior may be different depending on the Path/PathBuilder:
+    * {{{
+    * path                      path.isAbsolute  path.getName(2)      p.gN(2).isAbsolute     p.gN(2).pathAsString
+    * ------------------------  ---------------  ------------------   ---------------------  -------------------------
+    * gs://bucket/path/to/file  true             gs://bucket/file     false                  gs://bucket/file
+    * /mount/path/to/file       true             /mount/path/to/file  false                  to
+    * mount/path/to/file        false            mount/path/to/file   false                  to
+    * }}}
+    *
+    * @return Path as a string
+    */
+  def pathAsString: String
+
+  /**
+    * Returns the unencoded uri of the string of the string stripping off the scheme.
+    *
+    * This represents the old model of {{{path.toUri.getHost + path.toUri.getPath}}}, but doesn't encode the string.
+    *
+    * Examples:
+    * {{{
+    * path                          path.pathWithoutScheme
+    * ---------------------------   -----------------------
+    * gs://bucket/path/to/file      bucket/path/to/file
+    * gs://bucket/path/to/my file   bucket/path/to/my file
+    * /mount/path/to/file           /mount/path/to/file
+    * /mount/path/to/my file        /mount/path/to/my file
+    * }}}
+    *
+    * This path is not relative to the root of the underlying path:
+    * {{{
+    * path                       path.root      path.pathWithoutScheme   path.root.resolve(path.pathWithoutScheme)
+    * ------------------------   ------------   ----------------------   -------------------------------------------
+    * gs://bucket/path/to/file   gs://bucket/   bucket/path/to/file      gs://bucket/bucket/path/to/file  (BAD!!)
+    * /mount/path/to/file        /              /mount/path/to/file      /mount/path/to/file  (ok, but still don't.)
+    * }}}
+    *
+    * The returned value is also different from a [[java.net.URI]], in that the value should not be encoded:
+    * {{{
+    * path                          path.pathWithoutScheme    path.toUri
+    * ---------------------------   -----------------------   ------------------------------------------
+    * gs://bucket/path/to/file      bucket/path/to/file       gs://bucket/path/to/file
+    * gs://bucket/path/to/my file   bucket/path/to/my file    gs://bucket/path/to/my%20file  (BAD!!)
+    * /mount/path/to/file           /mount/path/to/file       file:///mount/path/to/file
+    * /mount/path/to/my file        /mount/path/to/my file    file:///mount/path/to/my%20file  (BAD!!)
+    * }}}
+    *
+    * If isAbsolute returns false, the behavior may be different depending on the Path/PathBuilder:
+    * {{{
+    * path                      path.isAbsolute  path.getName(2)   p.gN(2).isAbsolute     p.gN(2).pathWithoutScheme
+    * ------------------------  ---------------  ----------------  ---------------------  -------------------------
+    * gs://bucket/path/to/file  true             gs://bucket/file  false                  bucket/file
+    * /mount/path/to/file       true             to                false                  /<pwd>/to
+    * }}}
+
+    *
+    * @return A "relative" path
+    */
+  def pathWithoutScheme: String
+
+  // Used by various extension traits within this scala package
+  private[path] final def nioPathPrivate: NioPath = nioPath
+
+  // Used within BetterFileMethods
+  private[path] final def betterFile: better.files.File = nioPathPrivate
+
+  // Some Path methods return null.
+  private[path] final def newPathOrNull(nioPath: NioPath) = Option(nioPath).map(newPath).orNull
+}
diff --git a/core/src/main/scala/cromwell/core/path/PathCopier.scala b/core/src/main/scala/cromwell/core/path/PathCopier.scala
index 8318f8202..92c84c52c 100644
--- a/core/src/main/scala/cromwell/core/path/PathCopier.scala
+++ b/core/src/main/scala/cromwell/core/path/PathCopier.scala
@@ -1,31 +1,26 @@
 package cromwell.core.path
 
 import java.io.IOException
-import java.nio.file.Path
-
-import better.files._
 
 import scala.util.{Failure, Try}
-import FileImplicits._
 
 object PathCopier {
 
   /*
-   * Remove p1 from p2 as long as they match.
+   * Remove common prefix found in path1 from path2 as long the directories match.
    */
-  private def truncateCommonRoot(p1: Path, p2: Path): String = {
-    def names(p: Path) = 0 until p.getNameCount map p.getName
+  private[path] def truncateCommonRoot(path1: Path, path2: Path): String = {
+    val string1 = path1.toAbsolutePath.pathWithoutScheme
+    val string2 = path2.toAbsolutePath.pathWithoutScheme
 
-    val names1 = names(p1)
+    val regexIncludingSlashes = "(?<=/)|(?=/)" // http://stackoverflow.com/q/2206378
+    val tokens1 = string1.split(regexIncludingSlashes)
+    val tokens2 = string2.split(regexIncludingSlashes)
 
-    val truncated = names(p2).zipWithIndex.dropWhile {
-      case (n1, n2) => n2 < names1.size && n1.equals(names1(n2))
-    } map { _._1 }
+    val matchingTokens: Array[(String, String)] = tokens1.zip(tokens2).takeWhile(Function.tupled(_ == _))
+    val matchingPrefix = matchingTokens.map({ case (str, _) => str }).mkString
 
-    truncated match {
-      case empty if empty.isEmpty => ""
-      case truncs => truncs.reduceLeft(_.resolve(_)).toString
-    }
+    string2.stripPrefix(matchingPrefix).replaceAll("^/+", "")
   }
 
   def getDestinationFilePath(sourceContextPath: Path, sourceFilePath: Path, destinationDirPath: Path): Path = {
@@ -45,13 +40,13 @@ object PathCopier {
     * Copies from source to destination. NOTE: Copies are not atomic, and may create a partial copy.
     */
   def copy(sourceFilePath: Path, destinationFilePath: Path): Try[Unit] = {
-    Option(File(destinationFilePath).parent).foreach(_.createPermissionedDirectories())
+    Option(destinationFilePath.parent).foreach(_.createPermissionedDirectories())
     Try {
-      File(sourceFilePath).copyTo(destinationFilePath, overwrite = true)
+      sourceFilePath.copyTo(destinationFilePath, overwrite = true)
 
       ()
     } recoverWith {
-      case ex => Failure(new IOException(s"Failed to copy ${sourceFilePath.toUri} to ${destinationFilePath.toUri}", ex))
+      case ex => Failure(new IOException(s"Failed to copy $sourceFilePath to $destinationFilePath", ex))
     }
   }
 }
diff --git a/core/src/main/scala/cromwell/core/path/PathFactory.scala b/core/src/main/scala/cromwell/core/path/PathFactory.scala
index 92af7a3b0..62a38ae83 100644
--- a/core/src/main/scala/cromwell/core/path/PathFactory.scala
+++ b/core/src/main/scala/cromwell/core/path/PathFactory.scala
@@ -1,9 +1,5 @@
 package cromwell.core.path
 
-import java.nio.file.Path
-
-import better.files.File
-
 import scala.util.Success
 
 /**
@@ -11,34 +7,29 @@ import scala.util.Success
   */
 trait PathFactory {
   /**
-    * Path builders to be applied (in order) to attempt to build a java.nio.Path from a string.
+    * Path builders to be applied (in order) to attempt to build a Path from a string.
     */
   def pathBuilders: List[PathBuilder]
 
   /**
-    * Function applied after a string is successfully resolved to a java.nio.Path
+    * Function applied after a string is successfully resolved to a Path
     */
   def postMapping(path: Path): Path = path
 
   /**
-    * Function applied before a string is attempted to be resolved to a java.nio.Path
+    * Function applied before a string is attempted to be resolved to a Path
     */
   def preMapping(string: String): String = string
 
   /**
-    * Attempts to build a java.nio.Path from a String
+    * Attempts to build a Path from a String
     */
   def buildPath(string: String): Path = PathFactory.buildPath(string, pathBuilders, preMapping, postMapping)
-
-  /**
-    * Attempts to build a better.files.File from a String
-    */
-  def buildFile(string: String): File = PathFactory.buildFile(string, pathBuilders, preMapping, postMapping)
 }
 
 object PathFactory {
   /**
-    * Attempts to build a java.nio.Path from a String
+    * Attempts to build a Path from a String
     */
   def buildPath(string: String,
                 pathBuilders: List[PathBuilder],
@@ -49,11 +40,4 @@ object PathFactory {
       throw PathParsingException(s"Could not find suitable filesystem among $pathBuilderNames to parse $string.")
     }
   }
-
-  def buildFile(string: String,
-                pathBuilders: List[PathBuilder],
-                preMapping: String => String = identity[String],
-                postMapping: Path => Path = identity[Path]): File = File(buildPath(string, pathBuilders, preMapping, postMapping))
-
-  def pathPlusSuffix(path: File, suffix: String): File = path.sibling(s"${path.name}.$suffix")
 }
diff --git a/core/src/main/scala/cromwell/core/path/PathImplicits.scala b/core/src/main/scala/cromwell/core/path/PathImplicits.scala
deleted file mode 100644
index fa5533ef3..000000000
--- a/core/src/main/scala/cromwell/core/path/PathImplicits.scala
+++ /dev/null
@@ -1,17 +0,0 @@
-package cromwell.core.path
-
-import java.nio.file.Path
-
-object PathImplicits {
-  implicit class EnhancedPath(val path: Path) extends AnyVal {
-    def swapExt(oldExt: String, newExt: String): Path = {
-      path.getFileSystem.getPath(s"${path.toString.stripSuffix(oldExt)}$newExt")
-    }
-
-    def untailed = UntailedWriter(path)
-
-    def tailed(tailedSize: Int) = TailedWriter(path, tailedSize)
-
-    def toRealString: String = java.net.URLDecoder.decode(path.toUri.toString, "UTF-8")
-  }
-}
diff --git a/core/src/main/scala/cromwell/core/path/PathObjectMethods.scala b/core/src/main/scala/cromwell/core/path/PathObjectMethods.scala
new file mode 100644
index 000000000..d109faa79
--- /dev/null
+++ b/core/src/main/scala/cromwell/core/path/PathObjectMethods.scala
@@ -0,0 +1,19 @@
+package cromwell.core.path
+
+/**
+  * Overrides java.lang.Object methods for Path
+  */
+trait PathObjectMethods {
+  self: Path =>
+
+  override def toString: String = pathAsString
+
+  override def equals(obj: Any) = {
+    obj match {
+      case other: Path => nioPathPrivate == other.nioPathPrivate
+      case _ => false
+    }
+  }
+
+  override def hashCode = nioPathPrivate.hashCode()
+}
diff --git a/core/src/main/scala/cromwell/core/path/PathWriter.scala b/core/src/main/scala/cromwell/core/path/PathWriter.scala
index e24fb74fe..5a6028101 100644
--- a/core/src/main/scala/cromwell/core/path/PathWriter.scala
+++ b/core/src/main/scala/cromwell/core/path/PathWriter.scala
@@ -1,20 +1,16 @@
 package cromwell.core.path
 
 import java.io.Writer
-import java.nio.file.Path
 
 import scala.collection.immutable.Queue
 
-
-
 /**
   * Used with a `ProcessLogger`, writes lines with a newline.
   */
 trait PathWriter {
-  import better.files._
 
   val path: Path
-  lazy val writer: Writer = File(path).newBufferedWriter
+  lazy val writer: Writer = path.newBufferedWriter
 
   /**
     * Passed to `ProcessLogger` to add a new line.
@@ -73,4 +69,3 @@ case class TailedWriter(path: Path, tailedSize: Int) extends PathWriter {
     }
   }
 }
-
diff --git a/core/src/main/scala/cromwell/core/path/package.scala b/core/src/main/scala/cromwell/core/path/package.scala
new file mode 100644
index 000000000..27dcbb53d
--- /dev/null
+++ b/core/src/main/scala/cromwell/core/path/package.scala
@@ -0,0 +1,5 @@
+package cromwell.core
+
+package object path {
+  type NioPath = java.nio.file.Path
+}
diff --git a/core/src/main/scala/cromwell/util/FileUtil.scala b/core/src/main/scala/cromwell/util/FileUtil.scala
deleted file mode 100644
index 28cd9d072..000000000
--- a/core/src/main/scala/cromwell/util/FileUtil.scala
+++ /dev/null
@@ -1,39 +0,0 @@
-package cromwell.util
-
-import java.nio.file.Path
-
-import better.files._
-import wdl4s.values.Hashable
-
-import scala.annotation.tailrec
-import scala.util.{Failure, Success, Try}
-
-object FileUtil {
-  def swapExt(filePath: String, oldExt: String, newExt: String): String = {
-    filePath.stripSuffix(oldExt) + newExt
-  }
-
-  def parseTsv(tsv: String): Try[Array[Array[String]]] = {
-    val table = tsv.split("\n").map(_.split("\t"))
-    table.map(_.length).toSet match {
-      case s if s.size > 1 => Failure(new UnsupportedOperationException("TSV is not uniform"))
-      case _ => Success(table)
-    }
-  }
-
-  implicit class EnhancedFile(val file: Path) extends AnyVal with Hashable {
-    def md5Sum: String = File(file).md5.toLowerCase // toLowerCase for backwards compatibility
-  }
-
-  @tailrec
-  final private def followSymlinks(file: better.files.File): better.files.File = {
-    file.symbolicLink match {
-      case Some(target) => followSymlinks(target)
-      case None => file
-    }
-  }
-
-  implicit class EnvenBetterFile(val file: better.files.File) extends AnyVal {
-    def followSymlinks = FileUtil.followSymlinks(file)
-  }
-}
diff --git a/core/src/test/scala/cromwell/core/path/DefaultPathBuilderSpec.scala b/core/src/test/scala/cromwell/core/path/DefaultPathBuilderSpec.scala
new file mode 100644
index 000000000..06bb45700
--- /dev/null
+++ b/core/src/test/scala/cromwell/core/path/DefaultPathBuilderSpec.scala
@@ -0,0 +1,433 @@
+package cromwell.core.path
+
+import cromwell.util.TestFileUtil
+import org.scalatest.prop.Tables.Table
+import org.scalatest.{FlatSpecLike, Matchers, Suite}
+
+class DefaultPathBuilderSpec extends Suite with FlatSpecLike with Matchers with PathBuilderSpecUtils with TestFileUtil {
+
+  private val pwd = BetterFileMethods.Cmds.pwd
+  private val parentOption = Option(pwd.parent)
+  private val grandParentOption = parentOption.flatMap(parent => Option(parent.parent))
+
+  private val pwdPath = pwd.pathAsString
+  private val pwdPathPrefix = if (pwd.pathAsString == "/") "" else pwdPath
+
+  private val parentPath = parentOption.map(_.pathAsString).orNull
+  private val grandParentPath = grandParentOption.map(_.pathAsString).orNull
+
+  private val pwdFileNameOption = Option(pwd.getFileName)
+  private val pwdName = pwdFileNameOption.map(_.pathAsString).getOrElse("")
+
+  private val parentFileNameOption = parentOption.flatMap(parent => Option(parent.getFileName))
+  private val parentName = parentFileNameOption.map(_.pathAsString).getOrElse("")
+
+  behavior of "DefaultPathBuilder"
+
+  it should "generate a md5 hash from a file" in {
+    val f1 = createCannedFile("file1", "content")
+    val f2 = createCannedFile("file2", "content")
+    val f3 = createCannedFile("file3", "other content")
+    f1.md5 should be("9A0364B9E99BB480DD25E1F0284C8555")
+    f1.md5 should be(f2.md5)
+    f1.md5 shouldNot be(f3.md5)
+  }
+
+  it should behave like truncateCommonRoots(DefaultPathBuilder, pathsToTruncate)
+
+  goodPaths foreach { goodPath =>
+    it should behave like buildGoodPath(DefaultPathBuilder, goodPath)
+  }
+
+  badPaths foreach { badPath =>
+    it should behave like buildBadPath(DefaultPathBuilder, badPath)
+  }
+
+  private def pathsToTruncate = Table(
+    ("context", "file", "relative"),
+    ("/", "/path/to/file", "path/to/file"),
+    ("/path/to/my/dir", "/path/to/my/dir/file", "file"),
+    ("/path/to/my/dir", "/path/to/my/dir//file", "file"),
+    ("/path/to/my//dir", "/path/to/my/dir/file", "file"),
+    ("/path/to/my//dir", "/path/to/my/dir//file", "file"),
+    ("/path/to/my/dir", "/path/./to/my/dir/file", "./to/my/dir/file"),
+    ("/path/to/my/dir/with/file", "/path/to/other/dir/with/file", "other/dir/with/file")
+  )
+
+  private def goodPaths = Seq(
+
+    // Normal paths, not normalized
+
+    GoodPath(
+      description = "a path",
+      path = "/hello/world",
+      normalize = false,
+      pathAsString = "/hello/world",
+      pathWithoutScheme = "/hello/world",
+      parent = "/hello",
+      getParent = "/hello",
+      root = "/",
+      name = "world",
+      getFileName = "world",
+      toUriHost = null,
+      toUriPath = "/",
+      toUriStartsWith = "file:///",
+      toUriEndsWith = "/hello/world",
+      getNameCount = 2,
+      isAbsolute = true,
+      isDirectory = false),
+
+    GoodPath(
+      description = "a relative path",
+      path = "hello/world",
+      normalize = false,
+      pathAsString = "hello/world",
+      pathWithoutScheme = s"$pwdPathPrefix/hello/world",
+      parent = pwdPathPrefix + "/hello",
+      getParent = "hello",
+      root = "/",
+      name = "world",
+      getFileName = "world",
+      toUriHost = null,
+      toUriPath = "/",
+      toUriStartsWith = "file:///",
+      toUriEndsWith = "/hello/world",
+      getNameCount = 2,
+      isAbsolute = false,
+      isDirectory = false),
+
+    GoodPath(
+      description = "a path with spaces",
+      path = "/hello/world/with spaces",
+      normalize = false,
+      pathAsString = "/hello/world/with spaces",
+      pathWithoutScheme = "/hello/world/with spaces",
+      parent = "/hello/world",
+      getParent = "/hello/world",
+      root = "/",
+      name = "with spaces",
+      getFileName = "with spaces",
+      toUriHost = null,
+      toUriPath = "/",
+      toUriStartsWith = "file:///",
+      toUriEndsWith = "/hello/world/with%20spaces",
+      getNameCount = 3,
+      isAbsolute = true,
+      isDirectory = false),
+
+    GoodPath(
+      description = "a path with encode spaces",
+      path = "/hello/world/encoded%20spaces",
+      normalize = false,
+      pathAsString = "/hello/world/encoded%20spaces",
+      pathWithoutScheme = "/hello/world/encoded%20spaces",
+      parent = "/hello/world",
+      getParent = "/hello/world",
+      root = "/",
+      name = "encoded%20spaces",
+      getFileName = "encoded%20spaces",
+      toUriHost = null,
+      toUriPath = "/",
+      toUriStartsWith = "file:///",
+      toUriEndsWith = "/hello/world/encoded%2520spaces",
+      getNameCount = 3,
+      isAbsolute = true,
+      isDirectory = false),
+
+    GoodPath(
+      description = "a path with non-ascii characters",
+      path = "/hello/world/with non ascii £€",
+      normalize = false,
+      pathAsString = "/hello/world/with non ascii £€",
+      pathWithoutScheme = "/hello/world/with non ascii £€",
+      parent = "/hello/world",
+      getParent = "/hello/world",
+      root = "/",
+      name = "with non ascii £€",
+      getFileName = "with non ascii £€",
+      toUriHost = null,
+      toUriPath = "/",
+      toUriStartsWith = "file:///",
+      toUriEndsWith = "/hello/world/with%20non%20ascii%20%C2%A3%E2%82%AC",
+      getNameCount = 3,
+      isAbsolute = true,
+      isDirectory = false),
+
+    // Special paths
+
+    GoodPath(
+      description = "an empty path",
+      path = "",
+      normalize = false,
+      pathAsString = "",
+      pathWithoutScheme = pwdPath,
+      parent = parentPath,
+      getParent = null,
+      root = "/",
+      name = pwdName,
+      getFileName = "",
+      toUriHost = null,
+      toUriPath = "/",
+      toUriStartsWith = "file:///",
+      toUriEndsWith = "/",
+      getNameCount = 1,
+      isAbsolute = false,
+      isDirectory = true),
+
+    GoodPath(
+      description = "a path from /",
+      path = "/",
+      normalize = false,
+      pathAsString = "/",
+      pathWithoutScheme = "/",
+      parent = null,
+      getParent = null,
+      root = "/",
+      name = "",
+      getFileName = null,
+      toUriHost = null,
+      toUriPath = "/",
+      toUriStartsWith = "file:///",
+      toUriEndsWith = "file:///",
+      getNameCount = 0,
+      isAbsolute = true,
+      isDirectory = true),
+
+    GoodPath(
+      description = "a path from .",
+      path = ".",
+      normalize = false,
+      pathAsString = ".",
+      pathWithoutScheme = s"$pwdPathPrefix/.",
+      parent = parentPath,
+      getParent = null,
+      root = "/",
+      name = pwdName,
+      getFileName = ".",
+      toUriHost = null,
+      toUriPath = "/",
+      toUriStartsWith = "file:///",
+      toUriEndsWith = "/./",
+      getNameCount = 1,
+      isAbsolute = false,
+      isDirectory = true),
+
+    GoodPath(
+      description = "a path from ..",
+      path = "..",
+      normalize = false,
+      pathAsString = "..",
+      pathWithoutScheme = s"$pwdPathPrefix/..",
+      parent = grandParentPath,
+      getParent = null,
+      root = "/",
+      name = parentName,
+      getFileName = "..",
+      toUriHost = null,
+      toUriPath = "/",
+      toUriStartsWith = "file:///",
+      toUriEndsWith = "/../",
+      getNameCount = 1,
+      isAbsolute = false,
+      isDirectory = true),
+
+    GoodPath(
+      description = "a path including .",
+      path = "/hello/world/with/./dots",
+      normalize = false,
+      pathAsString = "/hello/world/with/./dots",
+      pathWithoutScheme = "/hello/world/with/./dots",
+      parent = "/hello/world/with",
+      getParent = "/hello/world/with/.",
+      root = "/",
+      name = "dots",
+      getFileName = "dots",
+      toUriHost = null,
+      toUriPath = "/",
+      toUriStartsWith = "file:///",
+      toUriEndsWith = "/hello/world/with/./dots",
+      getNameCount = 5,
+      isAbsolute = true,
+      isDirectory = false),
+
+    GoodPath(
+      description = "a path including ..",
+      path = "/hello/world/with/../dots",
+      normalize = false,
+      pathAsString = "/hello/world/with/../dots",
+      pathWithoutScheme = "/hello/world/with/../dots",
+      parent = "/hello/world",
+      getParent = "/hello/world/with/..",
+      root = "/",
+      name = "dots",
+      getFileName = "dots",
+      toUriHost = null,
+      toUriPath = "/",
+      toUriStartsWith = "file:///",
+      toUriEndsWith = "/hello/world/with/../dots",
+      getNameCount = 5,
+      isAbsolute = true,
+      isDirectory = false),
+
+    // Normalized
+
+    GoodPath(
+      description = "an normalized empty path",
+      path = "",
+      normalize = true,
+      pathAsString = "",
+      pathWithoutScheme = pwdPath,
+      parent = parentPath,
+      getParent = null,
+      root = "/",
+      name = pwdName,
+      getFileName = "",
+      toUriHost = null,
+      toUriPath = "/",
+      toUriStartsWith = "file:///",
+      toUriEndsWith = "/",
+      getNameCount = 1,
+      isAbsolute = false,
+      isDirectory = true),
+
+    GoodPath(
+      description = "a normalized path from /",
+      path = "/",
+      normalize = true,
+      pathAsString = "/",
+      pathWithoutScheme = "/",
+      parent = null,
+      getParent = null,
+      root = "/",
+      name = "",
+      getFileName = null,
+      toUriHost = null,
+      toUriPath = "/",
+      toUriStartsWith = "file:///",
+      toUriEndsWith = "file:///",
+      getNameCount = 0,
+      isAbsolute = true,
+      isDirectory = true),
+
+    GoodPath(
+      description = "a normalized path from .",
+      path = ".",
+      normalize = true,
+      pathAsString = "",
+      pathWithoutScheme = pwdPath,
+      parent = parentPath,
+      getParent = null,
+      root = "/",
+      name = pwdName,
+      getFileName = "",
+      toUriHost = null,
+      toUriPath = "/",
+      toUriStartsWith = "file:///",
+      toUriEndsWith = "/",
+      getNameCount = 1,
+      isAbsolute = false,
+      isDirectory = true),
+
+    GoodPath(
+      description = "a normalized path from ..",
+      path = "..",
+      normalize = true,
+      pathAsString = "..",
+      pathWithoutScheme = s"$pwdPathPrefix/..",
+      parent = grandParentPath,
+      getParent = null,
+      root = "/",
+      name = parentName,
+      getFileName = "..",
+      toUriHost = null,
+      toUriPath = "/",
+      toUriStartsWith = "file:///",
+      toUriEndsWith = "/../",
+      getNameCount = 1,
+      isAbsolute = false,
+      isDirectory = true),
+
+    GoodPath(
+      description = "a normalized path including a .",
+      path = "/hello/world/with/./dots",
+      normalize = true,
+      pathAsString = "/hello/world/with/dots",
+      pathWithoutScheme = "/hello/world/with/dots",
+      parent = "/hello/world/with",
+      getParent = "/hello/world/with",
+      root = "/",
+      name = "dots",
+      getFileName = "dots",
+      toUriHost = null,
+      toUriPath = "/",
+      toUriStartsWith = "file:///",
+      toUriEndsWith = "/hello/world/with/dots",
+      getNameCount = 4,
+      isAbsolute = true,
+      isDirectory = false),
+
+    GoodPath(
+      description = "a normalized path including ..",
+      path = "/hello/world/with/../dots",
+      normalize = true,
+      pathAsString = "/hello/world/dots",
+      pathWithoutScheme = "/hello/world/dots",
+      parent = "/hello/world",
+      getParent = "/hello/world",
+      root = "/",
+      name = "dots",
+      getFileName = "dots",
+      toUriHost = null,
+      toUriPath = "/",
+      toUriStartsWith = "file:///",
+      toUriEndsWith = "/hello/world/dots",
+      getNameCount = 3,
+      isAbsolute = true,
+      isDirectory = false),
+
+    // URI
+
+    GoodPath(
+      description = "a path from a file uri",
+      path = "file:///hello/world",
+      normalize = false,
+      pathAsString = "/hello/world",
+      pathWithoutScheme = "/hello/world",
+      parent = "/hello",
+      getParent = "/hello",
+      root = "/",
+      name = "world",
+      getFileName = "world",
+      toUriHost = null,
+      toUriPath = "/",
+      toUriStartsWith = "file:///",
+      toUriEndsWith = "/hello/world",
+      getNameCount = 2,
+      isAbsolute = true,
+      isDirectory = false),
+
+    GoodPath(
+      description = "a path from a file uri with encoded spaces",
+      path = "file:///hello/world/encoded%20spaces",
+      normalize = false,
+      pathAsString = "/hello/world/encoded%20spaces",
+      pathWithoutScheme = "/hello/world/encoded%20spaces",
+      parent = "/hello/world",
+      getParent = "/hello/world",
+      root = "/",
+      name = "encoded%20spaces",
+      getFileName = "encoded%20spaces",
+      toUriHost = null,
+      toUriPath = "/",
+      toUriStartsWith = "file:///",
+      toUriEndsWith = "/hello/world/encoded%2520spaces",
+      getNameCount = 3,
+      isAbsolute = true,
+      isDirectory = false)
+  )
+
+  private def badPaths = Seq(
+    BadPath("a https path", "https://hello/world", "Cannot build a local path from https://hello/world"),
+    BadPath("a gcs path", "gs://hello/world", "Cannot build a local path from gs://hello/world")
+  )
+}
diff --git a/core/src/test/scala/cromwell/core/path/MappedPathBuilderSpec.scala b/core/src/test/scala/cromwell/core/path/MappedPathBuilderSpec.scala
new file mode 100644
index 000000000..f34881229
--- /dev/null
+++ b/core/src/test/scala/cromwell/core/path/MappedPathBuilderSpec.scala
@@ -0,0 +1,101 @@
+package cromwell.core.path
+
+import org.scalatest.prop.Tables.Table
+import org.scalatest.{FlatSpecLike, Matchers, Suite}
+
+class MappedPathBuilderSpec extends Suite  with FlatSpecLike with Matchers with PathBuilderSpecUtils {
+  behavior of "MappedPathBuilder"
+
+  it should behave like truncateCommonRoots(prefixedPathBuilder, pathsToTruncate)
+
+  goodPaths foreach { goodPath =>
+    it should behave like buildGoodPath(prefixedPathBuilder, goodPath)
+  }
+
+  badPaths foreach { badPath =>
+    it should behave like buildBadPath(prefixedPathBuilder, badPath)
+  }
+
+  private def pathsToTruncate = Table(
+    ("context", "file", "relative"),
+    ("gs://bucket", "gs://bucket/path/to/file", "path/to/file"),
+    ("gs://bucket/path/to/my/dir", "gs://bucket/path/to/my/dir/file", "file"),
+    ("gs://bucket/path/to/my/dir", "gs://bucket/path/to/my/dir//file", "file"),
+    ("gs://bucket/path/to/my//dir", "gs://bucket/path/to/my/dir/file", "file"),
+    ("gs://bucket/path/to/my//dir", "gs://bucket/path/to/my/dir//file", "file"),
+    ("gs://bucket/path/to/my/dir", "gs://bucket/path/./to/my/dir/file", "./to/my/dir/file"),
+    ("gs://bucket/path/to/my/dir/with/file", "gs://bucket/path/to/other/dir/with/file", "other/dir/with/file")
+  )
+
+  private def goodPaths = Seq(
+    GoodPath(
+      description = "a prefixed path",
+      path = "gs://hello/world",
+      normalize = false,
+      pathAsString = "/my/mapped/dir/hello/world",
+      pathWithoutScheme = "hello/world",
+      parent = "/my/mapped/dir/hello",
+      getParent = "/my/mapped/dir/hello",
+      root = "/",
+      name = "world",
+      getFileName = "world",
+      toUriHost = null,
+      toUriPath = "/my/mapped/dir/hello/world",
+      toUriStartsWith = "file:///",
+      toUriEndsWith = "/my/mapped/dir/hello/world",
+      getNameCount = 5,
+      isAbsolute = true,
+      isDirectory = false),
+
+    GoodPath(
+      description = "an empty prefixed path",
+      path = "gs://",
+      normalize = false,
+      pathAsString = "/my/mapped/dir",
+      pathWithoutScheme = "",
+      parent = "/my/mapped",
+      getParent = "/my/mapped",
+      root = "/",
+      name = "dir",
+      getFileName = "dir",
+      toUriHost = null,
+      toUriPath = "/my/mapped/dir",
+      toUriStartsWith = "file:///",
+      toUriEndsWith = "/my/mapped/dir",
+      getNameCount = 3,
+      isAbsolute = true,
+      isDirectory = false),
+
+    GoodPath(
+      description = "a prefixed path starting with .",
+      path = "gs://./hello/world",
+      normalize = false,
+      pathAsString = "/my/mapped/dir/./hello/world",
+      pathWithoutScheme = "./hello/world",
+      parent = "/my/mapped/dir/hello",
+      getParent = "/my/mapped/dir/./hello",
+      root = "/",
+      name = "world",
+      getFileName = "world",
+      toUriHost = null,
+      toUriPath = "/my/mapped/dir/./hello/world",
+      toUriStartsWith = "file:///",
+      toUriEndsWith = "/my/mapped/dir/./hello/world",
+      getNameCount = 6,
+      isAbsolute = true,
+      isDirectory = false)
+  )
+
+  private def badPaths = Seq(
+    BadPath("an empty path", "", " must start with gs://"),
+    BadPath("a https path", "https://hello/world", "https://hello/world must start with gs://"),
+    BadPath("a file uri path", "file:///hello/world", "file:///hello/world must start with gs://"),
+    BadPath("a relative file path", "hello/world", "hello/world must start with gs://"),
+    BadPath("an absolute file path", "/hello/world", "/hello/world must start with gs://"),
+    BadPath("an mapped directory path", "/my/mapped/dir/hello/world", "/my/mapped/dir/hello/world must start with gs://")
+  )
+
+  private lazy val prefixedPathBuilder = {
+    new MappedPathBuilder("gs://", "/my/mapped/dir")
+  }
+}
diff --git a/core/src/test/scala/cromwell/core/path/PathBuilderSpecUtils.scala b/core/src/test/scala/cromwell/core/path/PathBuilderSpecUtils.scala
new file mode 100644
index 000000000..fffd582aa
--- /dev/null
+++ b/core/src/test/scala/cromwell/core/path/PathBuilderSpecUtils.scala
@@ -0,0 +1,126 @@
+package cromwell.core.path
+
+import org.scalatest.FlatSpecLike
+import org.scalatest.Matchers._
+import org.scalatest.prop.TableDrivenPropertyChecks._
+import org.scalatest.prop._
+
+case class GoodPath(description: String,
+                    path: String,
+                    normalize: Boolean,
+                    pathAsString: String,
+                    pathWithoutScheme: String,
+                    parent: String,
+                    getParent: String,
+                    root: String,
+                    name: String,
+                    getFileName: String,
+                    toUriHost: String,
+                    toUriPath: String,
+                    toUriStartsWith: String,
+                    toUriEndsWith: String,
+                    getNameCount: Int,
+                    isAbsolute: Boolean,
+                    isDirectory: Boolean)
+
+case class BadPath(description: String, path: String, exceptionMessage: String)
+
+/**
+  * Tests various methods of Path objects.
+  */
+trait PathBuilderSpecUtils {
+  this: FlatSpecLike =>
+
+  def truncateCommonRoots(builder: => PathBuilder, pathsToTruncate: TableFor3[String, String, String]) = {
+    behavior of s"PathCopier"
+
+    it should "truncate common roots" in {
+      forAll(pathsToTruncate) { (context, file, relative) =>
+        val contextPath = builder.build(context).get
+        val filePath = builder.build(file).get
+        val actual = PathCopier.truncateCommonRoot(contextPath, filePath)
+        actual should be(relative)
+      }
+    }
+  }
+
+  def buildGoodPath(builder: => PathBuilder, goodPath: GoodPath) = {
+    behavior of s"Building ${goodPath.description}"
+
+    lazy val path = {
+      val path = builder.build(goodPath.path).get
+      if (goodPath.normalize) path.normalize() else path
+    }
+
+    it should "match expected pathAsString" in
+      withClue(s"for path ${goodPath.path}${if (goodPath.normalize) " (normalized)" else ""}:") {
+        path.pathAsString should be(goodPath.pathAsString)
+      }
+
+    it should "match expected pathWithoutScheme" in
+      withClue(s"for path ${goodPath.path}${if (goodPath.normalize) " (normalized)" else ""}:") {
+        path.pathWithoutScheme should be(goodPath.pathWithoutScheme)
+      }
+
+    it should "match expected parent" in
+      withClue(s"for path ${goodPath.path}${if (goodPath.normalize) " (normalized)" else ""}:") {
+        Option(path.parent).map(_.pathAsString).orNull should be(goodPath.parent)
+      }
+
+    it should "match expected getParent" in
+      withClue(s"for path ${goodPath.path}${if (goodPath.normalize) " (normalized)" else ""}:") {
+        Option(path.getParent).map(_.pathAsString).orNull should be(goodPath.getParent)
+      }
+
+    it should "match expected root" in
+      withClue(s"for path ${goodPath.path}${if (goodPath.normalize) " (normalized)" else ""}:") {
+        Option(path.root).map(_.pathAsString).orNull should be(goodPath.root)
+      }
+
+    it should "match expected name" in
+      withClue(s"for path ${goodPath.path}${if (goodPath.normalize) " (normalized)" else ""}:") {
+        path.name should be(goodPath.name)
+      }
+
+    it should "match expected getFileName" in
+      withClue(s"for path ${goodPath.path}${if (goodPath.normalize) " (normalized)" else ""}:") {
+        Option(path.getFileName).map(_.pathAsString).orNull should be(goodPath.getFileName)
+      }
+
+    it should "match expected start of toUri" in
+      withClue(s"for path ${goodPath.path}${if (goodPath.normalize) " (normalized)" else ""}:") {
+        path.toUri.toString should startWith(goodPath.toUriStartsWith)
+      }
+
+    it should "match expected end of toUri" in
+      withClue(s"for path ${goodPath.path}${if (goodPath.normalize) " (normalized)" else ""}:") {
+        path.toUri.toString should endWith(goodPath.toUriEndsWith)
+      }
+
+    it should "match expected getNameCount" in
+      withClue(s"for path ${goodPath.path}${if (goodPath.normalize) " (normalized)" else ""}:") {
+        path.getNameCount should be(goodPath.getNameCount)
+      }
+
+    it should "match expected isAbsolute" in
+      withClue(s"for path ${goodPath.path}${if (goodPath.normalize) " (normalized)" else ""}:") {
+        path.isAbsolute should be(goodPath.isAbsolute)
+      }
+
+    it should "match expected isDirectory" in
+      withClue(s"for path ${goodPath.path}${if (goodPath.normalize) " (normalized)" else ""}:") {
+        path.isDirectory should be(goodPath.isDirectory)
+      }
+  }
+
+  def buildBadPath(builder: => PathBuilder, badPath: BadPath) = {
+    behavior of s"Building ${badPath.description}"
+
+    it should "fail to build a path" in
+      withClue(s"for path ${badPath.path}:") {
+        val exception = intercept[Exception](builder.build(badPath.path).get)
+        exception.getMessage should be(badPath.exceptionMessage)
+      }
+  }
+
+}
diff --git a/core/src/test/scala/cromwell/core/path/RetryableFileSystemProxySpec.scala b/core/src/test/scala/cromwell/core/path/proxy/RetryableFileSystemProxySpec.scala
similarity index 99%
rename from core/src/test/scala/cromwell/core/path/RetryableFileSystemProxySpec.scala
rename to core/src/test/scala/cromwell/core/path/proxy/RetryableFileSystemProxySpec.scala
index 88dbf463d..71b0a7c96 100644
--- a/core/src/test/scala/cromwell/core/path/RetryableFileSystemProxySpec.scala
+++ b/core/src/test/scala/cromwell/core/path/proxy/RetryableFileSystemProxySpec.scala
@@ -1,4 +1,4 @@
-package cromwell.core.path
+package cromwell.core.path.proxy
 
 import java.io.FileNotFoundException
 import java.nio.channels.SeekableByteChannel
@@ -8,7 +8,7 @@ import java.nio.file.spi.FileSystemProvider
 import java.nio.file.{DirectoryStream, OpenOption, Path, StandardOpenOption}
 import java.util.concurrent.TimeoutException
 
-import cromwell.core.path.proxy.RetryableFileSystemProviderProxy
+import cromwell.core.path.CustomRetryParams
 import cromwell.core.retry.Backoff
 import cromwell.core.{CromwellFatalException, TestKitSuite}
 import org.mockito.Matchers._
@@ -93,6 +93,8 @@ class RetryableFileSystemProxySpec extends TestKitSuite with FlatSpecLike with M
       }
     }
 
+    import java.nio.file.Path
+
     when(provider.move(any[Path], any[Path])).thenAnswer(answerUnit)
     when(provider.checkAccess(any[Path])).thenAnswer(answerUnit)
     when(provider.createDirectory(any[Path])).thenAnswer(answerUnit)
diff --git a/core/src/test/scala/cromwell/util/FileUtilSpec.scala b/core/src/test/scala/cromwell/util/FileUtilSpec.scala
deleted file mode 100644
index 8ea107dd7..000000000
--- a/core/src/test/scala/cromwell/util/FileUtilSpec.scala
+++ /dev/null
@@ -1,16 +0,0 @@
-package cromwell.util
-
-import org.scalatest.{Matchers, FlatSpec}
-import cromwell.util.FileUtil._
-
-class FileUtilSpec extends FlatSpec with Matchers with TestFileUtil {
-
-  it should "generate a md5 hash from a file" in {
-    val f1 = createCannedFile("file1", "content")
-    val f2 = createCannedFile("file2", "content")
-    val f3 = createCannedFile("file3", "other content")
-    f1.md5Sum should be("9a0364b9e99bb480dd25e1f0284c8555")
-    f1.md5Sum should be(f2.md5Sum)
-    f1.md5Sum shouldNot be(f3.md5Sum)
-  }
-}
diff --git a/core/src/test/scala/cromwell/util/SampleWdl.scala b/core/src/test/scala/cromwell/util/SampleWdl.scala
index b02b6db9d..0a685fdac 100644
--- a/core/src/test/scala/cromwell/util/SampleWdl.scala
+++ b/core/src/test/scala/cromwell/util/SampleWdl.scala
@@ -1,10 +1,9 @@
 package cromwell.util
 
-import java.nio.file.{Files, Path}
 import java.util.UUID
 
-import better.files._
-import cromwell.core.{WorkflowSourceFilesWithoutImports}
+import cromwell.core.WorkflowSourceFilesWithoutImports
+import cromwell.core.path.{DefaultPathBuilder, Path}
 import spray.json._
 import wdl4s._
 import wdl4s.types.{WdlArrayType, WdlStringType}
@@ -44,6 +43,7 @@ trait SampleWdl extends TestFileUtil {
       case i: WdlInteger => JsNumber(i.value)
       case f: WdlFloat => JsNumber(f.value)
       case f: WdlFile => JsString(f.value)
+      case p: Path => JsString(p.pathAsString)
     }
     def read(value: JsValue) = throw new NotImplementedError(s"Reading JSON not implemented: $value")
   }
@@ -55,7 +55,7 @@ trait SampleWdl extends TestFileUtil {
 
   def wdlJson: WdlJson = rawInputs.toJson.prettyPrint
 
-  def deleteFile(path: Path) = Files.delete(path)
+  def deleteFile(path: Path) = path.delete()
 }
 
 object SampleWdl {
@@ -585,14 +585,14 @@ object SampleWdl {
         |}
       """.stripMargin.replaceAll("RUNTIME", runtime)
 
-    val tempDir = File.newTemporaryDirectory("ArrayIO").path
-    val firstFile = createCannedFile(prefix = "first", contents = "foo\n", dir = Some(tempDir))
-    val secondFile = createCannedFile(prefix = "second", contents = "bar\nbaz\n", dir = Some(tempDir))
+    val tempDir = DefaultPathBuilder.createTempDirectory("ArrayIO")
+    val firstFile = createCannedFile(prefix = "first", contents = "foo\n", dir = Option(tempDir))
+    val secondFile = createCannedFile(prefix = "second", contents = "bar\nbaz\n", dir = Option(tempDir))
 
     override val rawInputs = Map(
-      "wf.find.root" -> tempDir.toString,
+      "wf.find.root" -> tempDir,
       "wf.find.pattern" -> "*.out", // createCannedFile makes files that have .out extension
-      "wf.files" -> Seq(firstFile.toString, secondFile.toString)
+      "wf.files" -> Seq(firstFile.pathAsString, secondFile.pathAsString)
     )
   }
 
@@ -863,8 +863,9 @@ object SampleWdl {
            |text file is 11
            |""".stripMargin
 
-    override lazy val rawInputs = Map("sc_test.do_prepare.input_file" -> createCannedFile("scatter",contents).toString,
-    "sc_test.do_scatter.salt" -> salt)
+    override lazy val rawInputs = Map(
+      "sc_test.do_prepare.input_file" -> createCannedFile("scatter", contents).pathAsString,
+      "sc_test.do_scatter.salt" -> salt)
   }
 
   object FileClobber extends SampleWdl {
@@ -881,14 +882,14 @@ object SampleWdl {
         |}
       """.stripMargin
 
-    val tempDir1 = Files.createTempDirectory("FileClobber1")
-    val tempDir2 = Files.createTempDirectory("FileClobber2")
+    val tempDir1 = DefaultPathBuilder.createTempDirectory("FileClobber1")
+    val tempDir2 = DefaultPathBuilder.createTempDirectory("FileClobber2")
     val firstFile = createFile(name = "file.txt", contents = "first file.txt", dir = tempDir1)
     val secondFile = createFile(name = "file.txt", contents = "second file.txt", dir = tempDir2)
 
     override val rawInputs = Map(
-      "two.x.in" -> firstFile.toString,
-      "two.y.in" -> secondFile.toString
+      "two.x.in" -> firstFile.pathAsString,
+      "two.y.in" -> secondFile.pathAsString
     )
   }
 
@@ -920,7 +921,7 @@ object SampleWdl {
     private val fileContents = s"foo bar baz"
 
     override val rawInputs: WorkflowRawInputs = Map(
-      "file_passing.f" -> createCannedFile("canned", fileContents).toString
+      "file_passing.f" -> createCannedFile("canned", fileContents).pathAsString
     )
   }
 
@@ -964,7 +965,7 @@ object SampleWdl {
     private val fileContents = s"foo bar baz"
 
     override val rawInputs: WorkflowRawInputs = Map(
-      "file_passing.f" -> createCannedFile("canned", fileContents).toString,
+      "file_passing.f" -> createCannedFile("canned", fileContents).pathAsString,
       "file_passing.a.salt" -> salt,
       "file_passing.b.salt" -> salt
     )
@@ -1014,8 +1015,8 @@ object SampleWdl {
 
     override val rawInputs = {
       Map(
-        "w.array_file" -> createCannedFile("array.txt", CannedArray).toString,
-        "w.map_file" -> createCannedFile("map.txt", CannedMap).toString
+        "w.array_file" -> createCannedFile("array.txt", CannedArray).pathAsString,
+        "w.map_file" -> createCannedFile("map.txt", CannedMap).pathAsString
       )
     }
   }
@@ -1043,18 +1044,18 @@ object SampleWdl {
         |}
       """.stripMargin
 
-    val tempDir = Files.createTempDirectory("ArrayOfArray")
-    val firstFile = createCannedFile(prefix = "first", contents = "foo\n", dir = Some(tempDir))
-    val secondFile = createCannedFile(prefix = "second", contents = "bar\nbaz\n", dir = Some(tempDir))
-    val thirdFile = createCannedFile(prefix = "third", contents = "third\n", dir = Some(tempDir))
-    val fourthFile = createCannedFile(prefix = "fourth", contents = "fourth\n", dir = Some(tempDir))
+    val tempDir = DefaultPathBuilder.createTempDirectory("ArrayOfArray")
+    val firstFile = createCannedFile(prefix = "first", contents = "foo\n", dir = Option(tempDir))
+    val secondFile = createCannedFile(prefix = "second", contents = "bar\nbaz\n", dir = Option(tempDir))
+    val thirdFile = createCannedFile(prefix = "third", contents = "third\n", dir = Option(tempDir))
+    val fourthFile = createCannedFile(prefix = "fourth", contents = "fourth\n", dir = Option(tempDir))
 
     override val rawInputs = Map(
       "wf.nested_file" ->
         WdlArray(WdlArrayType(WdlArrayType(WdlStringType)),
         Seq(
-          WdlArray(WdlArrayType(WdlStringType), Seq(firstFile.toString, secondFile.toString).map(WdlString)),
-          WdlArray(WdlArrayType(WdlStringType), Seq(thirdFile.toString, fourthFile.toString).map(WdlString))
+          WdlArray(WdlArrayType(WdlStringType), Seq(firstFile.pathAsString, secondFile.pathAsString).map(WdlString)),
+          WdlArray(WdlArrayType(WdlStringType), Seq(thirdFile.pathAsString, fourthFile.pathAsString).map(WdlString))
         )
       )
     )
@@ -1088,13 +1089,13 @@ object SampleWdl {
         |}
       """.stripMargin.replaceAll("RUNTIME", runtime)
 
-    val tempDir = Files.createTempDirectory("CallCachingHashingWdl")
-    val cannedFile = createCannedFile(prefix = "canned", contents = "file contents", dir = Some(tempDir))
+    val tempDir = DefaultPathBuilder.createTempDirectory("CallCachingHashingWdl")
+    val cannedFile = createCannedFile(prefix = "canned", contents = "file contents", dir = Option(tempDir))
     override val rawInputs = Map(
       "w.t.a" -> WdlInteger(1),
       "w.t.b" -> WdlFloat(1.1),
       "w.t.c" -> WdlString("foobar"),
-      "w.t.d" -> WdlFile(cannedFile.toString)
+      "w.t.d" -> WdlFile(cannedFile.pathAsString)
     )
   }
 
diff --git a/core/src/test/scala/cromwell/util/TestFileUtil.scala b/core/src/test/scala/cromwell/util/TestFileUtil.scala
index e02c485a3..a6bedd490 100644
--- a/core/src/test/scala/cromwell/util/TestFileUtil.scala
+++ b/core/src/test/scala/cromwell/util/TestFileUtil.scala
@@ -1,32 +1,30 @@
 package cromwell.util
 
-import java.nio.file.Path
 import java.nio.file.attribute.PosixFilePermission
 
-import better.files._
+import cromwell.core.path.{DefaultPathBuilder, Path}
 import wdl4s.values._
-import cromwell.core.path.FileImplicits._
 
 trait TestFileUtil {
   def createCannedFile(prefix: String, contents: String, dir: Option[Path] = None): Path = {
     val suffix = ".out"
-    val tempFile = File.newTemporaryFile(prefix, suffix, dir.map(File.apply))
+    val tempFile = DefaultPathBuilder.createTempFile(prefix, suffix, dir)
     tempFile.createPermissionedDirectories()
     tempFile.addPermission(PosixFilePermission.OTHERS_READ)
     tempFile.addPermission(PosixFilePermission.OTHERS_WRITE)
-    tempFile.write(contents).path
+    tempFile.write(contents)
   }
 
   def createFile(name: String, dir: Path, contents: String): Path = {
-    File(dir).createPermissionedDirectories()./(name).write(contents).path
+    dir.createPermissionedDirectories()./(name).write(contents)
   }
 }
 
 trait HashUtil extends TestFileUtil {
   // Files
-  val file1 = WdlFile(createCannedFile("refFile", "some content").toString)
-  val sameAsfile1 = WdlFile(createCannedFile("sameContent", "some content").toString)
-  val anotherFile = WdlFile(createCannedFile("differentContent", "different content").toString)
+  val file1 = WdlFile(createCannedFile("refFile", "some content").pathAsString)
+  val sameAsfile1 = WdlFile(createCannedFile("sameContent", "some content").pathAsString)
+  val anotherFile = WdlFile(createCannedFile("differentContent", "different content").pathAsString)
 
   // Strings
   val string1 = WdlString("some text")
diff --git a/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala b/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala
index 49685bf7e..ebf5df088 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala
@@ -1,20 +1,19 @@
 package cromwell.engine.workflow
 
-import java.nio.file.Path
 import java.util.UUID
 
 import akka.actor.FSM.{CurrentState, Transition}
 import akka.actor._
-import better.files._
 import cats.instances.try_._
 import cats.syntax.functor._
-import cromwell.core.retry.SimpleExponentialBackoff
-import cromwell.core._
 import cromwell.core.Dispatcher.EngineDispatcher
+import cromwell.core._
+import cromwell.core.path.Path
+import cromwell.core.retry.SimpleExponentialBackoff
 import cromwell.engine.workflow.SingleWorkflowRunnerActor._
 import cromwell.engine.workflow.WorkflowManagerActor.RetrieveNewWorkflows
-import cromwell.engine.workflow.workflowstore.{InMemoryWorkflowStore, WorkflowStoreEngineActor, WorkflowStoreSubmitActor}
 import cromwell.engine.workflow.workflowstore.WorkflowStoreActor.SubmitWorkflow
+import cromwell.engine.workflow.workflowstore.{InMemoryWorkflowStore, WorkflowStoreEngineActor, WorkflowStoreSubmitActor}
 import cromwell.jobstore.EmptyJobStoreActor
 import cromwell.server.CromwellRootActor
 import cromwell.services.metadata.MetadataService.{GetSingleWorkflowMetadataAction, GetStatus, WorkflowOutputs}
@@ -183,12 +182,12 @@ class SingleWorkflowRunnerActor(source: WorkflowSourceFilesCollection, metadataO
 
   private def outputMetadata(metadata: JsObject): Try[Unit] = {
     Try {
-      val path = File(metadataOutputPath.get)
+      val path = metadataOutputPath.get
       if (path.isDirectory) {
         log.error("Specified metadata path is a directory, should be a file: " + path)
       } else {
         log.info(s"$Tag writing metadata to $path")
-        path.createIfNotExists(asDirectory = false, createParents = true).write(metadata.prettyPrint)
+        path.createIfNotExists(createParents = true).write(metadata.prettyPrint)
       }
     } void
   }
diff --git a/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala b/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala
index 9caf91409..e92b537d6 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala
@@ -165,7 +165,7 @@ class WorkflowManagerActor(params: WorkflowManagerActorParams)
       val maxNewWorkflows = maxWorkflowsToLaunch min (maxWorkflowsRunning - stateData.workflows.size)
       params.workflowStore ! WorkflowStoreActor.FetchRunnableWorkflows(maxNewWorkflows)
       stay()
-    case Event(WorkflowStoreEngineActor.NoNewWorkflowsToStart, stateData) =>
+    case Event(WorkflowStoreEngineActor.NoNewWorkflowsToStart, _) =>
       log.debug("WorkflowStore provided no new workflows to start")
       scheduleNextNewWorkflowPoll()
       stay()
@@ -200,7 +200,7 @@ class WorkflowManagerActor(params: WorkflowManagerActorParams)
     /*
      Responses from services
      */
-    case Event(WorkflowFailedResponse(workflowId, inState, reasons), data) =>
+    case Event(WorkflowFailedResponse(workflowId, inState, reasons), _) =>
       log.error(s"$tag Workflow $workflowId failed (during $inState): ${expandFailureReasons(reasons)}")
       stay()
     /*
@@ -254,7 +254,7 @@ class WorkflowManagerActor(params: WorkflowManagerActorParams)
       context.actorOf(EngineStatsActor.props(data.workflows.values.toList, sndr), s"EngineStatsActor-${sndr.hashCode()}")
       stay()
     // Anything else certainly IS interesting:
-    case Event(unhandled, data) =>
+    case Event(unhandled, _) =>
       log.warning(s"$tag Unhandled message: $unhandled")
       stay()
   }
@@ -298,12 +298,11 @@ class WorkflowManagerActor(params: WorkflowManagerActorParams)
   }
 
   private def expandFailureReasons(reasons: Seq[Throwable]): String = {
-    import cromwell.core.path.PathImplicits._
     
     reasons map {
       case reason: ThrowableAggregation => expandFailureReasons(reason.throwables.toSeq)
       case reason: KnownJobFailureException =>
-        val stderrMessage = reason.stderrPath map { path => s"\nCheck the content of stderr for potential additional information: ${path.toRealString}" } getOrElse ""
+        val stderrMessage = reason.stderrPath map { path => s"\nCheck the content of stderr for potential additional information: ${path.pathAsString}" } getOrElse ""
         reason.getMessage + stderrMessage
       case reason =>
       reason.getMessage + "\n" + ExceptionUtils.getStackTrace(reason)
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/CopyWorkflowLogsActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/CopyWorkflowLogsActor.scala
index 920caf77a..6963280ff 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/CopyWorkflowLogsActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/CopyWorkflowLogsActor.scala
@@ -1,15 +1,13 @@
 package cromwell.engine.workflow.lifecycle
 
 import java.io.IOException
-import java.nio.file.Path
 
 import akka.actor.SupervisorStrategy.Restart
 import akka.actor.{Actor, ActorLogging, ActorRef, OneForOneStrategy, Props}
-import better.files._
 import cromwell.core.Dispatcher.IoDispatcher
 import cromwell.core._
 import cromwell.core.logging.WorkflowLogger
-import cromwell.core.path.FileImplicits._
+import cromwell.core.path.Path
 import cromwell.services.metadata.MetadataService.PutMetadataAction
 import cromwell.services.metadata.{MetadataEvent, MetadataKey, MetadataValue}
 
@@ -31,11 +29,11 @@ class CopyWorkflowLogsActor(serviceRegistryActor: ActorRef)
     with ActorLogging {
 
   def copyAndClean(src: Path, dest: Path) = {
-    File(dest).parent.createPermissionedDirectories()
+    dest.parent.createPermissionedDirectories()
 
-    File(src).copyTo(dest, overwrite = true)
+    src.copyTo(dest, overwrite = true)
     if (WorkflowLogger.isTemporary) {
-      File(src).delete()
+      src.delete()
     }
   }
 
@@ -44,8 +42,8 @@ class CopyWorkflowLogsActor(serviceRegistryActor: ActorRef)
       val workflowLogger = new WorkflowLogger(self.path.name, workflowId, Option(log))
 
       workflowLogger.workflowLogPath foreach { src =>
-        if (File(src).exists) {
-          val destPath = destinationDir.resolve(src.getFileName.toString)
+        if (src.exists) {
+          val destPath = destinationDir.resolve(src.name)
           workflowLogger.info(s"Copying workflow logs from $src to $destPath")
 
           copyAndClean(src, destPath)
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/CopyWorkflowOutputsActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/CopyWorkflowOutputsActor.scala
index 1dbf8b270..55229ee1d 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/CopyWorkflowOutputsActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/CopyWorkflowOutputsActor.scala
@@ -1,14 +1,12 @@
 package cromwell.engine.workflow.lifecycle
 
-import java.nio.file.Path
-
 import akka.actor.Props
 import cromwell.backend.BackendWorkflowFinalizationActor.{FinalizationResponse, FinalizationSuccess}
 import cromwell.backend.{AllBackendInitializationData, BackendConfigurationDescriptor, BackendInitializationData, BackendLifecycleActorFactory}
 import cromwell.core.Dispatcher.IoDispatcher
 import cromwell.core.WorkflowOptions._
 import cromwell.core._
-import cromwell.core.path.{PathCopier, PathFactory}
+import cromwell.core.path.{Path, PathCopier, PathFactory}
 import cromwell.engine.EngineWorkflowDescriptor
 import cromwell.engine.backend.{BackendConfiguration, CromwellBackends}
 import wdl4s.values.{WdlArray, WdlMap, WdlSingleFile, WdlValue}
@@ -57,7 +55,7 @@ class CopyWorkflowOutputsActor(workflowId: WorkflowId, val workflowDescriptor: E
       rootPath <- getBackendRootPath(backend, config).toSeq
       outputFiles = findFiles(workflowOutputs.values.map(_.wdlValue).toSeq)
       wdlFile <- outputFiles
-      wdlPath = rootPath.getFileSystem.getPath(wdlFile.value)
+      wdlPath = PathFactory.buildPath(wdlFile.value, pathBuilders)
     } yield (rootPath, wdlPath)
   }
 
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala
index ff415892f..6d3889a87 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala
@@ -1,9 +1,6 @@
 package cromwell.engine.workflow.lifecycle
 
-import java.nio.file.Files
-
 import akka.actor.{ActorRef, FSM, LoggingFSM, Props}
-import better.files.File
 import cats.data.NonEmptyList
 import cats.data.Validated._
 import cats.instances.list._
@@ -20,16 +17,17 @@ import cromwell.core._
 import cromwell.core.callcaching._
 import cromwell.core.labels.{Label, Labels}
 import cromwell.core.logging.WorkflowLogging
-import cromwell.core.path.PathBuilder
+import cromwell.core.path.BetterFileMethods.OpenOptions
+import cromwell.core.path.{DefaultPathBuilder, Path, PathBuilder}
 import cromwell.engine._
 import cromwell.engine.backend.CromwellBackends
 import cromwell.engine.workflow.lifecycle.MaterializeWorkflowDescriptorActor.{MaterializeWorkflowDescriptorActorData, MaterializeWorkflowDescriptorActorState}
-import cromwell.services.metadata.MetadataService.{PutMetadataAction, _}
+import cromwell.services.metadata.MetadataService._
 import cromwell.services.metadata.{MetadataEvent, MetadataKey, MetadataValue}
 import lenthall.exception.MessageAggregation
 import lenthall.validation.ErrorOr._
 import net.ceedubs.ficus.Ficus._
-import spray.json.{JsObject, _}
+import spray.json._
 import wdl4s._
 import wdl4s.expression.NoFunctions
 import wdl4s.values.{WdlString, WdlValue}
@@ -312,19 +310,16 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef,
     }
   }
 
-  private def validateImportsDirectory(zipContents: Array[Byte]): ErrorOr[File] = {
+  private def validateImportsDirectory(zipContents: Array[Byte]): ErrorOr[Path] = {
 
-    def makeZipFile(contents: Array[Byte]): Try[File] = Try {
-      val dependenciesPath = Files.createTempFile("", ".zip")
-      Files.write(dependenciesPath, contents)
+    def makeZipFile(contents: Array[Byte]): Try[Path] = Try {
+      DefaultPathBuilder.createTempFile("", ".zip").write(contents)(OpenOptions.default)
     }
 
-    def unZipFile(f: File) = Try {
+    def unZipFile(f: Path) = Try {
       val unzippedFile = f.unzip()
-      val unzippedFileContents = unzippedFile.toJava.listFiles().head
-
-      if (unzippedFileContents.isDirectory) File(unzippedFileContents.getPath)
-      else unzippedFile
+      val unzippedFileContents = unzippedFile.list.toSeq.head
+      if (unzippedFileContents.isDirectory) unzippedFileContents else unzippedFile
     }
 
     val importsFile = for {
@@ -334,25 +329,23 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef,
     } yield unzipped
 
     importsFile match {
-      case Success(unzippedDirectory: File) => unzippedDirectory.validNel
+      case Success(unzippedDirectory: Path) => unzippedDirectory.validNel
       case Failure(t) => t.getMessage.invalidNel
     }
   }
 
   private def validateNamespaceWithImports(w: WorkflowSourceFilesWithDependenciesZip): ErrorOr[WdlNamespaceWithWorkflow] = {
-    def getMetadatae(importsDir: File, prefix: String = ""): Seq[(String, File)] = {
+    def getMetadatae(importsDir: Path, prefix: String = ""): Seq[(String, Path)] = {
       importsDir.children.toSeq flatMap {
-        case f: File if f.isDirectory => getMetadatae(f, prefix + f.name + "/")
-        case f: File if f.name.endsWith(".wdl") => Seq((prefix + f.name, f))
+        case f: Path if f.isDirectory => getMetadatae(f, prefix + f.name + "/")
+        case f: Path if f.name.endsWith(".wdl") => Seq((prefix + f.name, f))
         case _ => Seq.empty
       }
     }
 
-    def writeMetadatae(importsDir: File) = {
-      import scala.collection.JavaConverters._
-
-      val wfImportEvents = getMetadatae(importsDir) map { case (name: String, f: File) =>
-        val contents = Files.readAllLines(f.path).asScala.mkString(System.lineSeparator())
+    def writeMetadatae(importsDir: Path) = {
+      val wfImportEvents = getMetadatae(importsDir) map { case (name: String, f: Path) =>
+        val contents = f.lines.mkString(System.lineSeparator())
         MetadataEvent(MetadataKey(workflowIdForLogging, None, WorkflowMetadataKeys.SubmissionSection, WorkflowMetadataKeys.SubmissionSection_Imports, name), MetadataValue(contents))
       }
       serviceRegistryActor ! PutMetadataAction(wfImportEvents)
@@ -360,10 +353,11 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef,
 
     validateImportsDirectory(w.importsZip) flatMap { importsDir =>
       writeMetadatae(importsDir)
+      val importsDirFile = better.files.File(importsDir.pathAsString) // For wdl4s better file compatibility
       val importResolvers: Seq[ImportResolver] = if (importLocalFilesystem) {
-        List(WdlNamespace.directoryResolver(importsDir), WdlNamespace.fileResolver)
+        List(WdlNamespace.directoryResolver(importsDirFile), WdlNamespace.fileResolver)
       } else {
-        List(WdlNamespace.directoryResolver(importsDir))
+        List(WdlNamespace.directoryResolver(importsDirFile))
       }
       val results = WdlNamespaceWithWorkflow.load(w.wdlSource, importResolvers)
       importsDir.delete(swallowIOExceptions = true)
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala
index 95cf3cbb8..f02339720 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala
@@ -1,13 +1,11 @@
 package cromwell.engine.workflow.lifecycle.execution.callcaching
 
-import java.nio.file.Path
-
 import cats.data.NonEmptyList
 import cromwell.backend.BackendJobExecutionActor.JobSucceededResponse
 import cromwell.core.ExecutionIndex.IndexEnhancedIndex
 import cromwell.core.WorkflowId
 import cromwell.core.callcaching.HashResult
-import cromwell.core.path.PathImplicits._
+import cromwell.core.path.Path
 import cromwell.core.simpleton.WdlValueSimpleton
 import cromwell.database.sql.SqlConverters._
 import cromwell.database.sql._
@@ -55,7 +53,7 @@ class CallCache(database: CallCachingSqlDatabase) {
 
     val jobDetritusToInsert: Iterable[CallCachingDetritusEntry] = {
       jobDetritus map {
-        case (fileName, filePath) => CallCachingDetritusEntry(fileName, filePath.toRealString.toClob)
+        case (fileName, filePath) => CallCachingDetritusEntry(fileName, filePath.pathAsString.toClob)
       }
     }
 
diff --git a/engine/src/test/scala/cromwell/ArrayWorkflowSpec.scala b/engine/src/test/scala/cromwell/ArrayWorkflowSpec.scala
index 0b1759716..5c97e856e 100644
--- a/engine/src/test/scala/cromwell/ArrayWorkflowSpec.scala
+++ b/engine/src/test/scala/cromwell/ArrayWorkflowSpec.scala
@@ -1,19 +1,16 @@
 package cromwell
 
-import java.nio.file.Files
-
 import akka.testkit._
-import better.files._
+import cromwell.core.path.DefaultPathBuilder
 import cromwell.util.SampleWdl
-import wdl4s.{ImportResolver, WdlNamespaceWithWorkflow}
 import wdl4s.expression.NoFunctions
 import wdl4s.types.{WdlArrayType, WdlFileType, WdlStringType}
 import wdl4s.values.{WdlArray, WdlFile, WdlInteger, WdlString}
-
+import wdl4s.{ImportResolver, WdlNamespaceWithWorkflow}
 
 class ArrayWorkflowSpec extends CromwellTestKitSpec {
-  val tmpDir = Files.createTempDirectory("ArrayWorkflowSpec")
-  val ns = WdlNamespaceWithWorkflow.load(SampleWdl.ArrayLiteral(tmpDir).wdlSource(""), Seq.empty[ImportResolver])
+  val tmpDir = DefaultPathBuilder.createTempDirectory("ArrayWorkflowSpec")
+  val ns = WdlNamespaceWithWorkflow.load(SampleWdl.ArrayLiteral(tmpDir).wdlSource(), Seq.empty[ImportResolver])
   val expectedArray = WdlArray(WdlArrayType(WdlFileType), Seq(WdlFile("f1"), WdlFile("f2"), WdlFile("f3")))
 
   "A task which contains a parameter " should {
@@ -38,7 +35,7 @@ class ArrayWorkflowSpec extends CromwellTestKitSpec {
       val expression = declaration.expression.getOrElse {
         fail("Expected an expression for declaration 'arr'")
       }
-      val value = expression.evaluate((s:String) => fail("No lookups"), NoFunctions).getOrElse {
+      val value = expression.evaluate((_: String) => fail("No lookups"), NoFunctions).getOrElse {
         fail("Expected expression for 'arr' to evaluate")
       }
       value shouldEqual WdlArray(WdlArrayType(WdlStringType), Seq(WdlString("f1"), WdlString("f2"), WdlString("f3")))
@@ -63,8 +60,8 @@ class ArrayWorkflowSpec extends CromwellTestKitSpec {
           )
         )
       )
-      val pwd = File(".")
-      val sampleWdl = SampleWdl.ArrayLiteral(pwd.path)
+      val pwd = DefaultPathBuilder.get(".")
+      val sampleWdl = SampleWdl.ArrayLiteral(pwd)
       runWdlAndAssertOutputs(
         sampleWdl,
         eventFilter = EventFilter.info(pattern = "Starting calls: wf.cat", occurrences = 1),
diff --git a/engine/src/test/scala/cromwell/CopyWorkflowOutputsSpec.scala b/engine/src/test/scala/cromwell/CopyWorkflowOutputsSpec.scala
index d8d910913..690e618b1 100644
--- a/engine/src/test/scala/cromwell/CopyWorkflowOutputsSpec.scala
+++ b/engine/src/test/scala/cromwell/CopyWorkflowOutputsSpec.scala
@@ -1,8 +1,7 @@
 package cromwell
 
-import java.nio.file.{Files, Paths}
-
 import akka.testkit.EventFilter
+import cromwell.core.path.DefaultPathBuilder
 import cromwell.util.SampleWdl
 import org.scalatest.prop.TableDrivenPropertyChecks._
 import org.scalatest.prop.Tables.Table
@@ -15,7 +14,7 @@ class CopyWorkflowOutputsSpec extends CromwellTestKitSpec {
     "copy workflow outputs" in {
       val workflowOutputsPath = "copy-workflow-outputs"
 
-      val tmpDir = Files.createTempDirectory(workflowOutputsPath).toAbsolutePath
+      val tmpDir = DefaultPathBuilder.createTempDirectory(workflowOutputsPath).toAbsolutePath
 
       val outputs = Table(
         ("call", "file"),
@@ -29,24 +28,23 @@ class CopyWorkflowOutputsSpec extends CromwellTestKitSpec {
         sampleWdl = SampleWdl.WorkflowOutputsWithFiles,
         eventFilter = EventFilter.info(
           pattern = "transition from FinalizingWorkflowState to WorkflowSucceededState", occurrences = 1),
-        runtime = "",
         workflowOptions = s""" { "final_workflow_outputs_dir": "$tmpDir" } """,
         expectedOutputs = Seq("A.out", "A.out2", "B.outs") map { o => ("wfoutputs." + o) -> CromwellTestKitSpec.AnyValueIsFine } toMap,
         allowOtherOutputs = false
       )
 
       forAll(outputs) { (call, file) =>
-        val path = tmpDir.resolve(Paths.get("wfoutputs", workflowId.id.toString, call, "execution", file))
+        val path = tmpDir / "wfoutputs" / workflowId.id.toString / call / "execution" / file
         path.toFile should exist
       }
-      val path = tmpDir.resolve(Paths.get("wfoutputs", workflowId.id.toString, "call-C", "execution", "out"))
+      val path = tmpDir / "wfoutputs" / workflowId.id.toString / "call-C" / "execution" / "out"
       path.toFile shouldNot exist
     }
 
     "copy scattered workflow outputs" in {
       val workflowOutputsPath = "copy-workflow-outputs"
 
-      val tmpDir = Files.createTempDirectory(workflowOutputsPath).toAbsolutePath
+      val tmpDir = DefaultPathBuilder.createTempDirectory(workflowOutputsPath).toAbsolutePath
 
       val shards = 0 to 9
       val outputNames = List("B1", "B2")
@@ -62,15 +60,14 @@ class CopyWorkflowOutputsSpec extends CromwellTestKitSpec {
         sampleWdl = SampleWdl.WorkflowScatterOutputsWithFileArrays,
         eventFilter = EventFilter.info(
           pattern = "transition from FinalizingWorkflowState to WorkflowSucceededState", occurrences = 1),
-        runtime = "",
         workflowOptions = s""" { "final_workflow_outputs_dir": "$tmpDir" } """,
         expectedOutputs = Map("wfoutputs.A.outs" -> CromwellTestKitSpec.AnyValueIsFine),
         allowOtherOutputs = false
       )
 
       forAll(outputs) { (call, file) =>
-        val path = tmpDir.resolve(Paths.get("wfoutputs", workflowId.id.toString, call, file))
-        Files.exists(path) shouldBe true
+        val path = tmpDir / "wfoutputs" / workflowId.id.toString / call / file
+        path.toFile should exist
       }
     }
   }
diff --git a/engine/src/test/scala/cromwell/CromwellTestKitSpec.scala b/engine/src/test/scala/cromwell/CromwellTestKitSpec.scala
index c8c526b68..ddfeafb07 100644
--- a/engine/src/test/scala/cromwell/CromwellTestKitSpec.scala
+++ b/engine/src/test/scala/cromwell/CromwellTestKitSpec.scala
@@ -1,6 +1,5 @@
 package cromwell
 
-import java.nio.file.Paths
 import java.util.UUID
 import java.util.concurrent.atomic.AtomicInteger
 
@@ -11,6 +10,8 @@ import com.typesafe.config.{Config, ConfigFactory}
 import cromwell.CromwellTestKitSpec._
 import cromwell.backend._
 import cromwell.core._
+import cromwell.core.path.BetterFileMethods.Cmds
+import cromwell.core.path.DefaultPathBuilder
 import cromwell.engine.backend.BackendConfigurationEntry
 import cromwell.engine.workflow.WorkflowManagerActor.RetrieveNewWorkflows
 import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheReadActor.{CacheLookupRequest, CacheResultMatchesForHashes}
@@ -180,6 +181,21 @@ object CromwellTestKitSpec {
     * the actual value was.
     */
   lazy val AnyValueIsFine: WdlValue = WdlString("Today you are you! That is truer than true! There is no one alive who is you-er than you!")
+
+  def replaceVariables(wdlValue: WdlValue, workflowId: WorkflowId): WdlValue = {
+    wdlValue match {
+      case WdlString(value) => WdlString(replaceVariables(value, workflowId))
+      case _ => wdlValue
+    }
+  }
+
+  def replaceVariables(value: String, workflowId: WorkflowId): String = {
+    val variables = Map("PWD" -> Cmds.pwd, "UUID" -> workflowId)
+    variables.foldLeft(value) {
+      case (result, (variableName, variableValue)) => result.replace(s"<<$variableName>>", s"$variableValue")
+    }
+  }
+
   lazy val DefaultConfig = ConfigFactory.load
   lazy val DefaultLocalBackendConfig = ConfigFactory.parseString(
     """
@@ -292,7 +308,8 @@ abstract class CromwellTestKitSpec(val twms: TestWorkflowManagerSystem = new Cro
 
   // Allow to use shouldEqual between 2 WdlValues while acknowledging for edge cases and checking for WdlType compatibilty
   implicit val wdlEquality = new Equality[WdlValue] {
-    def fileEquality(f1: String, f2: String) = Paths.get(f1).getFileName == Paths.get(f2).getFileName
+    def fileEquality(f1: String, f2: String) =
+      DefaultPathBuilder.get(f1).getFileName == DefaultPathBuilder.get(f2).getFileName
 
     override def areEqual(a: WdlValue, b: Any): Boolean = {
       val typeEquality = b match {
@@ -362,7 +379,7 @@ abstract class CromwellTestKitSpec(val twms: TestWorkflowManagerSystem = new Cro
 
     expectedOutputs foreach { case (outputFqn, expectedValue) =>
       val actualValue = outputs.getOrElse(outputFqn, throw OutputNotFoundException(outputFqn, actualOutputNames))
-      if (expectedValue != AnyValueIsFine) actualValue shouldEqual expectedValue
+      if (expectedValue != AnyValueIsFine) actualValue shouldEqual replaceVariables(expectedValue, workflowId)
     }
     if (!allowOtherOutputs) {
       outputs foreach { case (actualFqn, actualValue) =>
diff --git a/engine/src/test/scala/cromwell/MapWorkflowSpec.scala b/engine/src/test/scala/cromwell/MapWorkflowSpec.scala
index bc3b1f740..d66559a30 100644
--- a/engine/src/test/scala/cromwell/MapWorkflowSpec.scala
+++ b/engine/src/test/scala/cromwell/MapWorkflowSpec.scala
@@ -1,18 +1,18 @@
 package cromwell
 
 import akka.testkit._
-import better.files._
+import cromwell.core.path.DefaultPathBuilder
 import cromwell.util.SampleWdl
-import wdl4s.{ImportResolver, WdlNamespaceWithWorkflow}
 import wdl4s.expression.{NoFunctions, WdlFunctions}
 import wdl4s.types.{WdlFileType, WdlIntegerType, WdlMapType, WdlStringType}
 import wdl4s.values._
+import wdl4s.{ImportResolver, WdlNamespaceWithWorkflow}
 
 import scala.util.{Success, Try}
 
 class MapWorkflowSpec extends CromwellTestKitSpec {
-  private val pwd = File(".")
-  private val sampleWdl = SampleWdl.MapLiteral(pwd.path)
+  private val pwd = DefaultPathBuilder.get(".")
+  private val sampleWdl = SampleWdl.MapLiteral(pwd)
   val ns = WdlNamespaceWithWorkflow.load(sampleWdl.wdlSource(), Seq.empty[ImportResolver])
   val expectedMap = WdlMap(WdlMapType(WdlFileType, WdlStringType), Map(
     WdlFile("f1") -> WdlString("alice"),
@@ -23,7 +23,8 @@ class MapWorkflowSpec extends CromwellTestKitSpec {
 
   "A task which contains a parameter " should {
     "accept an array for the value" in {
-      val sampleWdl = SampleWdl.MapLiteral(pwd.path)
+      val sampleWdl = SampleWdl.MapLiteral(pwd)
+      val callDir = "<<PWD>>/cromwell-executions/wf/<<UUID>>/call-write_map/inputs<<PWD>>"
       runWdlAndAssertOutputs(
         sampleWdl = sampleWdl,
         EventFilter.info(pattern = "Starting calls: wf.read_map:NA:1, wf.write_map:NA:1", occurrences = 1),
@@ -32,9 +33,8 @@ class MapWorkflowSpec extends CromwellTestKitSpec {
             WdlString("x") -> WdlInteger(500),
             WdlString("y") -> WdlInteger(600),
             WdlString("z") -> WdlInteger(700)
-          )) //, TODO: We now process file paths used as keys. So file paths like 'f1' will actually be mapped
-          // to ${PWD}/cromwell-executions/wf/<id>/call-write_map/inputs${PWD}/f1
-          //"wf.write_map.contents" -> WdlString("f1\talice\nf2\tbob\nf3\tchuck")
+          )),
+          "wf.write_map.contents" -> WdlString(s"$callDir/f1\talice\n$callDir/f2\tbob\n$callDir/f3\tchuck")
         )
       )
       sampleWdl.cleanup()
@@ -71,7 +71,7 @@ class MapWorkflowSpec extends CromwellTestKitSpec {
       command shouldEqual "cat /test/map/path"
     }
     "Coerce Map[String, String] to Map[String, Int] when running the workflow" in {
-      val sampleWdl = SampleWdl.MapLiteral(pwd.path)
+      val sampleWdl = SampleWdl.MapLiteral(pwd)
       runWdlAndAssertOutputs(
         sampleWdl,
         eventFilter = EventFilter.info(pattern = "Starting calls: wf.read_map:NA:1, wf.write_map:NA:1", occurrences = 1),
diff --git a/engine/src/test/scala/cromwell/engine/EngineFunctionsSpec.scala b/engine/src/test/scala/cromwell/engine/EngineFunctionsSpec.scala
index 14ccbd6bd..cdefcaf6a 100644
--- a/engine/src/test/scala/cromwell/engine/EngineFunctionsSpec.scala
+++ b/engine/src/test/scala/cromwell/engine/EngineFunctionsSpec.scala
@@ -1,9 +1,7 @@
 package cromwell.engine
 
-import java.nio.file.Path
-
 import cromwell.backend.wdl.{ReadLikeFunctions, WriteFunctions}
-import cromwell.core.path.{DefaultPathBuilder, PathBuilder}
+import cromwell.core.path.{DefaultPathBuilder, Path, PathBuilder}
 import org.scalatest.prop.TableDrivenPropertyChecks._
 import org.scalatest.prop.Tables.Table
 import org.scalatest.{FlatSpec, Matchers}
@@ -24,7 +22,7 @@ class EngineFunctionsSpec extends FlatSpec with Matchers {
 
   def expectFailure(value: Try[WdlValue]) = value match {
     case Success(s) => fail(s"$s: Expected this function invocation to fail")
-    case Failure(ex) => // expected
+    case Failure(_) => // expected
   }
   "EngineFunctions" should "all initially be undefined" in {
     val stdFunctions = Seq(
@@ -39,9 +37,9 @@ class EngineFunctionsSpec extends FlatSpec with Matchers {
 
   "sub" should "replace a string according to a pattern" in {
     class TestEngineFn extends WdlStandardLibraryImpl {
-      override def glob(path: String, pattern: String): Seq[String] = ???
+      override def glob(path: String, pattern: String): Seq[String] = throw new NotImplementedError
       override def pathBuilders: List[PathBuilder] = List(DefaultPathBuilder)
-      override def writeDirectory: Path = ???
+      override def writeDirectory: Path = throw new NotImplementedError
     }
 
     val engineFn = new TestEngineFn
diff --git a/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala
index 73b65c5c5..f3b68539a 100644
--- a/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala
+++ b/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala
@@ -1,16 +1,15 @@
 package cromwell.engine.workflow
 
-import java.nio.file.Path
 import java.time.OffsetDateTime
 
 import akka.actor._
 import akka.pattern.ask
 import akka.testkit.TestKit
 import akka.util.Timeout
-import better.files._
 import com.typesafe.config.ConfigFactory
 import cromwell.CromwellTestKitSpec._
 import cromwell.core.WorkflowSourceFilesCollection
+import cromwell.core.path.{DefaultPathBuilder, Path}
 import cromwell.engine.backend.BackendSingletonCollection
 import cromwell.engine.workflow.SingleWorkflowRunnerActor.RunWorkflow
 import cromwell.engine.workflow.SingleWorkflowRunnerActorSpec._
@@ -32,13 +31,13 @@ import scala.util._
  *
  * Currently, as instance of the actor system are created via an instance of CromwellTestkitSpec, and the
  * SingleWorkflowRunnerActor also tests halting its actor system, each spec is currently in a separate instance of the
- * CromwellTestkitSpec.
+ * CromwellTestKitSpec.
  */
 object SingleWorkflowRunnerActorSpec {
 
-  def tempFile() = File.newTemporaryFile("metadata.", ".json")
+  def tempFile() = DefaultPathBuilder.createTempFile("metadata.", ".json")
 
-  def tempDir() = File.newTemporaryDirectory("metadata.dir.")
+  def tempDir() = DefaultPathBuilder.createTempDirectory("metadata.dir.")
 
   implicit class OptionJsValueEnhancer(val jsValue: Option[JsValue]) extends AnyVal {
     def toOffsetDateTime = OffsetDateTime.parse(jsValue.toStringValue)
@@ -116,7 +115,7 @@ class SingleWorkflowRunnerActorWithMetadataSpec extends SingleWorkflowRunnerActo
     within(TimeoutDuration) {
       singleWorkflowActor(
         sampleWdl = wdlFile,
-        outputFile = Option(metadataFile.path))
+        outputFile = Option(metadataFile))
         TestKit.shutdownActorSystem(system, TimeoutDuration)
     }
     val metadataFileContent = metadataFile.contentAsString
@@ -188,7 +187,7 @@ class SingleWorkflowRunnerActorWithMetadataOnFailureSpec extends SingleWorkflowR
     "fail to run a workflow and still output metadata" in {
       val testStart = OffsetDateTime.now
       within(TimeoutDuration) {
-        singleWorkflowActor(sampleWdl = GoodbyeWorld, outputFile = Option(metadataFile.path))
+        singleWorkflowActor(sampleWdl = GoodbyeWorld, outputFile = Option(metadataFile))
       }
       TestKit.shutdownActorSystem(system, TimeoutDuration)
 
@@ -240,7 +239,7 @@ class SingleWorkflowRunnerActorWithBadMetadataSpec extends SingleWorkflowRunnerA
   "A SingleWorkflowRunnerActor" should {
     "successfully run a workflow requesting a bad metadata path" in {
       within(TimeoutDuration) {
-        val runner = createRunnerActor(outputFile = Option(metadataDir.path))
+        val runner = createRunnerActor(outputFile = Option(metadataDir))
         waitForErrorWithException(s"Specified metadata path is a directory, should be a file: $metadataDir") {
           val futureResult = runner.ask(RunWorkflow)(30.seconds, implicitly)
           Await.ready(futureResult, Duration.Inf)
diff --git a/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala
index 646487e1f..1995b480b 100644
--- a/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala
+++ b/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala
@@ -1,22 +1,21 @@
 package cromwell.engine.workflow
 
-import java.nio.file.Paths
-
 import akka.actor.{Actor, ActorRef}
 import akka.testkit.{TestActorRef, TestFSMRef, TestProbe}
 import com.typesafe.config.{Config, ConfigFactory}
+import cromwell._
 import cromwell.backend.{AllBackendInitializationData, JobExecutionMap}
 import cromwell.core._
+import cromwell.core.path.DefaultPathBuilder
 import cromwell.engine.EngineWorkflowDescriptor
 import cromwell.engine.backend.BackendSingletonCollection
 import cromwell.engine.workflow.WorkflowActor._
-import cromwell.engine.workflow.lifecycle.{CopyWorkflowLogsActor, EngineLifecycleActorAbortCommand}
 import cromwell.engine.workflow.lifecycle.MaterializeWorkflowDescriptorActor.MaterializeWorkflowDescriptorFailureResponse
 import cromwell.engine.workflow.lifecycle.WorkflowFinalizationActor.{StartFinalizationCommand, WorkflowFinalizationSucceededResponse}
 import cromwell.engine.workflow.lifecycle.WorkflowInitializationActor.{WorkflowInitializationAbortedResponse, WorkflowInitializationFailedResponse}
 import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor.{WorkflowExecutionAbortedResponse, WorkflowExecutionFailedResponse, WorkflowExecutionSucceededResponse}
+import cromwell.engine.workflow.lifecycle.{CopyWorkflowLogsActor, EngineLifecycleActorAbortCommand}
 import cromwell.util.SampleWdl.ThreeStep
-import cromwell._
 import org.scalatest.BeforeAndAfter
 import org.scalatest.concurrent.Eventually
 
@@ -31,7 +30,7 @@ class WorkflowActorSpec extends CromwellTestKitSpec with WorkflowDescriptorBuild
     }
   })
 
-  val mockDir = Paths.get("/where/to/copy/wf/logs")
+  val mockDir = DefaultPathBuilder.get("/where/to/copy/wf/logs")
   val mockWorkflowOptions = s"""{ "final_workflow_log_dir" : "$mockDir" }"""
 
   var currentWorkflowId: WorkflowId = _
diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsPathBuilder.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsPathBuilder.scala
index 376a032cc..ae6c1f77b 100644
--- a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsPathBuilder.scala
+++ b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsPathBuilder.scala
@@ -1,7 +1,6 @@
 package cromwell.filesystems.gcs
 
 import java.net.URI
-import java.nio.file.Path
 import java.nio.file.spi.FileSystemProvider
 
 import akka.actor.ActorSystem
@@ -14,7 +13,7 @@ import com.google.common.base.Preconditions._
 import com.google.common.net.UrlEscapers
 import cromwell.core.WorkflowOptions
 import cromwell.core.path.proxy.{PathProxy, RetryableFileSystemProviderProxy}
-import cromwell.core.path.{CustomRetryParams, PathBuilder}
+import cromwell.core.path.{CustomRetryParams, NioPath, Path, PathBuilder}
 import cromwell.filesystems.gcs.GcsPathBuilder._
 import cromwell.filesystems.gcs.auth.GoogleAuthMode
 
@@ -40,8 +39,8 @@ object GcsPathBuilder {
     Try(checkValid(getUri(str))).isSuccess
   }
 
-  def isGcsPath(path: Path): Boolean = {
-    path.getFileSystem.provider().getScheme == CloudStorageFileSystem.URI_SCHEME
+  def isGcsPath(nioPath: NioPath): Boolean = {
+    nioPath.getFileSystem.provider().getScheme == CloudStorageFileSystem.URI_SCHEME
   }
 
   def getUri(string: String) = URI.create(UrlEscapers.urlFragmentEscaper().escape(string))
@@ -76,23 +75,26 @@ class GcsPathBuilder(authMode: GoogleAuthMode,
    * com.google.cloud.storage.Storage has some batching capabilities but not for copying.
    * In order to support batch copy, we need a com.google.api.services.storage.Storage.
    */
-  def getHash(path: Path): Try[String] = {
-    path match {
-      case gcsPath: CloudStoragePath => Try(storageOptions.service().get(gcsPath.bucket(), gcsPath.toRealPath().toString).crc32c())
-      case proxy: PathProxy =>
-        val gcsPath = proxy.unbox(classOf[CloudStoragePath]).get
-        Try(storageOptions.service().get(gcsPath.bucket(), gcsPath.toRealPath().toString).crc32c())
-      case other => Failure(new IllegalArgumentException(s"$other is not a CloudStoragePath"))
+  def getHash(builtPath: Path): Try[String] = {
+    builtPath match {
+      case GcsPath(path) => path match {
+        case gcsPath: CloudStoragePath => Try(storageOptions.service().get(gcsPath.bucket(), gcsPath.toRealPath().toString).crc32c())
+        case proxy: PathProxy =>
+          val gcsPath = proxy.unbox(classOf[CloudStoragePath]).get
+          Try(storageOptions.service().get(gcsPath.bucket(), gcsPath.toRealPath().toString).crc32c())
+        case other => Failure(new IllegalArgumentException(s"$other is not a CloudStoragePath"))
+      }
+      case other => Failure(new IllegalArgumentException(s"$other is not a GcsPath"))
     }
   }
 
   def getProjectId = storageOptions.projectId()
 
-  def build(string: String): Try[Path] = {
+  def build(string: String): Try[GcsPath] = {
     Try {
       val uri = getUri(string)
       GcsPathBuilder.checkValid(uri)
-      provider.getPath(uri)
+      GcsPath(provider.getPath(uri))
     }
   }
 
@@ -110,3 +112,20 @@ class RetryableGcsPathBuilder(authMode: GoogleAuthMode,
 
   override def getHash(path: Path) = provider.withRetry(() => super.getHash(path))
 }
+
+case class GcsPath private[gcs](nioPath: NioPath) extends Path {
+  override protected def newPath(nioPath: NioPath): GcsPath = GcsPath(nioPath)
+
+  override def pathAsString: String = java.net.URLDecoder.decode(nioPath.toUri.toString, "UTF-8")
+
+  override def pathWithoutScheme: String = {
+    val gcsPath = cloudStoragePath
+    gcsPath.bucket + gcsPath.toAbsolutePath.toString
+  }
+
+  private def cloudStoragePath: CloudStoragePath = nioPath match {
+    case gcsPath: CloudStoragePath => gcsPath
+    case pathProxy: PathProxy => pathProxy.unbox(classOf[CloudStoragePath]).get
+    case _ => throw new RuntimeException(s"Internal path was not a cloud storage path: $nioPath")
+  }
+}
diff --git a/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GcsPathBuilderSpec.scala b/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GcsPathBuilderSpec.scala
index 6f7b00dbb..697313b41 100644
--- a/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GcsPathBuilderSpec.scala
+++ b/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GcsPathBuilderSpec.scala
@@ -2,33 +2,25 @@ package cromwell.filesystems.gcs
 
 import com.google.cloud.RetryParams
 import com.google.cloud.storage.contrib.nio.CloudStorageConfiguration
-import cromwell.core.path.CustomRetryParams
+import cromwell.core.path._
 import cromwell.core.path.proxy.RetryableFileSystemProviderProxy
 import cromwell.core.{TestKitSuite, WorkflowOptions}
 import cromwell.filesystems.gcs.auth.{GoogleAuthMode, GoogleAuthModeSpec}
+import org.scalatest.prop.Tables.Table
 import org.scalatest.{FlatSpecLike, Matchers}
 
-class GcsPathBuilderSpec extends TestKitSuite with FlatSpecLike with Matchers {
+class GcsPathBuilderSpec extends TestKitSuite with FlatSpecLike with Matchers with PathBuilderSpecUtils {
 
-  behavior of "GcsPathBuilderSpec"
+  behavior of "GcsPathBuilder"
 
   it should "create a path with a retryable provider" in {
-    GoogleAuthModeSpec.assumeHasApplicationDefaultCredentials()
-
-    val retryablePathBuilder = new RetryableGcsPathBuilder(
-      GoogleAuthMode.NoAuthMode,
-      RetryParams.defaultInstance(),
-      CustomRetryParams.Default,
-      CloudStorageConfiguration.DEFAULT,
-      WorkflowOptions.empty
-    )
-
     val path = retryablePathBuilder.build("gs://bucket/object")
     path.isSuccess shouldBe true
-    path.get.getFileSystem.provider() shouldBe a[RetryableFileSystemProviderProxy[_]]
+    path.get.nioPath.getFileSystem.provider() shouldBe a[RetryableFileSystemProviderProxy[_]]
   }
 
   it should "use google project credentials when provided in the workflow options" in {
+    GoogleAuthModeSpec.assumeHasApplicationDefaultCredentials()
 
     val wfOptionsWithProject = WorkflowOptions.fromMap(Map("google_project" -> "my_project")).get
 
@@ -41,4 +33,343 @@ class GcsPathBuilderSpec extends TestKitSuite with FlatSpecLike with Matchers {
 
     gcsPathBuilderWithProjectInfo.getProjectId shouldBe "my_project"
   }
+
+  it should behave like truncateCommonRoots(retryablePathBuilder, pathsToTruncate)
+
+  goodPaths foreach { goodPath =>
+    it should behave like buildGoodPath(retryablePathBuilder, goodPath)
+  }
+
+  badPaths foreach { badPath =>
+    it should behave like buildBadPath(retryablePathBuilder, badPath)
+  }
+
+  private def pathsToTruncate = Table(
+    ("context", "file", "relative"),
+    ("gs://bucket", "gs://bucket/path/to/file", "path/to/file"),
+    ("gs://bucket/path/to/my/dir", "gs://bucket/path/to/my/dir/file", "file"),
+    ("gs://bucket/path/to/my/dir", "gs://bucket/path/to/my/dir//file", "file"),
+    // NOTE: Next two are different from the DefaultPathBuilder. "//" doesn't build to "/" in the GcsPathBuilder
+    ("gs://bucket/path/to/my//dir", "gs://bucket/path/to/my/dir/file", "dir/file"),
+    ("gs://bucket/path/to/my//dir", "gs://bucket/path/to/my/dir//file", "dir//file"),
+    ("gs://bucket/path/to/my/dir", "gs://bucket/path/./to/my/dir/file", "./to/my/dir/file"),
+    ("gs://bucket/path/to/my/dir/with/file", "gs://bucket/path/to/other/dir/with/file", "other/dir/with/file")
+  )
+
+  private def bucket = "mymadeupbucket"
+
+  private def goodPaths = Seq(
+    GoodPath(
+      description = "a path with spaces",
+      path = s"gs://$bucket/hello/world/with spaces",
+      normalize = false,
+      pathAsString = s"gs://$bucket/hello/world/with spaces",
+      pathWithoutScheme = s"$bucket/hello/world/with spaces",
+      parent = s"gs://$bucket/hello/world/",
+      getParent = s"gs://$bucket/hello/world/",
+      root = s"gs://$bucket/",
+      name = "with spaces",
+      getFileName = s"gs://$bucket/with spaces",
+      toUriHost = bucket,
+      toUriPath = "/hello/world/with%20spaces",
+      toUriStartsWith = s"gs://$bucket/hello/world/with%20spaces",
+      toUriEndsWith = s"gs://$bucket/hello/world/with%20spaces",
+      getNameCount = 3,
+      isAbsolute = true,
+      isDirectory = false),
+
+    GoodPath(
+      description = "a path with non-ascii",
+      path = s"gs://$bucket/hello/world/with non ascii £€",
+      normalize = false,
+      pathAsString = s"gs://$bucket/hello/world/with non ascii £€",
+      pathWithoutScheme = s"$bucket/hello/world/with non ascii £€",
+      parent = s"gs://$bucket/hello/world/",
+      getParent = s"gs://$bucket/hello/world/",
+      root = s"gs://$bucket/",
+      name = "with non ascii £€",
+      getFileName = s"gs://$bucket/with non ascii £€",
+      toUriHost = bucket,
+      toUriPath = "/hello/world/with%20non%20ascii%20£€",
+      toUriStartsWith = s"gs://$bucket/hello/world/with%20non%20ascii%20£€",
+      toUriEndsWith = s"gs://$bucket/hello/world/with%20non%20ascii%20£€",
+      getNameCount = 3,
+      isAbsolute = true,
+      isDirectory = false),
+
+    GoodPath(
+      description = "a gs uri path with encoded characters",
+      path = s"gs://$bucket/hello/world/encoded%20spaces",
+      normalize = false,
+      pathAsString = s"gs://$bucket/hello/world/encoded%20spaces",
+      pathWithoutScheme = s"$bucket/hello/world/encoded%20spaces",
+      parent = s"gs://$bucket/hello/world/",
+      getParent = s"gs://$bucket/hello/world/",
+      root = s"gs://$bucket/",
+      name = "encoded%20spaces",
+      getFileName = s"gs://$bucket/encoded%20spaces",
+      toUriHost = bucket,
+      toUriPath = "/hello/world/encoded%2520spaces",
+      toUriStartsWith = s"gs://$bucket/hello/world/encoded%2520spaces",
+      toUriEndsWith = s"gs://$bucket/hello/world/encoded%2520spaces",
+      getNameCount = 3,
+      isAbsolute = true,
+      isDirectory = false),
+
+    GoodPath(
+      description = "a bucket only path",
+      path = s"gs://$bucket",
+      normalize = false,
+      pathAsString = s"gs://$bucket/",
+      pathWithoutScheme = s"$bucket/",
+      parent = null,
+      getParent = null,
+      root = s"gs://$bucket/",
+      name = "",
+      getFileName = s"gs://$bucket/",
+      toUriHost = bucket,
+      toUriPath = "/",
+      toUriStartsWith = s"gs://$bucket/",
+      toUriEndsWith = s"gs://$bucket/",
+      getNameCount = 1,
+      isAbsolute = false,
+      isDirectory = true),
+
+    GoodPath(
+      description = "a bucket only path ending in a /",
+      path = s"gs://$bucket/",
+      normalize = false,
+      pathAsString = s"gs://$bucket/",
+      pathWithoutScheme = s"$bucket/",
+      parent = null,
+      getParent = null,
+      root = s"gs://$bucket/",
+      name = "",
+      getFileName = null,
+      toUriHost = bucket,
+      toUriPath = "/",
+      toUriStartsWith = s"gs://$bucket/",
+      toUriEndsWith = s"gs://$bucket/",
+      getNameCount = 0,
+      isAbsolute = true,
+      isDirectory = true),
+
+    GoodPath(
+      description = "a file at the top of the bucket",
+      path = s"gs://$bucket/hello",
+      normalize = false,
+      pathAsString = s"gs://$bucket/hello",
+      pathWithoutScheme = s"$bucket/hello",
+      parent = s"gs://$bucket/",
+      getParent = s"gs://$bucket/",
+      root = s"gs://$bucket/",
+      name = "hello",
+      getFileName = s"gs://$bucket/hello",
+      toUriHost = bucket,
+      toUriPath = "/hello",
+      toUriStartsWith = s"gs://$bucket/hello",
+      toUriEndsWith = s"gs://$bucket/hello",
+      getNameCount = 1,
+      isAbsolute = true,
+      isDirectory = false),
+
+    GoodPath(
+      description = "a path ending in /",
+      path = s"gs://$bucket/hello/world/",
+      normalize = false,
+      pathAsString = s"gs://$bucket/hello/world/",
+      pathWithoutScheme = s"$bucket/hello/world/",
+      parent = s"gs://$bucket/hello/",
+      getParent = s"gs://$bucket/hello/",
+      root = s"gs://$bucket/",
+      name = "world",
+      getFileName = s"gs://$bucket/world",
+      toUriHost = bucket,
+      toUriPath = "/hello/world/",
+      toUriStartsWith = s"gs://$bucket/hello/world/",
+      toUriEndsWith = s"gs://$bucket/hello/world/",
+      getNameCount = 2,
+      isAbsolute = true,
+      isDirectory = true),
+
+    // Special paths
+
+    GoodPath(
+      description = "a bucket with a path .",
+      path = s"gs://$bucket/.",
+      normalize = false,
+      pathAsString = s"gs://$bucket/.",
+      pathWithoutScheme = s"$bucket/.",
+      parent = null,
+      getParent = s"gs://$bucket/",
+      root = s"gs://$bucket/",
+      name = "",
+      getFileName = s"gs://$bucket/.",
+      toUriHost = bucket,
+      toUriPath = "/.",
+      toUriStartsWith = s"gs://$bucket/.",
+      toUriEndsWith = s"gs://$bucket/.",
+      getNameCount = 1,
+      isAbsolute = true,
+      isDirectory = true),
+
+    GoodPath(
+      description = "a bucket with a path ..",
+      path = s"gs://$bucket/..",
+      normalize = false,
+      pathAsString = s"gs://$bucket/..",
+      pathWithoutScheme = s"$bucket/..",
+      parent = null,
+      getParent = s"gs://$bucket/",
+      root = null,
+      name = "",
+      getFileName = s"gs://$bucket/..",
+      toUriHost = bucket,
+      toUriPath = "/..",
+      toUriStartsWith = s"gs://$bucket/..",
+      toUriEndsWith = s"gs://$bucket/..",
+      getNameCount = 1,
+      isAbsolute = true,
+      isDirectory = true),
+
+    GoodPath(
+      description = "a bucket including . in the path",
+      path = s"gs://$bucket/hello/./world",
+      normalize = false,
+      pathAsString = s"gs://$bucket/hello/./world",
+      pathWithoutScheme = s"$bucket/hello/./world",
+      parent = s"gs://$bucket/hello/",
+      getParent = s"gs://$bucket/hello/./",
+      root = s"gs://$bucket/",
+      name = "world",
+      getFileName = s"gs://$bucket/world",
+      toUriHost = bucket,
+      toUriPath = "/hello/./world",
+      toUriStartsWith = s"gs://$bucket/hello/./world",
+      toUriEndsWith = s"gs://$bucket/hello/./world",
+      getNameCount = 3,
+      isAbsolute = true,
+      isDirectory = false),
+
+    GoodPath(
+      description = "a bucket including .. in the path",
+      path = s"gs://$bucket/hello/../world",
+      normalize = false,
+      pathAsString = s"gs://$bucket/hello/../world",
+      pathWithoutScheme = s"$bucket/hello/../world",
+      parent = s"gs://$bucket/",
+      getParent = s"gs://$bucket/hello/../",
+      root = s"gs://$bucket/",
+      name = "world",
+      getFileName = s"gs://$bucket/world",
+      toUriHost = bucket,
+      toUriPath = "/hello/../world",
+      toUriStartsWith = s"gs://$bucket/hello/../world",
+      toUriEndsWith = s"gs://$bucket/hello/../world",
+      getNameCount = 3,
+      isAbsolute = true,
+      isDirectory = false),
+
+    // Normalized
+
+    GoodPath(
+      description = "a bucket with a normalized path .",
+      path = s"gs://$bucket/.",
+      normalize = true,
+      pathAsString = s"gs://$bucket/",
+      pathWithoutScheme = s"$bucket/",
+      parent = null,
+      getParent = null,
+      root = s"gs://$bucket/",
+      name = "",
+      getFileName = null,
+      toUriHost = bucket,
+      toUriPath = "/",
+      toUriStartsWith = s"gs://$bucket/",
+      toUriEndsWith = s"gs://$bucket/",
+      getNameCount = 0,
+      isAbsolute = true,
+      isDirectory = true),
+
+    GoodPath(
+      description = "a bucket with a normalized path ..",
+      path = s"gs://$bucket/..",
+      normalize = true,
+      pathAsString = s"gs://$bucket/",
+      pathWithoutScheme = s"$bucket/",
+      parent = null,
+      getParent = null,
+      root = s"gs://$bucket/",
+      name = "",
+      getFileName = s"gs://$bucket/",
+      toUriHost = bucket,
+      toUriPath = "/",
+      toUriStartsWith = s"gs://$bucket/",
+      toUriEndsWith = s"gs://$bucket/",
+      getNameCount = 1,
+      isAbsolute = false,
+      isDirectory = true),
+
+    GoodPath(
+      description = "a bucket including . in the normalized path",
+      path = s"gs://$bucket/hello/./world",
+      normalize = true,
+      pathAsString = s"gs://$bucket/hello/world",
+      pathWithoutScheme = s"$bucket/hello/world",
+      parent = s"gs://$bucket/hello/",
+      getParent = s"gs://$bucket/hello/",
+      root = s"gs://$bucket/",
+      name = "world",
+      getFileName = s"gs://$bucket/world",
+      toUriHost = bucket,
+      toUriPath = "/hello/world",
+      toUriStartsWith = s"gs://$bucket/hello/world",
+      toUriEndsWith = s"gs://$bucket/hello/world",
+      getNameCount = 2,
+      isAbsolute = true,
+      isDirectory = false),
+
+    GoodPath(
+      description = "a bucket including .. in the normalized path",
+      path = s"gs://$bucket/hello/../world",
+      normalize = true,
+      pathAsString = s"gs://$bucket/world",
+      pathWithoutScheme = s"$bucket/world",
+      parent = s"gs://$bucket/",
+      getParent = s"gs://$bucket/",
+      root = s"gs://$bucket/",
+      name = "world",
+      getFileName = s"gs://$bucket/world",
+      toUriHost = bucket,
+      toUriPath = "/world",
+      toUriStartsWith = s"gs://$bucket/world",
+      toUriEndsWith = s"gs://$bucket/world",
+      getNameCount = 1,
+      isAbsolute = true,
+      isDirectory = false)
+  )
+
+  private def badPaths = Seq(
+    BadPath("an empty path", "", " does not have a gcs scheme"),
+    BadPath("an bucketless path", "gs://", "Expected authority at index 5: gs://"),
+    BadPath("a bucket named .", "gs://./hello/world", "gs://./hello/world does not have a host"),
+    BadPath("a non ascii bucket name", "gs://nonasciibucket£€/hello/world",
+      "gs://nonasciibucket%C2%A3%E2%82%AC/hello/world does not have a host"),
+    BadPath("a https path", "https://hello/world", "Cloud Storage URIs must have 'gs' scheme: https://hello/world"),
+    BadPath("a file uri path", "file:///hello/world", "Cloud Storage URIs must have 'gs' scheme: file:///hello/world"),
+    BadPath("a relative file path", "hello/world", "hello/world does not have a gcs scheme"),
+    BadPath("an absolute file path", "/hello/world", "/hello/world does not have a gcs scheme")
+  )
+
+  private lazy val retryablePathBuilder = {
+    GoogleAuthModeSpec.assumeHasApplicationDefaultCredentials()
+
+    new RetryableGcsPathBuilder(
+      GoogleAuthMode.NoAuthMode,
+      RetryParams.defaultInstance(),
+      CustomRetryParams.Default,
+      CloudStorageConfiguration.DEFAULT,
+      WorkflowOptions.empty
+    )
+  }
 }
diff --git a/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/auth/GoogleAuthModeSpec.scala b/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/auth/GoogleAuthModeSpec.scala
index 327f98fe7..4f3dcd56f 100644
--- a/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/auth/GoogleAuthModeSpec.scala
+++ b/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/auth/GoogleAuthModeSpec.scala
@@ -3,16 +3,22 @@ package cromwell.filesystems.gcs.auth
 import cromwell.core.WorkflowOptions
 import org.scalatest.Assertions._
 
+import scala.util.{Failure, Try}
+
 object GoogleAuthModeSpec {
   def assumeHasApplicationDefaultCredentials(): Unit = {
-    try {
-      val authMode = ApplicationDefaultMode("application-default")
-      val workflowOptions = WorkflowOptions.empty
-      authMode.authCredentials(workflowOptions)
-      authMode.credential(workflowOptions)
-      ()
-    } catch {
-      case exception: Exception => cancel(exception.getMessage)
+    tryApplicationDefaultCredentials match {
+      case Failure(exception) => cancel(exception.getMessage)
+      case _ =>
     }
+    ()
+  }
+
+  private lazy val tryApplicationDefaultCredentials: Try[Unit] = Try {
+    val authMode = ApplicationDefaultMode("application-default")
+    val workflowOptions = WorkflowOptions.empty
+    authMode.authCredentials(workflowOptions)
+    authMode.credential(workflowOptions)
+    ()
   }
 }
diff --git a/project/Dependencies.scala b/project/Dependencies.scala
index b78ffabb7..0440159ee 100644
--- a/project/Dependencies.scala
+++ b/project/Dependencies.scala
@@ -1,7 +1,7 @@
 import sbt._
 
 object Dependencies {
-  lazy val lenthallV = "0.20"
+  lazy val lenthallV = "0.21-e1b7822-SNAP"
   lazy val wdl4sV = "0.10-76bf06b-SNAP"
   lazy val sprayV = "1.3.3"
   /*
diff --git a/services/src/main/scala/cromwell/services/metadata/MetadataQuery.scala b/services/src/main/scala/cromwell/services/metadata/MetadataQuery.scala
index 9015d59f0..a176bc1c5 100644
--- a/services/src/main/scala/cromwell/services/metadata/MetadataQuery.scala
+++ b/services/src/main/scala/cromwell/services/metadata/MetadataQuery.scala
@@ -1,11 +1,9 @@
 package cromwell.services.metadata
 
-import java.nio.file.Path
 import java.time.OffsetDateTime
 
 import cats.data.NonEmptyList
 import cromwell.core.WorkflowId
-import cromwell.core.path.PathImplicits._
 import org.slf4j.{Logger, LoggerFactory}
 import wdl4s.values.{WdlBoolean, WdlFloat, WdlInteger, WdlOptionalValue, WdlValue}
 
@@ -47,7 +45,6 @@ object MetadataValue {
       case _: Int | Long => new MetadataValue(value.toString, MetadataInt)
       case _: Double | Float => new MetadataValue(value.toString, MetadataNumber)
       case _: Boolean => new MetadataValue(value.toString, MetadataBoolean)
-      case path: Path => new MetadataValue(path.toRealString, MetadataString)
       case other => new MetadataValue(other.toString, MetadataString)
     }
   }
diff --git a/src/main/scala/cromwell/CromwellCommandLine.scala b/src/main/scala/cromwell/CromwellCommandLine.scala
index 3b1b55155..73d4c2543 100644
--- a/src/main/scala/cromwell/CromwellCommandLine.scala
+++ b/src/main/scala/cromwell/CromwellCommandLine.scala
@@ -1,13 +1,10 @@
 package cromwell
 
-import java.nio.file.{Files, Path, Paths}
-
-import better.files._
 import cats.data.Validated._
 import cats.syntax.cartesian._
 import cats.syntax.validated._
+import cromwell.core.path.{DefaultPathBuilder, Path}
 import cromwell.core.{WorkflowSourceFilesCollection, WorkflowSourceFilesWithDependenciesZip, WorkflowSourceFilesWithoutImports}
-import cromwell.util.FileUtil._
 import lenthall.exception.MessageAggregation
 import lenthall.validation.ErrorOr._
 import org.slf4j.LoggerFactory
@@ -50,9 +47,9 @@ object RunSingle {
 
   def apply(args: Seq[String]): RunSingle = {
     val pathParameters = SingleRunPathParameters(
-      wdlPath = Paths.get(args.head).toAbsolutePath,
-      inputsPath = argPath(args, 1, Option(".inputs"), checkDefaultExists = false),
-      optionsPath = argPath(args, 2, Option(".options"), checkDefaultExists = true),
+      wdlPath = DefaultPathBuilder.get(args.head).toAbsolutePath,
+      inputsPath = argPath(args, 1, Option("inputs"), checkDefaultExists = false),
+      optionsPath = argPath(args, 2, Option("options")),
       metadataPath = argPath(args, 3, None),
       importPath = argPath(args, 4, None),
       labelsPath = argPath(args, 5, None)
@@ -64,7 +61,8 @@ object RunSingle {
     val labelsJson = readJson("Labels", pathParameters.labelsPath)
 
     val sourceFileCollection = pathParameters.importPath match {
-      case Some(p) => (wdl |@| inputsJson |@| optionsJson |@| labelsJson) map { (w, i, o, l) => WorkflowSourceFilesWithDependenciesZip.apply(w, i, o, l, Files.readAllBytes(p)) }
+      case Some(p) => (wdl |@| inputsJson |@| optionsJson |@| labelsJson) map { (w, i, o, l) =>
+        WorkflowSourceFilesWithDependenciesZip.apply(w, i, o, l, p.loadBytes) }
       case None => (wdl |@| inputsJson |@| optionsJson |@| labelsJson) map WorkflowSourceFilesWithoutImports.apply
     }
 
@@ -85,17 +83,17 @@ object RunSingle {
   private def writeableMetadataPath(path: Option[Path]): ErrorOr[Unit] = {
     path match {
       case Some(p) if !metadataPathIsWriteable(p) => s"Unable to write to metadata directory: $p".invalidNel
-      case otherwise => ().validNel
+      case _ => ().validNel
     }
   }
 
   /** Read the path to a string. */
   private def readContent(inputDescription: String, path: Path): ErrorOr[String] = {
-    if (!Files.exists(path)) {
+    if (!path.exists) {
       s"$inputDescription does not exist: $path".invalidNel
-    } else if (!Files.isReadable(path)) {
+    } else if (!path.isReadable) {
       s"$inputDescription is not readable: $path".invalidNel
-    } else File(path).contentAsString.validNel
+    } else path.contentAsString.validNel
   }
 
   /** Read the path to a string, unless the path is None, in which case returns "{}". */
@@ -107,7 +105,7 @@ object RunSingle {
   }
 
   private def metadataPathIsWriteable(metadataPath: Path): Boolean = {
-    Try(File(metadataPath).createIfNotExists(asDirectory = false, createParents = true).append("")) match {
+    Try(metadataPath.createIfNotExists(createParents = true).append("")) match {
       case Success(_) => true
       case Failure(_) => false
     }
@@ -127,12 +125,13 @@ object RunSingle {
 
     // To return a default, swap the extension, and then maybe check if the file exists.
     def defaultPath = defaultExt
-      .map(ext => swapExt(args.head, ".wdl", ext))
-      .filter(path => !checkDefaultExists || Files.exists(Paths.get(path)))
+      .map(ext => DefaultPathBuilder.get(args.head).swapExt("wdl", ext))
+      .filter(path => !checkDefaultExists || path.exists)
+      .map(_.pathAsString)
 
     // Return the path for the arg index, or the default, but remove "-" paths.
     for {
       path <- args.lift(index) orElse defaultPath filterNot (_ == "-")
-    } yield Paths.get(path).toAbsolutePath
+    } yield DefaultPathBuilder.get(path).toAbsolutePath
   }
 }
diff --git a/src/test/scala/cromwell/CromwellCommandLineSpec.scala b/src/test/scala/cromwell/CromwellCommandLineSpec.scala
index 8fadbcce4..bd6da383a 100644
--- a/src/test/scala/cromwell/CromwellCommandLineSpec.scala
+++ b/src/test/scala/cromwell/CromwellCommandLineSpec.scala
@@ -1,7 +1,6 @@
 package cromwell
 
-import better.files._
-import cromwell.core.path.PathImplicits._
+import cromwell.core.path.{DefaultPathBuilder, Path}
 import cromwell.util.SampleWdl
 import cromwell.util.SampleWdl.{FileClobber, FilePassingWorkflow, ThreeStep}
 import org.scalatest.{FlatSpec, Matchers}
@@ -82,10 +81,10 @@ class CromwellCommandLineSpec extends FlatSpec with Matchers {
   }
 
   it should "run if imports directory is a .zip file" in {
-    val wdlDir = File.newTemporaryDirectory("wdlDirectory")
+    val wdlDir = DefaultPathBuilder.createTempDirectory("wdlDirectory")
 
-    val filePassing = File.newTemporaryFile("filePassing", ".wdl", Option(wdlDir))
-    val fileClobber = File.newTemporaryFile("fileClobber", ".wdl", Option(wdlDir))
+    val filePassing = DefaultPathBuilder.createTempFile("filePassing", ".wdl", Option(wdlDir))
+    val fileClobber = DefaultPathBuilder.createTempFile("fileClobber", ".wdl", Option(wdlDir))
     filePassing write FilePassingWorkflow.wdlSource()
     fileClobber write FileClobber.wdlSource()
 
@@ -109,18 +108,18 @@ object CromwellCommandLineSpec {
    */
   case class WdlAndInputs(sampleWdl: SampleWdl, optionsJson: String = "{}") {
     // Track all the temporary files we create, and delete them after the test.
-    private var tempFiles = Vector.empty[File]
+    private var tempFiles = Vector.empty[Path]
 
     lazy val wdlFile = {
-      val file = File.newTemporaryFile(s"${sampleWdl.name}.", ".wdl")
+      val file = DefaultPathBuilder.createTempFile(s"${sampleWdl.name}.", ".wdl")
       tempFiles :+= file
-      file write sampleWdl.wdlSource("")
+      file write sampleWdl.wdlSource()
     }
 
     lazy val wdl = wdlFile.pathAsString
 
     lazy val inputsFile = {
-      val file = File(wdlFile.path.swapExt(".wdl", ".inputs"))
+      val file = wdlFile.swapExt("wdl", "inputs")
       tempFiles :+= file
       file write sampleWdl.wdlJson
     }
@@ -128,7 +127,7 @@ object CromwellCommandLineSpec {
     lazy val inputs = inputsFile.pathAsString
 
     lazy val optionsFile = {
-      val file = File(wdlFile.path.swapExt(".wdl", ".options"))
+      val file = wdlFile.swapExt("wdl", "options")
       tempFiles :+= file
       file write optionsJson
     }
@@ -136,7 +135,7 @@ object CromwellCommandLineSpec {
     lazy val options = optionsFile.pathAsString
 
     lazy val metadataFile = {
-      val path = File(wdlFile.path.swapExt(".wdl", ".metadata.json"))
+      val path = wdlFile.swapExt("wdl", "metadata.json")
       tempFiles :+= path
       path
     }
diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala
index e36a5f971..be5437ce4 100644
--- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala
+++ b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala
@@ -12,6 +12,7 @@ import cromwell.backend.io.JobPathsWithDocker
 import cromwell.backend.sfs.{SharedFileSystem, SharedFileSystemExpressionFunctions}
 import cromwell.backend.wdl.Command
 import cromwell.core.path.JavaWriterImplicits._
+import cromwell.core.path.Obsolete._
 import cromwell.core.path.{DefaultPathBuilder, PathBuilder}
 import cromwell.services.keyvalue.KeyValueServiceActor._
 import cromwell.services.metadata.CallMetadataKeys
@@ -42,7 +43,6 @@ class HtCondorJobExecutionActor(override val jobDescriptor: BackendJobDescriptor
   extends BackendJobExecutionActor with CachedResultLocalization with SharedFileSystem {
 
   import HtCondorJobExecutionActor._
-  import better.files._
 
   private val tag = s"CondorJobExecutionActor-${jobDescriptor.call.fullyQualifiedName}:"
   override val pathBuilders: List[PathBuilder] = HtCondorJobExecutionActor.pathBuilders
diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorWrapper.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorWrapper.scala
index 35af6dd1d..cbc9c24e9 100644
--- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorWrapper.scala
+++ b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorWrapper.scala
@@ -1,13 +1,10 @@
 package cromwell.backend.impl.htcondor
 
-import java.nio.file.{Files, Path}
-
-import better.files._
 import com.typesafe.scalalogging.StrictLogging
 import cromwell.backend.impl.htcondor
-import cromwell.core.path.{TailedWriter, UntailedWriter}
-import cromwell.core.path.PathImplicits._
 import cromwell.core.path.JavaWriterImplicits._
+import cromwell.core.path.Obsolete._
+import cromwell.core.path.{Path, TailedWriter, UntailedWriter}
 
 import scala.sys.process._
 
diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/localization/CachedResultLocalization.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/localization/CachedResultLocalization.scala
index f3c6ad632..027b0b287 100644
--- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/localization/CachedResultLocalization.scala
+++ b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/localization/CachedResultLocalization.scala
@@ -1,10 +1,8 @@
 package cromwell.backend.impl.htcondor.caching.localization
 
-import java.nio.file.{Files, Path, Paths}
-
-import better.files.File
-import cromwell.core.path.FileImplicits._
-import cromwell.core.{JobOutput, _}
+import cromwell.core._
+import cromwell.core.path.Obsolete._
+import cromwell.core.path.Path
 import wdl4s.types.{WdlArrayType, WdlFileType}
 import wdl4s.values.{WdlArray, WdlSingleFile, WdlValue}
 
diff --git a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorCommandSpec.scala b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorCommandSpec.scala
index 9a03fecca..d6778fadd 100644
--- a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorCommandSpec.scala
+++ b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorCommandSpec.scala
@@ -1,7 +1,6 @@
 package cromwell.backend.impl.htcondor
 
-import better.files._
-
+import cromwell.core.path.Obsolete._
 import org.scalatest.{Matchers, WordSpecLike}
 
 class HtCondorCommandSpec extends WordSpecLike with Matchers {
diff --git a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActorSpec.scala b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActorSpec.scala
index 33bc5c71c..3081feba7 100644
--- a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActorSpec.scala
+++ b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActorSpec.scala
@@ -1,11 +1,9 @@
 package cromwell.backend.impl.htcondor
 
 import java.io.Writer
-import java.nio.file.{Files, Path}
 
 import akka.actor.{Actor, Props}
 import akka.testkit.{ImplicitSender, TestActorRef}
-import better.files._
 import com.typesafe.config.ConfigFactory
 import cromwell.backend.BackendJobExecutionActor.{JobFailedNonRetryableResponse, JobSucceededResponse}
 import cromwell.backend.impl.htcondor.caching.CacheActor
@@ -14,7 +12,8 @@ import cromwell.backend.impl.htcondor.caching.model.CachedExecutionResult
 import cromwell.backend.io.JobPathsWithDocker
 import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptor, BackendSpec}
 import cromwell.core._
-import cromwell.core.path.{PathWriter, TailedWriter, UntailedWriter}
+import cromwell.core.path.Obsolete._
+import cromwell.core.path.{Path, PathWriter, TailedWriter, UntailedWriter}
 import cromwell.services.keyvalue.KeyValueServiceActor.{KvGet, KvPair, KvPut}
 import org.mockito.Mockito
 import org.mockito.Mockito._
diff --git a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/caching/localization/CachedResultLocalizationSpec.scala b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/caching/localization/CachedResultLocalizationSpec.scala
index e0e847e56..2e3b66656 100644
--- a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/caching/localization/CachedResultLocalizationSpec.scala
+++ b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/caching/localization/CachedResultLocalizationSpec.scala
@@ -1,8 +1,7 @@
 package cromwell.backend.impl.htcondor.caching.localization
 
-import java.nio.file.Files
-
-import cromwell.core.{JobOutput, CallOutputs}
+import cromwell.core.path.Obsolete._
+import cromwell.core.{CallOutputs, JobOutput}
 import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpecLike}
 import wdl4s.types.{WdlArrayType, WdlFileType}
 import wdl4s.values.{WdlArray, WdlSingleFile, WdlString}
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala
index a9e5d761c..dbdccd846 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala
@@ -1,11 +1,9 @@
 package cromwell.backend.impl.jes
 
 import java.net.SocketTimeoutException
-import java.nio.file.{Path, Paths}
 
 import akka.actor.ActorRef
 import com.google.api.client.googleapis.json.GoogleJsonResponseException
-import com.google.cloud.storage.contrib.nio.CloudStoragePath
 import cromwell.backend._
 import cromwell.backend.async.{AbortedExecutionHandle, ExecutionHandle, FailedNonRetryableExecutionHandle, FailedRetryableExecutionHandle, PendingExecutionHandle}
 import cromwell.backend.impl.jes.RunStatus.TerminalRunStatus
@@ -13,9 +11,9 @@ import cromwell.backend.impl.jes.io._
 import cromwell.backend.impl.jes.statuspolling.JesPollingActorClient
 import cromwell.backend.standard.{StandardAsyncExecutionActor, StandardAsyncExecutionActorParams, StandardAsyncJob}
 import cromwell.core._
-import cromwell.core.path.PathImplicits._
-import cromwell.core.path.proxy.PathProxy
+import cromwell.core.path.{DefaultPathBuilder, Path}
 import cromwell.core.retry.SimpleExponentialBackoff
+import cromwell.filesystems.gcs.GcsPath
 import wdl4s._
 import wdl4s.expression.NoFunctions
 import wdl4s.values._
@@ -64,9 +62,9 @@ class JesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn
 
   override lazy val retryable: Boolean = jobDescriptor.key.attempt <= runtimeAttributes.preemptible
   private lazy val cmdInput =
-    JesFileInput(ExecParamName, jesCallPaths.script.toRealString, Paths.get(jesCallPaths.scriptFilename), workingDisk)
+    JesFileInput(ExecParamName, jesCallPaths.script.pathAsString, DefaultPathBuilder.get(jesCallPaths.scriptFilename), workingDisk)
   private lazy val jesCommandLine = s"/bin/bash ${cmdInput.containerPath}"
-  private lazy val rcJesOutput = JesFileOutput(returnCodeFilename, returnCodeGcsPath.toRealString, Paths.get(returnCodeFilename), workingDisk)
+  private lazy val rcJesOutput = JesFileOutput(returnCodeFilename, returnCodeGcsPath.pathAsString, DefaultPathBuilder.get(returnCodeFilename), workingDisk)
 
   private lazy val standardParameters = Seq(rcJesOutput)
 
@@ -82,7 +80,7 @@ class JesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn
 
   private def gcsAuthParameter: Option[JesInput] = {
     if (jesAttributes.auths.gcs.requiresAuthFile || dockerConfiguration.isDefined)
-      Option(JesLiteralInput(ExtraConfigParamName, jesCallPaths.gcsAuthFilePath.toRealString))
+      Option(JesLiteralInput(ExtraConfigParamName, jesCallPaths.gcsAuthFilePath.pathAsString))
     else None
   }
 
@@ -95,7 +93,7 @@ class JesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn
                                     jobDescriptor: BackendJobDescriptor): Iterable[JesInput] = {
     (remotePathArray zip localPathArray zipWithIndex) flatMap {
       case ((remotePath, localPath), index) =>
-        Seq(JesFileInput(s"$jesNamePrefix-$index", remotePath.valueString, Paths.get(localPath.valueString), workingDisk))
+        Seq(JesFileInput(s"$jesNamePrefix-$index", remotePath.valueString, DefaultPathBuilder.get(localPath.valueString), workingDisk))
     }
   }
 
@@ -107,9 +105,7 @@ class JesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn
     */
   private def relativeLocalizationPath(file: WdlFile): WdlFile = {
     getPath(file.value) match {
-      case Success(path) =>
-        val value: WdlSource = path.toUri.getHost + path.toUri.getPath
-        WdlFile(value, file.isGlob)
+      case Success(path) => WdlFile(path.pathWithoutScheme, file.isGlob)
       case _ => file
     }
   }
@@ -137,7 +133,7 @@ class JesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn
     * @throws Exception if the `path` does not live in one of the supplied `disks`
     */
   private def relativePathAndAttachedDisk(path: String, disks: Seq[JesAttachedDisk]): (Path, JesAttachedDisk) = {
-    val absolutePath = Paths.get(path) match {
+    val absolutePath = DefaultPathBuilder.get(path) match {
       case p if !p.isAbsolute => JesWorkingDisk.MountPoint.resolve(p)
       case p => p
     }
@@ -171,7 +167,7 @@ class JesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn
   }
 
   private def generateJesSingleFileOutputs(wdlFile: WdlSingleFile): JesFileOutput = {
-    val destination = callRootPath.resolve(wdlFile.value.stripPrefix("/")).toRealString
+    val destination = callRootPath.resolve(wdlFile.value.stripPrefix("/")).pathAsString
     val (relpath, disk) = relativePathAndAttachedDisk(wdlFile.value, runtimeAttributes.disks)
     JesFileOutput(makeSafeJesReferenceName(wdlFile.value), destination, relpath, disk)
   }
@@ -180,17 +176,17 @@ class JesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn
     val globName = backendEngineFunctions.globName(wdlFile.value)
     val globDirectory = globName + "/"
     val globListFile = globName + ".list"
-    val gcsGlobDirectoryDestinationPath = callRootPath.resolve(globDirectory).toRealString
-    val gcsGlobListFileDestinationPath = callRootPath.resolve(globListFile).toRealString
+    val gcsGlobDirectoryDestinationPath = callRootPath.resolve(globDirectory).pathAsString
+    val gcsGlobListFileDestinationPath = callRootPath.resolve(globListFile).pathAsString
 
     val (_, globDirectoryDisk) = relativePathAndAttachedDisk(wdlFile.value, runtimeAttributes.disks)
 
     // We need both the glob directory and the glob list:
     List(
       // The glob directory:
-      JesFileOutput(makeSafeJesReferenceName(globDirectory), gcsGlobDirectoryDestinationPath, Paths.get(globDirectory + "*"), globDirectoryDisk),
+      JesFileOutput(makeSafeJesReferenceName(globDirectory), gcsGlobDirectoryDestinationPath, DefaultPathBuilder.get(globDirectory + "*"), globDirectoryDisk),
       // The glob list file:
-      JesFileOutput(makeSafeJesReferenceName(globListFile), gcsGlobListFileDestinationPath, Paths.get(globListFile), globDirectoryDisk)
+      JesFileOutput(makeSafeJesReferenceName(globListFile), gcsGlobListFileDestinationPath, DefaultPathBuilder.get(globListFile), globDirectoryDisk)
     )
   }
 
@@ -229,7 +225,7 @@ class JesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn
       runIdForResumption,
       jobDescriptor = jobDescriptor,
       runtimeAttributes = runtimeAttributes,
-      callRootPath = callRootPath.toRealString,
+      callRootPath = callRootPath.pathAsString,
       commandLine = jesCommandLine,
       logFileName = jesLogFilename,
       jesParameters,
@@ -385,27 +381,11 @@ class JesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn
     }
   }
 
-  /**
-    * Takes a path in GCS and comes up with a local path which is unique for the given GCS path.
-    *
-    * Matches the path generated via relativeLocalizationPath and passed in as JesFileInput.local.
-    *
-    * @param mountPoint The mount point for inputs
-    * @param gcsPath The input path
-    * @return A path which is unique per input path
-    */
-  private def localFilePathFromCloudStoragePath(mountPoint: Path, gcsPath: CloudStoragePath): Path = {
-    mountPoint.resolve(gcsPath.bucket()).resolve(gcsPath.toUri.getPath.stripPrefix("/"))
-  }
-
   override def mapCommandLineWdlFile(wdlFile: WdlFile): WdlFile = {
     getPath(wdlFile.valueString) match {
-      case Success(gcsPath: CloudStoragePath) =>
-        WdlFile(localFilePathFromCloudStoragePath(workingDisk.mountPoint, gcsPath).toString, wdlFile.isGlob)
-      case Success(proxy: PathProxy) =>
-        proxy.unbox(classOf[CloudStoragePath]) map { gcsPath =>
-          WdlFile(localFilePathFromCloudStoragePath(workingDisk.mountPoint, gcsPath).toString, wdlFile.isGlob)
-        } getOrElse wdlFile
+      case Success(gcsPath: GcsPath) =>
+        val localPath = workingDisk.mountPoint.resolve(gcsPath.pathWithoutScheme).pathAsString
+        WdlFile(localPath, wdlFile.isGlob)
       case _ => wdlFile
     }
   }
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesExpressionFunctions.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesExpressionFunctions.scala
index dbfc866d7..ceda6846f 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesExpressionFunctions.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesExpressionFunctions.scala
@@ -1,15 +1,14 @@
 package cromwell.backend.impl.jes
 
 import cromwell.backend.standard.{StandardExpressionFunctions, StandardExpressionFunctionsParams}
-import cromwell.core.path.PathImplicits._
 import cromwell.filesystems.gcs.GcsPathBuilder
 
 class JesExpressionFunctions(standardParams: StandardExpressionFunctionsParams)
   extends StandardExpressionFunctions(standardParams) {
 
-  override def preMapping(str: String): String =
+  override def preMapping(str: String) =
     if (!GcsPathBuilder.isValidGcsUrl(str)) {
-      callContext.root.resolve(str.stripPrefix("/")).toRealString
+      callContext.root.resolve(str.stripPrefix("/")).pathAsString
     } else {
       str
     }
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesFinalizationActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesFinalizationActor.scala
index b5729dbbc..131cea544 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesFinalizationActor.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesFinalizationActor.scala
@@ -1,6 +1,5 @@
 package cromwell.backend.impl.jes
 
-import better.files._
 import cats.instances.future._
 import cats.syntax.functor._
 import cromwell.backend._
@@ -38,7 +37,7 @@ class JesFinalizationActor(val jesParams: JesFinalizationActorParams)
 
   private def deleteAuthenticationFile(): Future[Unit] = {
     (jesConfiguration.needAuthFileUpload, workflowPaths) match {
-      case (true, Some(paths: JesWorkflowPaths)) => Future { File(paths.gcsAuthFilePath).delete(false) } void
+      case (true, Some(paths: JesWorkflowPaths)) => Future { paths.gcsAuthFilePath.delete() } void
       case _ => Future.successful(())
     }
   }
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesInitializationActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesInitializationActor.scala
index d9aaede16..3b9ac58e8 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesInitializationActor.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesInitializationActor.scala
@@ -50,14 +50,14 @@ class JesInitializationActor(jesParams: JesInitializationActorParams)
 
   override def beforeAll(): Future[Option[BackendInitializationData]] = Future.fromTry(Try {
     if (jesConfiguration.needAuthFileUpload) writeAuthenticationFile(workflowPaths)
-    publishWorkflowRoot(workflowPaths.workflowRoot.toString)
+    publishWorkflowRoot(workflowPaths.workflowRoot.pathAsString)
     Option(initializationData)
   })
 
   private def writeAuthenticationFile(workflowPath: JesWorkflowPaths): Unit = {
     generateAuthJson(jesConfiguration.dockerCredentials, refreshTokenAuth) foreach { content =>
       val path = workflowPath.gcsAuthFilePath
-      workflowLogger.info(s"Creating authentication file for workflow ${workflowDescriptor.id} at \n ${path.toUri}")
+      workflowLogger.info(s"Creating authentication file for workflow ${workflowDescriptor.id} at \n $path")
       try {
         path.writeAsJson(content)
       } catch {
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobCachingActorHelper.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobCachingActorHelper.scala
index ad699f36b..ff6d8dc59 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobCachingActorHelper.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobCachingActorHelper.scala
@@ -1,13 +1,10 @@
 package cromwell.backend.impl.jes
 
-import java.nio.file.Path
-
 import akka.actor.Actor
-import better.files._
 import cromwell.backend.impl.jes.io.{JesAttachedDisk, JesWorkingDisk}
 import cromwell.backend.standard.StandardCachingActorHelper
 import cromwell.core.logging.JobLogging
-import cromwell.core.path.PathImplicits._
+import cromwell.core.path.Path
 
 trait JesJobCachingActorHelper extends StandardCachingActorHelper {
   this: Actor with JobLogging =>
@@ -45,12 +42,12 @@ trait JesJobCachingActorHelper extends StandardCachingActorHelper {
   lazy val jesAttributes: JesAttributes = jesConfiguration.jesAttributes
   lazy val monitoringScript: Option[JesInput] = {
     jesCallPaths.monitoringPath map { path =>
-      JesFileInput(s"$MonitoringParamName-in", path.toRealString,
+      JesFileInput(s"$MonitoringParamName-in", path.pathAsString,
         JesWorkingDisk.MountPoint.resolve(JesMonitoringScript), workingDisk)
     }
   }
   lazy val monitoringOutput: Option[JesFileOutput] = monitoringScript map { _ => JesFileOutput(s"$MonitoringParamName-out",
-    defaultMonitoringOutputPath.toString, File(JesMonitoringLogFile).path, workingDisk)
+    defaultMonitoringOutputPath.pathAsString, JesMonitoringLogFile, workingDisk)
   }
 
   override protected def nonStandardMetadata: Map[String, Any] = {
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobPaths.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobPaths.scala
index b1932b7c8..bd4090619 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobPaths.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobPaths.scala
@@ -1,10 +1,9 @@
 package cromwell.backend.impl.jes
 
-import java.nio.file.Path
-
 import akka.actor.ActorSystem
 import cromwell.backend.io.JobPaths
 import cromwell.backend.{BackendJobDescriptorKey, BackendWorkflowDescriptor}
+import cromwell.core.path.Path
 import cromwell.services.metadata.CallMetadataKeys
 
 object JesJobPaths {
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesParameters.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesParameters.scala
index e814ea04a..6595e71a2 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesParameters.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesParameters.scala
@@ -1,9 +1,8 @@
 package cromwell.backend.impl.jes
 
-import java.nio.file.Path
-
 import com.google.api.services.genomics.model.{LocalCopy, PipelineParameter}
 import cromwell.backend.impl.jes.io.JesAttachedDisk
+import cromwell.core.path.Path
 
 sealed trait JesParameter {
   def name: String
@@ -16,7 +15,7 @@ sealed trait JesInput extends JesParameter
 final case class JesFileInput(name: String, gcs: String, local: Path, mount: JesAttachedDisk) extends JesInput {
   def toGooglePipelineParameter = {
     new PipelineParameter().setName(name).setLocalCopy(
-      new LocalCopy().setDisk(mount.name).setPath(local.toString)
+      new LocalCopy().setDisk(mount.name).setPath(local.pathAsString)
     )
   }
   val toGoogleRunParameter: String = gcs
@@ -31,7 +30,7 @@ final case class JesLiteralInput(name: String, value: String) extends JesInput {
 final case class JesFileOutput(name: String, gcs: String, local: Path, mount: JesAttachedDisk) extends JesParameter {
   def toGooglePipelineParameter = {
     new PipelineParameter().setName(name).setLocalCopy(
-      new LocalCopy().setDisk(mount.name).setPath(local.toString)
+      new LocalCopy().setDisk(mount.name).setPath(local.pathAsString)
     )
   }
   val toGoogleRunParameter: String = gcs
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesWorkflowPaths.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesWorkflowPaths.scala
index 615525e2d..8b06f117b 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesWorkflowPaths.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesWorkflowPaths.scala
@@ -1,15 +1,12 @@
 package cromwell.backend.impl.jes
 
-import java.nio.file.Path
-
 import akka.actor.ActorSystem
 import com.typesafe.config.Config
 import cromwell.backend.impl.jes.JesAsyncBackendJobExecutionActor.WorkflowOptionKeys
 import cromwell.backend.io.WorkflowPaths
 import cromwell.backend.{BackendJobDescriptorKey, BackendWorkflowDescriptor}
 import cromwell.core.WorkflowOptions
-import cromwell.core.path.PathBuilder
-import cromwell.core.path.PathImplicits._
+import cromwell.core.path.{Path, PathBuilder}
 import cromwell.filesystems.gcs.{GcsPathBuilderFactory, RetryableGcsPathBuilder}
 
 import scala.language.postfixOps
@@ -45,7 +42,7 @@ class JesWorkflowPaths(val workflowDescriptor: BackendWorkflowDescriptor,
     // The default auth file bucket is always at the root of the root workflow
     val defaultBucket = executionRoot.resolve(workflowDescriptor.rootWorkflow.unqualifiedName).resolve(workflowDescriptor.rootWorkflowId.toString)
 
-    val bucket = workflowDescriptor.workflowOptions.get(JesWorkflowPaths.AuthFilePathOptionKey) getOrElse defaultBucket.toRealString
+    val bucket = workflowDescriptor.workflowOptions.get(JesWorkflowPaths.AuthFilePathOptionKey) getOrElse defaultBucket.pathAsString
     val authBucket = GcsPathBuilderFactory(genomicsCredentials).withOptions(workflowOptions).build(bucket) recover {
       case ex => throw new Exception(s"Invalid gcs auth_bucket path $bucket", ex)
     } get
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/RunStatus.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/RunStatus.scala
index 45684bb11..c2a260b0b 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/RunStatus.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/RunStatus.scala
@@ -1,9 +1,8 @@
 package cromwell.backend.impl.jes
 
-import java.nio.file.Path
-
 import cromwell.backend.impl.jes.errors.JesError
 import cromwell.core.ExecutionEvent
+import cromwell.core.path.Path
 
 sealed trait RunStatus {
   import RunStatus._
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/errors/JesError.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/errors/JesError.scala
index 2952ef70e..225941b73 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/errors/JesError.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/errors/JesError.scala
@@ -1,8 +1,7 @@
 package cromwell.backend.impl.jes.errors
 
-import java.nio.file.Path
-
 import cromwell.backend.impl.jes.RunStatus
+import cromwell.core.path.Path
 
 import scala.util.Try
 
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/errors/JesKnownJobFailure.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/errors/JesKnownJobFailure.scala
index 1ca9a771f..dca8b2595 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/errors/JesKnownJobFailure.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/errors/JesKnownJobFailure.scala
@@ -1,15 +1,14 @@
 package cromwell.backend.impl.jes.errors
 
-import java.nio.file.Path
-
 import cromwell.backend.async.KnownJobFailureException
-import cromwell.core.path.PathImplicits._
+import cromwell.core.path.Path
 
 sealed trait JesKnownJobFailure extends KnownJobFailureException
 
-case class FailedToDelocalizeFailure(message: String, jobTag: String, stderrPath: Option[Path]) extends JesKnownJobFailure {
+case class FailedToDelocalizeFailure(message: String, jobTag: String, stderrPath: Option[Path])
+  extends JesKnownJobFailure {
   lazy val stderrMessage = stderrPath map { p =>
-    s"3) Look into the stderr (${p.toRealString}) file for evidence that some of the output files the command is expected to create were not created."
+    s"3) Look into the stderr (${p.pathAsString}) file for evidence that some of the output files the command is expected to create were not created."
   } getOrElse ""
   
   lazy val missingFilesMessage = if (message.contains("No URLs matched")) {
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/JesAttachedDisk.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/JesAttachedDisk.scala
index 38e702dc2..672ab159a 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/JesAttachedDisk.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/JesAttachedDisk.scala
@@ -1,13 +1,12 @@
 package cromwell.backend.impl.jes.io
 
-import java.nio.file.{Path, Paths}
-
 import cats.data.Validated._
 import cats.syntax.cartesian._
 import cats.syntax.validated._
 import com.google.api.services.genomics.model.Disk
-import lenthall.validation.ErrorOr._
+import cromwell.core.path.{DefaultPathBuilder, Path}
 import lenthall.exception.MessageAggregation
+import lenthall.validation.ErrorOr._
 import wdl4s.values._
 
 import scala.util.Try
@@ -29,7 +28,7 @@ object JesAttachedDisk {
         }
       case MountedDiskPattern(mountPoint, sizeGb, diskType) =>
         (validateLong(sizeGb) |@| validateDiskType(diskType)) map { (s, dt) =>
-          JesEmptyMountedDisk(dt, s.toInt, Paths.get(mountPoint))
+          JesEmptyMountedDisk(dt, s.toInt, DefaultPathBuilder.get(mountPoint))
         }
       case _ => s"Disk strings should be of the format 'local-disk SIZE TYPE' or '/mount/point SIZE TYPE'".invalidNel
     }
@@ -72,17 +71,17 @@ trait JesAttachedDisk {
       .setType(diskType.googleTypeName)
       .setAutoDelete(true)
       .setSizeGb(sizeGb)
-      .setMountPoint(mountPoint.toAbsolutePath.toString)
+      .setMountPoint(mountPoint.toAbsolutePath.pathAsString)
   }
 }
 
 case class JesEmptyMountedDisk(diskType: DiskType, sizeGb: Int, mountPoint: Path) extends JesAttachedDisk {
-  val name = s"d-${mountPoint.toString.md5Sum}"
+  val name = s"d-${mountPoint.pathAsString.md5Sum}"
   override def toString: String = s"$mountPoint $sizeGb ${diskType.diskTypeName}"
 }
 
 object JesWorkingDisk {
-  val MountPoint: Path = Paths.get("/cromwell_root")
+  val MountPoint: Path = DefaultPathBuilder.get("/cromwell_root")
   val Name = "local-disk"
   val Default = JesWorkingDisk(DiskType.SSD, 10)
 }
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/package.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/package.scala
index acfc61f0a..592add17c 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/package.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/package.scala
@@ -1,20 +1,18 @@
 package cromwell.backend.impl.jes
 
-import java.nio.file.{Files, Path}
-
 import com.google.api.client.http.HttpResponseException
 import com.google.cloud.storage.contrib.nio.CloudStorageOptions
+import cromwell.core.path.Path
 
 package object io {
   implicit class PathEnhanced(val path: Path) extends AnyVal {
-    import better.files._
 
-    def writeAsJson(content: String): File = {
-      Files.write(path, content.getBytes, CloudStorageOptions.withMimeType("application/json"))
+    def writeAsJson(content: String): Path = {
+      path.writeBytes(content.getBytes.toIterator)(Seq(CloudStorageOptions.withMimeType("application/json")))
     }
 
-    def writeAsText(content: String): File = {
-      Files.write(path, content.getBytes, CloudStorageOptions.withMimeType("text/plain"))
+    def writeAsText(content: String): Path = {
+      path.writeBytes(content.getBytes.toIterator)(Seq(CloudStorageOptions.withMimeType("text/plain")))
     }
   }
 
diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala
index 3ddc1bc8d..15d5395fa 100644
--- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala
+++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala
@@ -1,11 +1,9 @@
 package cromwell.backend.impl.jes
 
-import java.nio.file.Paths
 import java.util.UUID
 
 import akka.actor.{ActorRef, Props}
 import akka.testkit.{ImplicitSender, TestActorRef, TestDuration, TestProbe}
-import com.google.cloud.storage.contrib.nio.CloudStoragePath
 import cromwell.backend.BackendJobExecutionActor.BackendJobExecutionResponse
 import cromwell.backend._
 import cromwell.backend.async.AsyncBackendJobExecutionActor.{Execute, ExecutionMode}
@@ -19,10 +17,9 @@ import cromwell.backend.wdl.WdlFileMapper
 import cromwell.core._
 import cromwell.core.labels.Labels
 import cromwell.core.logging.JobLogger
-import cromwell.core.path.PathBuilder
-import cromwell.core.path.PathImplicits._
+import cromwell.core.path.{DefaultPathBuilder, PathBuilder}
 import cromwell.filesystems.gcs.auth.GoogleAuthMode.NoAuthMode
-import cromwell.filesystems.gcs.{GcsPathBuilder, GcsPathBuilderFactory}
+import cromwell.filesystems.gcs.{GcsPath, GcsPathBuilder, GcsPathBuilderFactory}
 import cromwell.util.SampleWdl
 import org.scalatest._
 import org.scalatest.prop.Tables.Table
@@ -378,21 +375,21 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
     val jesInputs = testActorRef.underlyingActor.generateJesInputs(jobDescriptor)
     jesInputs should have size 8
     jesInputs should contain(JesFileInput(
-      "stringToFileMap-0", "gs://path/to/stringTofile1", Paths.get("path/to/stringTofile1"), workingDisk))
+      "stringToFileMap-0", "gs://path/to/stringTofile1", DefaultPathBuilder.get("path/to/stringTofile1"), workingDisk))
     jesInputs should contain(JesFileInput(
-      "stringToFileMap-1", "gs://path/to/stringTofile2", Paths.get("path/to/stringTofile2"), workingDisk))
+      "stringToFileMap-1", "gs://path/to/stringTofile2", DefaultPathBuilder.get("path/to/stringTofile2"), workingDisk))
     jesInputs should contain(JesFileInput(
-      "fileToStringMap-0", "gs://path/to/fileToString1", Paths.get("path/to/fileToString1"), workingDisk))
+      "fileToStringMap-0", "gs://path/to/fileToString1", DefaultPathBuilder.get("path/to/fileToString1"), workingDisk))
     jesInputs should contain(JesFileInput(
-      "fileToStringMap-1", "gs://path/to/fileToString2", Paths.get("path/to/fileToString2"), workingDisk))
+      "fileToStringMap-1", "gs://path/to/fileToString2", DefaultPathBuilder.get("path/to/fileToString2"), workingDisk))
     jesInputs should contain(JesFileInput(
-      "fileToFileMap-0", "gs://path/to/fileToFile1Key", Paths.get("path/to/fileToFile1Key"), workingDisk))
+      "fileToFileMap-0", "gs://path/to/fileToFile1Key", DefaultPathBuilder.get("path/to/fileToFile1Key"), workingDisk))
     jesInputs should contain(JesFileInput(
-      "fileToFileMap-1", "gs://path/to/fileToFile1Value", Paths.get("path/to/fileToFile1Value"), workingDisk))
+      "fileToFileMap-1", "gs://path/to/fileToFile1Value", DefaultPathBuilder.get("path/to/fileToFile1Value"), workingDisk))
     jesInputs should contain(JesFileInput(
-      "fileToFileMap-2", "gs://path/to/fileToFile2Key", Paths.get("path/to/fileToFile2Key"), workingDisk))
+      "fileToFileMap-2", "gs://path/to/fileToFile2Key", DefaultPathBuilder.get("path/to/fileToFile2Key"), workingDisk))
     jesInputs should contain(JesFileInput(
-      "fileToFileMap-3", "gs://path/to/fileToFile2Value", Paths.get("path/to/fileToFile2Value"), workingDisk))
+      "fileToFileMap-3", "gs://path/to/fileToFile2Value", DefaultPathBuilder.get("path/to/fileToFile2Value"), workingDisk))
   }
 
   def makeJesActorRef(sampleWdl: SampleWdl, callName: LocallyQualifiedName, inputs: Map[FullyQualifiedName, WdlValue],
@@ -424,11 +421,11 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
     val workflowId = jesBackend.workflowId
     val jesInputs = jesBackend.generateJesInputs(jobDescriptor)
     jesInputs should have size 1
-    jesInputs should contain(JesFileInput("in-0", "gs://blah/b/c.txt", Paths.get("blah/b/c.txt"), workingDisk))
+    jesInputs should contain(JesFileInput("in-0", "gs://blah/b/c.txt", DefaultPathBuilder.get("blah/b/c.txt"), workingDisk))
     val jesOutputs = jesBackend.generateJesOutputs(jobDescriptor)
     jesOutputs should have size 1
     jesOutputs should contain(JesFileOutput("out",
-      s"gs://my-cromwell-workflows-bucket/file_passing/$workflowId/call-a/out", Paths.get("out"), workingDisk))
+      s"gs://my-cromwell-workflows-bucket/file_passing/$workflowId/call-a/out", DefaultPathBuilder.get("out"), workingDisk))
   }
 
   it should "generate correct JesInputs when a command line contains a write_lines call in it" in {
@@ -448,7 +445,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
     val jesInputs = jesBackend.generateJesInputs(jobDescriptor)
     jesInputs should have size 1
     jesInputs should contain(JesFileInput(
-      "c6fd5c91-0", "gs://some/path/file.txt", Paths.get("some/path/file.txt"), workingDisk))
+      "c6fd5c91-0", "gs://some/path/file.txt", DefaultPathBuilder.get("some/path/file.txt"), workingDisk))
     val jesOutputs = jesBackend.generateJesOutputs(jobDescriptor)
     jesOutputs should have size 0
   }
@@ -478,8 +475,8 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
 
     val jesInputs = testActorRef.underlyingActor.generateJesInputs(jobDescriptor)
     jesInputs should have size 2
-    jesInputs should contain(JesFileInput("fileArray-0", "gs://path/to/file1", Paths.get("path/to/file1"), workingDisk))
-    jesInputs should contain(JesFileInput("fileArray-1", "gs://path/to/file2", Paths.get("path/to/file2"), workingDisk))
+    jesInputs should contain(JesFileInput("fileArray-0", "gs://path/to/file1", DefaultPathBuilder.get("path/to/file1"), workingDisk))
+    jesInputs should contain(JesFileInput("fileArray-1", "gs://path/to/file2", DefaultPathBuilder.get("path/to/file2"), workingDisk))
   }
 
   it should "generate correct JesFileInputs from a WdlFile" in {
@@ -507,22 +504,22 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
 
     val jesInputs = testActorRef.underlyingActor.generateJesInputs(jobDescriptor)
     jesInputs should have size 2
-    jesInputs should contain(JesFileInput("file1-0", "gs://path/to/file1", Paths.get("path/to/file1"), workingDisk))
-    jesInputs should contain(JesFileInput("file2-0", "gs://path/to/file2", Paths.get("path/to/file2"), workingDisk))
+    jesInputs should contain(JesFileInput("file1-0", "gs://path/to/file1", DefaultPathBuilder.get("path/to/file1"), workingDisk))
+    jesInputs should contain(JesFileInput("file2-0", "gs://path/to/file2", DefaultPathBuilder.get("path/to/file2"), workingDisk))
   }
 
   it should "convert local Paths back to corresponding GCS paths in JesOutputs" in {
     val jesOutputs = Set(
       JesFileOutput("/cromwell_root/path/to/file1", "gs://path/to/file1",
-        Paths.get("/cromwell_root/path/to/file1"), workingDisk),
+        DefaultPathBuilder.get("/cromwell_root/path/to/file1"), workingDisk),
       JesFileOutput("/cromwell_root/path/to/file2", "gs://path/to/file2",
-        Paths.get("/cromwell_root/path/to/file2"), workingDisk),
+        DefaultPathBuilder.get("/cromwell_root/path/to/file2"), workingDisk),
       JesFileOutput("/cromwell_root/path/to/file3", "gs://path/to/file3",
-        Paths.get("/cromwell_root/path/to/file3"), workingDisk),
+        DefaultPathBuilder.get("/cromwell_root/path/to/file3"), workingDisk),
       JesFileOutput("/cromwell_root/path/to/file4", "gs://path/to/file4",
-        Paths.get("/cromwell_root/path/to/file4"), workingDisk),
+        DefaultPathBuilder.get("/cromwell_root/path/to/file4"), workingDisk),
       JesFileOutput("/cromwell_root/path/to/file5", "gs://path/to/file5",
-        Paths.get("/cromwell_root/path/to/file5"), workingDisk)
+        DefaultPathBuilder.get("/cromwell_root/path/to/file5"), workingDisk)
     )
     val outputValues = Seq(
       WdlFile("/cromwell_root/path/to/file1"),
@@ -584,7 +581,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
       props, s"TestableJesJobExecutionActor-${jobDescriptor.workflowDescriptor.id}")
 
     testActorRef.underlyingActor.monitoringScript shouldBe
-      Some(JesFileInput("monitoring-in", "gs://path/to/script", Paths.get("/cromwell_root/monitoring.sh"), workingDisk))
+      Some(JesFileInput("monitoring-in", "gs://path/to/script", DefaultPathBuilder.get("/cromwell_root/monitoring.sh"), workingDisk))
   }
 
   it should "not create a JesFileInput for the monitoring script, when not specified" in {
@@ -629,14 +626,14 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
 
     val jesBackend = testActorRef.underlyingActor
 
-    jesBackend.jesCallPaths.stdout should be(a[CloudStoragePath])
-    jesBackend.jesCallPaths.stdout.toRealString shouldBe
+    jesBackend.jesCallPaths.stdout should be(a[GcsPath])
+    jesBackend.jesCallPaths.stdout.pathAsString shouldBe
       "gs://path/to/gcs_root/wf_hello/e6236763-c518-41d0-9688-432549a8bf7c/call-hello/hello-stdout.log"
-    jesBackend.jesCallPaths.stderr should be(a[CloudStoragePath])
-    jesBackend.jesCallPaths.stderr.toRealString shouldBe
+    jesBackend.jesCallPaths.stderr should be(a[GcsPath])
+    jesBackend.jesCallPaths.stderr.pathAsString shouldBe
       "gs://path/to/gcs_root/wf_hello/e6236763-c518-41d0-9688-432549a8bf7c/call-hello/hello-stderr.log"
-    jesBackend.jesCallPaths.jesLogPath should be(a[CloudStoragePath])
-    jesBackend.jesCallPaths.jesLogPath.toRealString shouldBe
+    jesBackend.jesCallPaths.jesLogPath should be(a[GcsPath])
+    jesBackend.jesCallPaths.jesLogPath.pathAsString shouldBe
       "gs://path/to/gcs_root/wf_hello/e6236763-c518-41d0-9688-432549a8bf7c/call-hello/hello.log"
   }
 
@@ -661,14 +658,14 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
 
     val jesBackend = testActorRef.underlyingActor
 
-    jesBackend.jesCallPaths.stdout should be(a[CloudStoragePath])
-    jesBackend.jesCallPaths.stdout.toRealString shouldBe
+    jesBackend.jesCallPaths.stdout should be(a[GcsPath])
+    jesBackend.jesCallPaths.stdout.pathAsString shouldBe
       "gs://path/to/gcs_root/w/e6236763-c518-41d0-9688-432549a8bf7d/call-B/shard-2/B-2-stdout.log"
-    jesBackend.jesCallPaths.stderr should be(a[CloudStoragePath])
-    jesBackend.jesCallPaths.stderr.toRealString shouldBe
+    jesBackend.jesCallPaths.stderr should be(a[GcsPath])
+    jesBackend.jesCallPaths.stderr.pathAsString shouldBe
       "gs://path/to/gcs_root/w/e6236763-c518-41d0-9688-432549a8bf7d/call-B/shard-2/B-2-stderr.log"
-    jesBackend.jesCallPaths.jesLogPath should be(a[CloudStoragePath])
-    jesBackend.jesCallPaths.jesLogPath.toRealString shouldBe
+    jesBackend.jesCallPaths.jesLogPath should be(a[GcsPath])
+    jesBackend.jesCallPaths.jesLogPath.pathAsString shouldBe
       "gs://path/to/gcs_root/w/e6236763-c518-41d0-9688-432549a8bf7d/call-B/shard-2/B-2.log"
   }
 
diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAttachedDiskSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAttachedDiskSpec.scala
index 3e0f19943..37eb1b14c 100644
--- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAttachedDiskSpec.scala
+++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAttachedDiskSpec.scala
@@ -1,9 +1,8 @@
 package cromwell.backend.impl.jes
 
-import java.nio.file.Paths
-
 import com.google.api.services.genomics.model.Disk
 import cromwell.backend.impl.jes.io.{DiskType, JesAttachedDisk, JesEmptyMountedDisk, JesWorkingDisk}
+import cromwell.core.path.DefaultPathBuilder
 import org.scalatest.prop.TableDrivenPropertyChecks._
 import org.scalatest.prop.Tables.Table
 import org.scalatest.{FlatSpec, Matchers, TryValues}
@@ -13,8 +12,8 @@ import scala.util.Failure
 class JesAttachedDiskSpec extends FlatSpec with Matchers with TryValues {
   val validTable = Table(
     ("unparsed", "parsed"),
-    ("/mnt 3 SSD", JesEmptyMountedDisk(DiskType.SSD, 3, Paths.get("/mnt"))),
-    ("/mnt/my_path 10 HDD", JesEmptyMountedDisk(DiskType.HDD, 10, Paths.get("/mnt/my_path"))),
+    ("/mnt 3 SSD", JesEmptyMountedDisk(DiskType.SSD, 3, DefaultPathBuilder.get("/mnt"))),
+    ("/mnt/my_path 10 HDD", JesEmptyMountedDisk(DiskType.HDD, 10, DefaultPathBuilder.get("/mnt/my_path"))),
     ("local-disk 100 SSD", JesWorkingDisk(DiskType.SSD, 100)),
     ("local-disk 100 LOCAL", JesWorkingDisk(DiskType.LOCAL, 100))
   )
diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesCallPathsSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesCallPathsSpec.scala
index 2c508523b..b8859d6cb 100644
--- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesCallPathsSpec.scala
+++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesCallPathsSpec.scala
@@ -2,7 +2,6 @@ package cromwell.backend.impl.jes
 
 import cromwell.backend.BackendSpec
 import cromwell.core.TestKitSuite
-import cromwell.core.path.PathImplicits._
 import cromwell.filesystems.gcs.auth.GoogleAuthModeSpec
 import cromwell.util.SampleWdl
 import org.scalatest.{FlatSpecLike, Matchers}
@@ -38,13 +37,13 @@ class JesCallPathsSpec extends TestKitSuite with FlatSpecLike with Matchers with
     val jesConfiguration = new JesConfiguration(JesBackendConfigurationDescriptor)
 
     val callPaths = JesJobPaths(jobDescriptorKey, workflowDescriptor, jesConfiguration)
-    callPaths.returnCode.toRealString should
+    callPaths.returnCode.pathAsString should
       be(s"gs://my-cromwell-workflows-bucket/wf_hello/${workflowDescriptor.id}/call-hello/hello-rc.txt")
-    callPaths.stdout.toRealString should
+    callPaths.stdout.pathAsString should
       be(s"gs://my-cromwell-workflows-bucket/wf_hello/${workflowDescriptor.id}/call-hello/hello-stdout.log")
-    callPaths.stderr.toRealString should
+    callPaths.stderr.pathAsString should
       be(s"gs://my-cromwell-workflows-bucket/wf_hello/${workflowDescriptor.id}/call-hello/hello-stderr.log")
-    callPaths.jesLogPath.toRealString should
+    callPaths.jesLogPath.pathAsString should
       be(s"gs://my-cromwell-workflows-bucket/wf_hello/${workflowDescriptor.id}/call-hello/hello.log")
   }
 
@@ -56,7 +55,7 @@ class JesCallPathsSpec extends TestKitSuite with FlatSpecLike with Matchers with
     val jesConfiguration = new JesConfiguration(JesBackendConfigurationDescriptor)
 
     val callPaths = JesJobPaths(jobDescriptorKey, workflowDescriptor, jesConfiguration)
-    callPaths.callContext.root.toRealString should
+    callPaths.callContext.root.pathAsString should
       be(s"gs://my-cromwell-workflows-bucket/wf_hello/${workflowDescriptor.id}/call-hello")
     callPaths.callContext.stdout should
       be(s"gs://my-cromwell-workflows-bucket/wf_hello/${workflowDescriptor.id}/call-hello/hello-stdout.log")
diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesConfigurationSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesConfigurationSpec.scala
index f94991670..23b029086 100644
--- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesConfigurationSpec.scala
+++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesConfigurationSpec.scala
@@ -1,8 +1,8 @@
 package cromwell.backend.impl.jes
 
-import better.files.File
 import com.typesafe.config.{ConfigFactory, ConfigValueFactory}
 import cromwell.backend.BackendConfigurationDescriptor
+import cromwell.core.path.DefaultPathBuilder
 import org.scalatest.prop.TableDrivenPropertyChecks
 import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers}
 
@@ -10,11 +10,10 @@ class JesConfigurationSpec extends FlatSpec with Matchers with TableDrivenProper
 
   behavior of "JesConfigurationSpec"
 
-  val mockFile = File.newTemporaryFile()
+  val mockFile = DefaultPathBuilder.createTempFile()
 
   override def afterAll(): Unit = {
-    mockFile.delete(true)
-
+    mockFile.delete(swallowIOExceptions = true)
     ()
   }
 
diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesWorkflowPathsSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesWorkflowPathsSpec.scala
index 2b3794356..8cc12fd4b 100644
--- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesWorkflowPathsSpec.scala
+++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesWorkflowPathsSpec.scala
@@ -2,7 +2,6 @@ package cromwell.backend.impl.jes
 
 import cromwell.backend.BackendSpec
 import cromwell.core.TestKitSuite
-import cromwell.core.path.PathImplicits._
 import cromwell.filesystems.gcs.auth.GoogleAuthModeSpec
 import cromwell.util.SampleWdl
 import org.scalatest.{FlatSpecLike, Matchers}
@@ -21,10 +20,10 @@ class JesWorkflowPathsSpec extends TestKitSuite with FlatSpecLike with Matchers
     val jesConfiguration = new JesConfiguration(JesBackendConfigurationDescriptor)
 
     val workflowPaths = JesWorkflowPaths(workflowDescriptor, jesConfiguration)(system)
-    workflowPaths.executionRoot.toRealString should be("gs://my-cromwell-workflows-bucket/")
-    workflowPaths.workflowRoot.toRealString should
+    workflowPaths.executionRoot.pathAsString should be("gs://my-cromwell-workflows-bucket/")
+    workflowPaths.workflowRoot.pathAsString should
       be(s"gs://my-cromwell-workflows-bucket/wf_hello/${workflowDescriptor.id}/")
-    workflowPaths.gcsAuthFilePath.toRealString should
+    workflowPaths.gcsAuthFilePath.pathAsString should
       be(s"gs://my-cromwell-workflows-bucket/wf_hello/${workflowDescriptor.id}/${workflowDescriptor.id}_auth.json")
   }
 }
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigAsyncJobExecutionActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigAsyncJobExecutionActor.scala
index da3ee2e87..e2eaeaee3 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigAsyncJobExecutionActor.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigAsyncJobExecutionActor.scala
@@ -1,12 +1,9 @@
 package cromwell.backend.impl.sfs.config
 
-import java.nio.file.Path
-
-import better.files._
 import cromwell.backend.impl.sfs.config.ConfigConstants._
 import cromwell.backend.sfs._
 import cromwell.backend.standard.{StandardAsyncExecutionActorParams, StandardAsyncJob}
-import cromwell.core.path.PathFactory._
+import cromwell.core.path.Path
 import wdl4s._
 import wdl4s.expression.NoFunctions
 import wdl4s.values.WdlString
@@ -31,7 +28,7 @@ sealed trait ConfigAsyncJobExecutionActor extends SharedFileSystemAsyncJobExecut
     * will grab the job id from the stdout.
     */
   override lazy val processArgs: SharedFileSystemCommand = {
-    val submitScript = pathPlusSuffix(jobPaths.script, "submit")
+    val submitScript = jobPaths.script.plusExt("submit")
     val submitInputs = standardInputs ++ dockerInputs ++ runtimeAttributeInputs
     val submitTaskName = if (isDockerRun) SubmitDockerTask else SubmitTask
     writeTaskScript(submitScript, submitTaskName, submitInputs)
@@ -46,19 +43,16 @@ sealed trait ConfigAsyncJobExecutionActor extends SharedFileSystemAsyncJobExecut
     * @param taskName The name of the task to retrieve from the precomputed wdl namespace.
     * @param inputs   The customized inputs to this task.
     */
-  def writeTaskScript(script: File, taskName: String, inputs: WorkflowCoercedInputs): Unit = {
+  def writeTaskScript(script: Path, taskName: String, inputs: WorkflowCoercedInputs): Unit = {
     val task = configInitializationData.wdlNamespace.findTask(taskName).
       getOrElse(throw new RuntimeException(s"Unable to find task $taskName"))
     val inputsWithFqns = inputs map { case (k, v) => s"$taskName.$k" -> v }
     val command = task.instantiateCommand(task.inputsFromMap(inputsWithFqns), NoFunctions).get
     jobLogger.info(s"executing: $command")
     val scriptBody =
-      s"""
-
-#!/bin/bash
-$command
-
-""".trim + "\n"
+      s"""|#!/bin/bash
+          |SCRIPT_COMMAND
+          |""".stripMargin.replace("SCRIPT_COMMAND", command)
     script.write(scriptBody)
     ()
   }
@@ -70,10 +64,10 @@ $command
   private lazy val standardInputs: WorkflowCoercedInputs = {
     Map(
       JobNameInput -> WdlString(jobName),
-      CwdInput -> WdlString(jobPaths.callRoot.toString),
-      StdoutInput -> WdlString(jobPaths.stdout.toString),
-      StderrInput -> WdlString(jobPaths.stderr.toString),
-      ScriptInput -> WdlString(jobPaths.script.toString)
+      CwdInput -> WdlString(jobPaths.callRoot.pathAsString),
+      StdoutInput -> WdlString(jobPaths.stdout.pathAsString),
+      StderrInput -> WdlString(jobPaths.stderr.pathAsString),
+      ScriptInput -> WdlString(jobPaths.script.pathAsString)
     )
   }
 
@@ -83,7 +77,7 @@ $command
   private lazy val dockerInputs: WorkflowCoercedInputs = {
     if (isDockerRun) {
       Map(
-        DockerCwdInput -> WdlString(jobPathsWithDocker.callDockerRoot.toString)
+        DockerCwdInput -> WdlString(jobPathsWithDocker.callDockerRoot.pathAsString)
       )
     } else {
       Map.empty
@@ -132,7 +126,7 @@ class DispatchedConfigAsyncJobExecutionActor(override val standardParams: Standa
     */
   override def getJob(exitValue: Int, stdout: Path, stderr: Path): StandardAsyncJob = {
     val jobIdRegex = configurationDescriptor.backendConfig.getString(JobIdRegexConfig).r
-    val output = File(stdout).contentAsString.stripLineEnd
+    val output = stdout.contentAsString.stripLineEnd
     output match {
       case jobIdRegex(jobId) => StandardAsyncJob(jobId)
       case _ =>
@@ -170,7 +164,7 @@ class DispatchedConfigAsyncJobExecutionActor(override val standardParams: Standa
     * @return A runnable command.
     */
   private def jobScriptArgs(job: StandardAsyncJob, suffix: String, task: String): SharedFileSystemCommand = {
-    val script = pathPlusSuffix(jobPaths.script, suffix)
+    val script = jobPaths.script.plusExt(suffix)
     writeTaskScript(script, task, Map(JobIdInput -> WdlString(job.jobId)))
     SharedFileSystemCommand("/bin/bash", script)
   }
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendFileHashing.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendFileHashing.scala
index f8ff92cfd..afe8011c3 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendFileHashing.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendFileHashing.scala
@@ -1,7 +1,6 @@
 package cromwell.backend.impl.sfs.config
 
 import akka.event.LoggingAdapter
-import better.files._
 import cromwell.backend.callcaching.FileHashingActor.SingleFileHashRequest
 import cromwell.core.path.DefaultPathBuilder
 import cromwell.util.TryWithResource._
@@ -15,7 +14,7 @@ private[config] object ConfigBackendFileHashing {
       case failure => throw new RuntimeException("Failed to construct path to hash", failure)
     } get
 
-    tryWithResource(() => File(path).newInputStream) { inputStream =>
+    tryWithResource(() => path.newInputStream) { inputStream =>
       org.apache.commons.codec.digest.DigestUtils.md5Hex(inputStream)
     }
   }
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategy.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategy.scala
index 3f528022f..2d89d1267 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategy.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategy.scala
@@ -1,12 +1,10 @@
 package cromwell.backend.impl.sfs.config
 
 import akka.event.LoggingAdapter
-import better.files.File
 import com.typesafe.config.Config
 import cromwell.backend.callcaching.FileHashingActor.SingleFileHashRequest
 import cromwell.backend.standard.StandardInitializationData
-import cromwell.core.path.PathFactory
-import cromwell.util.FileUtil._
+import cromwell.core.path.{Path, PathFactory}
 import cromwell.util.TryWithResource._
 import net.ceedubs.ficus.Ficus._
 import org.apache.commons.codec.digest.DigestUtils
@@ -33,7 +31,7 @@ object ConfigHashingStrategy {
 
 abstract class ConfigHashingStrategy {
   def checkSiblingMd5: Boolean
-  protected def hash(file: File): Try[String]
+  protected def hash(file: Path): Try[String]
   protected def description: String
 
   protected lazy val checkSiblingMessage: String =
@@ -42,7 +40,7 @@ abstract class ConfigHashingStrategy {
   def getHash(request: SingleFileHashRequest, log: LoggingAdapter): Try[String] = {
     def usingStandardInitData(initData: StandardInitializationData) = {
       val pathBuilders = initData.workflowPaths.pathBuilders
-      val file = PathFactory.buildFile(request.file.valueString, pathBuilders).followSymlinks
+      val file = PathFactory.buildPath(request.file.valueString, pathBuilders).followSymbolicLinks
 
       if (checkSiblingMd5) {
         precomputedMd5(file) match {
@@ -58,7 +56,7 @@ abstract class ConfigHashingStrategy {
     }
   }
 
-  private def precomputedMd5(file: File): Option[File] = {
+  private def precomputedMd5(file: Path): Option[Path] = {
     val md5 = file.sibling(s"${file.name}.md5")
     if (md5.exists) Option(md5) else None
   }
@@ -69,15 +67,15 @@ abstract class ConfigHashingStrategy {
 }
 
 final case class HashPathStrategy(checkSiblingMd5: Boolean) extends ConfigHashingStrategy {
-  override def hash(file: File): Try[String] = {
-    Try(DigestUtils.md5Hex(file.path.toAbsolutePath.toString))
+  override def hash(file: Path): Try[String] = {
+    Try(DigestUtils.md5Hex(file.toAbsolutePath.pathAsString))
   }
 
   override val description = "hash file path"
 }
 
 final case class HashFileStrategy(checkSiblingMd5: Boolean) extends ConfigHashingStrategy {
-  override protected def hash(file: File): Try[String] = {
+  override protected def hash(file: Path): Try[String] = {
     tryWithResource(() => file.newInputStream) { DigestUtils.md5Hex }
   }
 
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/BackgroundAsyncJobExecutionActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/BackgroundAsyncJobExecutionActor.scala
index 5d5ff1eb9..44c584187 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/BackgroundAsyncJobExecutionActor.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/BackgroundAsyncJobExecutionActor.scala
@@ -1,14 +1,11 @@
 package cromwell.backend.sfs
 
-import java.nio.file.Path
-
-import better.files._
 import cromwell.backend.standard.StandardAsyncJob
-import cromwell.core.path.PathFactory._
+import cromwell.core.path.Path
 
 trait BackgroundAsyncJobExecutionActor extends SharedFileSystemAsyncJobExecutionActor {
 
-  lazy val backgroundScript: File = pathPlusSuffix(jobPaths.script, "background")
+  lazy val backgroundScript = jobPaths.script.plusExt("background")
 
   override def writeScriptContents(): Unit = {
     super.writeScriptContents()
@@ -30,14 +27,14 @@ trait BackgroundAsyncJobExecutionActor extends SharedFileSystemAsyncJobExecution
   }
 
   override def makeProcessRunner(): ProcessRunner = {
-    val stdout = pathPlusSuffix(jobPaths.stdout, "background")
-    val stderr = pathPlusSuffix(jobPaths.stderr, "background")
+    val stdout = jobPaths.stdout.plusExt("background")
+    val stderr = jobPaths.stderr.plusExt("background")
     val argv = Seq("/bin/bash", backgroundScript)
-    new ProcessRunner(argv, stdout.path, stderr.path)
+    new ProcessRunner(argv, stdout, stderr)
   }
 
   override def getJob(exitValue: Int, stdout: Path, stderr: Path): StandardAsyncJob = {
-    val pid = File(stdout).contentAsString.stripLineEnd
+    val pid = stdout.contentAsString.stripLineEnd
     StandardAsyncJob(pid)
   }
 
@@ -46,12 +43,12 @@ trait BackgroundAsyncJobExecutionActor extends SharedFileSystemAsyncJobExecution
   }
 
   override def killArgs(job: StandardAsyncJob): SharedFileSystemCommand = {
-    val killScript = pathPlusSuffix(jobPaths.script, "kill")
+    val killScript = jobPaths.script.plusExt("kill")
     writeKillScript(killScript, job)
     SharedFileSystemCommand("/bin/bash", killScript)
   }
 
-  private def writeKillScript(killScript: File, job: StandardAsyncJob): Unit = {
+  private def writeKillScript(killScript: Path, job: StandardAsyncJob): Unit = {
     /*
     Use pgrep to find the children of a process, and recursively kill the children before killing the parent.
      */
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/ProcessRunner.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/ProcessRunner.scala
index bf6bebb28..64401db64 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/ProcessRunner.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/ProcessRunner.scala
@@ -1,6 +1,6 @@
 package cromwell.backend.sfs
 
-import java.nio.file.Path
+import cromwell.core.path.Path
 
 /**
   * Runs a process and sends the stdout and stderr to a file path.
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala
index f785872a7..272be2787 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala
@@ -1,7 +1,6 @@
 package cromwell.backend.sfs
 
 import java.io.{FileNotFoundException, IOException}
-import java.nio.file.{Path, Paths}
 
 import cats.instances.try_._
 import cats.syntax.functor._
@@ -10,9 +9,7 @@ import com.typesafe.scalalogging.StrictLogging
 import cromwell.backend.io.JobPaths
 import cromwell.backend.wdl.WdlFileMapper
 import cromwell.core.CromwellFatalExceptionMarker
-import cromwell.core.path.FileImplicits._
-import cromwell.core.path.PathFactory
-import cromwell.core.path.PathFactory._
+import cromwell.core.path.{DefaultPath, DefaultPathBuilder, Path, PathFactory}
 import lenthall.util.TryUtil
 import wdl4s.EvaluatedTaskInputs
 import wdl4s.values._
@@ -22,7 +19,6 @@ import scala.language.postfixOps
 import scala.util.{Failure, Success, Try}
 
 object SharedFileSystem extends StrictLogging {
-  import better.files._
 
   final case class AttemptedLookupResult(name: String, value: Try[WdlValue]) {
     def toPair: (String, Try[WdlValue]) = name -> value
@@ -36,34 +32,34 @@ object SharedFileSystem extends StrictLogging {
     }
   }
 
-  case class PairOfFiles(src: File, dst: File)
-  type DuplicationStrategy = (File, File) => Try[Unit]
+  case class PairOfFiles(src: Path, dst: Path)
+  type DuplicationStrategy = (Path, Path) => Try[Unit]
 
   /**
     * Return a `Success` result if the file has already been localized, otherwise `Failure`.
     */
-  private def localizePathAlreadyLocalized(originalPath: File, executionPath: File): Try[Unit] = {
+  private def localizePathAlreadyLocalized(originalPath: Path, executionPath: Path): Try[Unit] = {
     if (executionPath.exists) Success(()) else Failure(new RuntimeException(s"$originalPath doesn't exists"))
   }
 
-  private def localizePathViaCopy(originalPath: File, executionPath: File): Try[Unit] = {
+  private def localizePathViaCopy(originalPath: Path, executionPath: Path): Try[Unit] = {
     val action = Try {
       executionPath.parent.createPermissionedDirectories()
-      val executionTmpPath = pathPlusSuffix(executionPath, "tmp")
+      val executionTmpPath = executionPath.plusExt("tmp")
       originalPath.copyTo(executionTmpPath, overwrite = true).moveTo(executionPath, overwrite = true)
     }.void
     logOnFailure(action, "copy")
   }
 
-  private def localizePathViaHardLink(originalPath: File, executionPath: File): Try[Unit] = {
+  private def localizePathViaHardLink(originalPath: Path, executionPath: Path): Try[Unit] = {
     val action = Try {
       executionPath.parent.createPermissionedDirectories()
-      executionPath.linkTo(originalPath, symbolic = false)
+      executionPath.linkTo(originalPath)
     }.void
     logOnFailure(action, "hard link")
   }
 
-  private def localizePathViaSymbolicLink(originalPath: File, executionPath: File): Try[Unit] = {
+  private def localizePathViaSymbolicLink(originalPath: Path, executionPath: Path): Try[Unit] = {
       if (originalPath.isDirectory) Failure(new UnsupportedOperationException("Cannot localize directory with symbolic links"))
       else if (!originalPath.exists) Failure(new FileNotFoundException(originalPath.pathAsString))
       else {
@@ -80,10 +76,8 @@ object SharedFileSystem extends StrictLogging {
     action
   }
 
-  private def duplicate(description: String, source: File, dest: File, strategies: Stream[DuplicationStrategy]): Try[Unit] = {
-    import cromwell.util.FileUtil._
-
-    val attempts: Stream[Try[Unit]] = strategies.map(_ (source.followSymlinks, dest))
+  private def duplicate(description: String, source: Path, dest: Path, strategies: Stream[DuplicationStrategy]): Try[Unit] = {
+    val attempts: Stream[Try[Unit]] = strategies.map(_ (source.followSymbolicLinks, dest))
     attempts.find(_.isSuccess) getOrElse {
       TryUtil.sequence(attempts, s"Could not $description $source -> $dest").void
     }
@@ -92,7 +86,6 @@ object SharedFileSystem extends StrictLogging {
 
 trait SharedFileSystem extends PathFactory {
   import SharedFileSystem._
-  import better.files._
 
   def sharedFileSystemConfig: Config
 
@@ -132,9 +125,12 @@ trait SharedFileSystem extends PathFactory {
     localizePathAlreadyLocalized _ +: mappedDuplicationStrategies
   }
 
-  private def hostAbsoluteFilePath(callRoot: Path, pathString: String): File = {
+  private def hostAbsoluteFilePath(callRoot: Path, pathString: String): Path = {
     val wdlPath = PathFactory.buildPath(pathString, pathBuilders)
-    callRoot.resolve(wdlPath).toAbsolutePath
+    wdlPath match {
+      case _: DefaultPath if !wdlPath.isAbsolute => callRoot.resolve(wdlPath).toAbsolutePath
+      case _ => wdlPath
+    }
   }
 
   def outputMapper(job: JobPaths)(wdlValue: WdlValue): Try[WdlValue] = {
@@ -170,14 +166,8 @@ trait SharedFileSystem extends PathFactory {
   def localizeWdlFile(inputsRoot: Path, docker: Boolean)(value: WdlFile): WdlFile = {
     val strategies = if (docker) DockerLocalizers else Localizers
 
-    // Use URI to identify protocol scheme and strip it out
-    def stripProtocolScheme(path: Path): Path = {
-      val uri = path.toUri
-      val host = Option(uri.getHost)
-      val uriPath = uri.getPath
-
-      host map { h => Paths.get(h, uriPath) } getOrElse Paths.get(uriPath)
-    }
+    // Strip the protocol scheme
+    def stripProtocolScheme(path: Path): Path = DefaultPathBuilder.get(path.pathWithoutScheme)
 
     /*
       * Transform an original input path to a path in the call directory.
@@ -185,13 +175,13 @@ trait SharedFileSystem extends PathFactory {
       */
 
     def toCallPath(path: String): Try[PairOfFiles] = Try {
-      val src = buildFile(path)
+      val src = buildPath(path)
       // Strip out potential prefix protocol
-      val localInputPath = stripProtocolScheme(src.path)
-      val dest = if (File(inputsRoot).isParentOf(localInputPath)) File(localInputPath)
+      val localInputPath = stripProtocolScheme(src)
+      val dest = if (inputsRoot.isParentOf(localInputPath)) localInputPath
       else {
         // Concatenate call directory with absolute input path
-        File(Paths.get(inputsRoot.toString, localInputPath.toString))
+        DefaultPathBuilder.get(inputsRoot.pathAsString, localInputPath.pathAsString)
       }
 
       PairOfFiles(src, dest)
@@ -213,7 +203,7 @@ trait SharedFileSystem extends PathFactory {
                              (wdlFile: WdlFile): WdlFile = {
     val path = wdlFile.value
     val result = toDestPath(path) flatMap {
-      case PairOfFiles(src, dst) => duplicate("localize", src, dst, strategies) map { _ => WdlFile(dst.toString) }
+      case PairOfFiles(src, dst) => duplicate("localize", src, dst, strategies) map { _ => WdlFile(dst.pathAsString) }
     }
     result.get
   }
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala
index db97805cd..01aa20d9c 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala
@@ -1,16 +1,13 @@
 package cromwell.backend.sfs
 
-import java.nio.file.{FileAlreadyExistsException, Path}
+import java.nio.file.FileAlreadyExistsException
 
-import better.files._
 import cromwell.backend._
 import cromwell.backend.async.{ExecutionHandle, FailedNonRetryableExecutionHandle, PendingExecutionHandle}
 import cromwell.backend.io.JobPathsWithDocker
 import cromwell.backend.standard.{StandardAsyncExecutionActor, StandardAsyncJob}
 import cromwell.backend.validation._
-import cromwell.core.path.DefaultPathBuilder
-import cromwell.core.path.FileImplicits._
-import cromwell.core.path.PathFactory._
+import cromwell.core.path.{DefaultPathBuilder, Path}
 import cromwell.core.retry.SimpleExponentialBackoff
 import wdl4s.values.WdlFile
 
@@ -114,7 +111,7 @@ trait SharedFileSystemAsyncJobExecutionActor
     */
   override def mapCommandLineWdlFile(wdlFile: WdlFile): WdlFile = {
     val cleanPath = DefaultPathBuilder.build(wdlFile.valueString).get
-    WdlFile(if (isDockerRun) jobPathsWithDocker.toDockerPath(cleanPath).toString else cleanPath.toString)
+    WdlFile(if (isDockerRun) jobPathsWithDocker.toDockerPath(cleanPath).pathAsString else cleanPath.pathAsString)
   }
 
   override lazy val commandDirectory: Path = {
@@ -122,7 +119,7 @@ trait SharedFileSystemAsyncJobExecutionActor
   }
 
   override def execute(): ExecutionHandle = {
-    File(jobPaths.callExecutionRoot).createPermissionedDirectories()
+    jobPaths.callExecutionRoot.createPermissionedDirectories()
     writeScriptContents()
     val runner = makeProcessRunner()
     val exitValue = runner.run()
@@ -136,7 +133,7 @@ trait SharedFileSystemAsyncJobExecutionActor
   }
 
   def writeScriptContents(): Unit = {
-    File(jobPaths.script).write(commandScriptContents)
+    jobPaths.script.write(commandScriptContents)
     ()
   }
 
@@ -148,14 +145,14 @@ trait SharedFileSystemAsyncJobExecutionActor
     * @return A process runner that will relatively quickly submit the script asynchronously.
     */
   def makeProcessRunner(): ProcessRunner = {
-    val stdout = pathPlusSuffix(jobPaths.stdout, "submit")
-    val stderr = pathPlusSuffix(jobPaths.stderr, "submit")
-    new ProcessRunner(processArgs.argv, stdout.path, stderr.path)
+    val stdout = jobPaths.stdout.plusExt("submit")
+    val stderr = jobPaths.stderr.plusExt("submit")
+    new ProcessRunner(processArgs.argv, stdout, stderr)
   }
 
   override def recover(job: StandardAsyncJob): ExecutionHandle = {
     // To avoid race conditions, check for the rc file after checking if the job is alive.
-    if (isAlive(job) || File(jobPaths.returnCode).exists) {
+    if (isAlive(job) || jobPaths.returnCode.exists) {
       // If we're done, we'll get to the rc during the next poll.
       // Or if we're still running, return pending also.
       jobLogger.info(s"Recovering using job id: ${job.jobId}")
@@ -169,14 +166,14 @@ trait SharedFileSystemAsyncJobExecutionActor
 
   def isAlive(job: StandardAsyncJob): Boolean = {
     val argv = checkAliveArgs(job).argv
-    val stdout = pathPlusSuffix(jobPaths.stdout, "check")
-    val stderr = pathPlusSuffix(jobPaths.stderr, "check")
-    val checkAlive = new ProcessRunner(argv, stdout.path, stderr.path)
+    val stdout = jobPaths.stdout.plusExt("check")
+    val stderr = jobPaths.stderr.plusExt("check")
+    val checkAlive = new ProcessRunner(argv, stdout, stderr)
     checkAlive.run() == 0
   }
 
   override def tryAbort(job: StandardAsyncJob): Unit = {
-    val returnCodeTmp = pathPlusSuffix(jobPaths.returnCode, "kill")
+    val returnCodeTmp = jobPaths.returnCode.plusExt("kill")
     returnCodeTmp.write(s"$SIGTERM\n")
     try {
       returnCodeTmp.moveTo(jobPaths.returnCode)
@@ -186,15 +183,15 @@ trait SharedFileSystemAsyncJobExecutionActor
         returnCodeTmp.delete(true)
     }
     val argv = killArgs(job).argv
-    val stdout = pathPlusSuffix(jobPaths.stdout, "kill")
-    val stderr = pathPlusSuffix(jobPaths.stderr, "kill")
-    val killer = new ProcessRunner(argv, stdout.path, stderr.path)
+    val stdout = jobPaths.stdout.plusExt("kill")
+    val stderr = jobPaths.stderr.plusExt("kill")
+    val killer = new ProcessRunner(argv, stdout, stderr)
     killer.run()
     ()
   }
 
   override def pollStatus(handle: StandardAsyncPendingExecutionHandle): SharedFileSystemRunStatus = {
-    SharedFileSystemRunStatus(File(jobPaths.returnCode).exists)
+    SharedFileSystemRunStatus(jobPaths.returnCode.exists)
   }
 
   override def isTerminal(runStatus: StandardAsyncRunStatus): Boolean = {
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemCacheHitCopyingActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemCacheHitCopyingActor.scala
index 7c6ba612f..3821d02dc 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemCacheHitCopyingActor.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemCacheHitCopyingActor.scala
@@ -1,8 +1,7 @@
 package cromwell.backend.sfs
 
-import java.nio.file.Path
-
 import cromwell.backend.standard.{StandardCacheHitCopyingActor, StandardCacheHitCopyingActorParams}
+import cromwell.core.path.Path
 
 class SharedFileSystemCacheHitCopyingActor(standardParams: StandardCacheHitCopyingActorParams)
   extends StandardCacheHitCopyingActor(standardParams) with SharedFileSystemJobCachingActorHelper {
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemExpressionFunctions.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemExpressionFunctions.scala
index b3513bd97..754c9549b 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemExpressionFunctions.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemExpressionFunctions.scala
@@ -1,18 +1,11 @@
 package cromwell.backend.sfs
 
-import java.nio.file.Path
-
 import cromwell.backend.io._
 import cromwell.backend.standard.{DefaultStandardExpressionFunctionsParams, StandardExpressionFunctions, StandardExpressionFunctionsParams}
 import cromwell.core.CallContext
-import cromwell.core.path.PathBuilder
+import cromwell.core.path.{DefaultPath, Path, PathBuilder}
 
 object SharedFileSystemExpressionFunctions {
-  private val LocalFileSystemScheme = "file"
-
-  def isLocalPath(path: Path): Boolean =
-    path.toUri.getScheme == SharedFileSystemExpressionFunctions.LocalFileSystemScheme
-
   def apply(jobPaths: JobPaths, pathBuilders: List[PathBuilder]): SharedFileSystemExpressionFunctions = {
     new SharedFileSystemExpressionFunctions(pathBuilders, jobPaths.callContext)
   }
@@ -25,9 +18,10 @@ class SharedFileSystemExpressionFunctions(standardParams: StandardExpressionFunc
     this(DefaultStandardExpressionFunctionsParams(pathBuilders, callContext))
   }
 
-  override def postMapping(path: Path): Path =
-    if (!path.isAbsolute && SharedFileSystemExpressionFunctions.isLocalPath(path))
-      callContext.root.resolve(path)
-    else
-      path
+  override def postMapping(path: Path) = {
+    path match {
+      case _: DefaultPath if !path.isAbsolute => callContext.root.resolve(path)
+      case _ => path
+    }
+  }
 }
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemInitializationActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemInitializationActor.scala
index ba0c74fb4..6ca29cafe 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemInitializationActor.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemInitializationActor.scala
@@ -1,12 +1,10 @@
 package cromwell.backend.sfs
 
-import better.files._
 import cats.data.Validated.{Invalid, Valid}
 import cromwell.backend.BackendInitializationData
 import cromwell.backend.io.WorkflowPaths
 import cromwell.backend.standard.{StandardExpressionFunctions, StandardInitializationActor, StandardInitializationActorParams}
 import cromwell.backend.wfs.WorkflowPathBuilder
-import cromwell.core.path.FileImplicits._
 import cromwell.core.path.{DefaultPathBuilderFactory, PathBuilder, PathBuilderFactory}
 import cromwell.filesystems.gcs.{GcsPathBuilderFactory, GoogleConfiguration}
 import lenthall.exception.MessageAggregation
@@ -48,8 +46,8 @@ class SharedFileSystemInitializationActor(standardParams: StandardInitialization
 
   override def beforeAll(): Future[Option[BackendInitializationData]] = {
     Future.fromTry(Try {
-      publishWorkflowRoot(workflowPaths.workflowRoot.toString)
-      File(workflowPaths.workflowRoot).createPermissionedDirectories()
+      publishWorkflowRoot(workflowPaths.workflowRoot.pathAsString)
+      workflowPaths.workflowRoot.createPermissionedDirectories()
       Option(initializationData)
     })
   }
diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategySpec.scala b/supportedBackends/sfs/src/test/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategySpec.scala
index d9a3d765d..588d1a5c4 100644
--- a/supportedBackends/sfs/src/test/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategySpec.scala
+++ b/supportedBackends/sfs/src/test/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategySpec.scala
@@ -3,12 +3,11 @@ package cromwell.backend.impl.sfs.config
 import java.util.UUID
 
 import akka.event.LoggingAdapter
-import better.files._
 import com.typesafe.config.{ConfigFactory, ConfigValueFactory}
 import cromwell.backend.callcaching.FileHashingActor.SingleFileHashRequest
 import cromwell.backend.io.WorkflowPaths
 import cromwell.backend.standard.StandardInitializationData
-import cromwell.core.path.DefaultPathBuilder
+import cromwell.core.path.{DefaultPathBuilder, Path}
 import org.apache.commons.codec.digest.DigestUtils
 import org.scalatest.prop.TableDrivenPropertyChecks
 import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers}
@@ -23,8 +22,8 @@ class ConfigHashingStrategySpec extends FlatSpec with Matchers with TableDrivenP
 
   val steak = "Steak"
   val steakHash = DigestUtils.md5Hex(steak)
-  val file = File.newTemporaryFile()
-  val symLinksDir = File.newTemporaryDirectory("sym-dir")
+  val file = DefaultPathBuilder.createTempFile()
+  val symLinksDir = DefaultPathBuilder.createTempDirectory("sym-dir")
   val pathHash = DigestUtils.md5Hex(file.pathAsString)
   val md5File = file.sibling(s"${file.name}.md5")
   // Not the md5 value of "Steak". This is intentional so we can verify which hash is used depending on the strategy
@@ -37,11 +36,11 @@ class ConfigHashingStrategySpec extends FlatSpec with Matchers with TableDrivenP
 
   private def randomName(): String = UUID.randomUUID().toString
 
-  def mockRequest(withSibling: Boolean, symlink: Boolean = false) = {
+  def mockRequest(withSibling: Boolean, symlink: Boolean) = {
     if (withSibling && md5File.notExists) md5File.write(md5FileHash)
     val request = mock[SingleFileHashRequest]
     val requestFile = if (symlink) {
-      val symLink : File = symLinksDir./(s"symlink-${randomName()}")
+      val symLink: Path = symLinksDir./(s"symlink-${randomName()}")
       symLink.symbolicLinkTo(file)
       symLink
     } else file
@@ -99,7 +98,7 @@ class ConfigHashingStrategySpec extends FlatSpec with Matchers with TableDrivenP
       checkSibling.getHash(mockRequest(withMd5, symlink = false), mock[LoggingAdapter]) shouldBe Success(expected)
 
       val symLinkRequest: SingleFileHashRequest = mockRequest(withMd5, symlink = true)
-      val symlink = File(symLinkRequest.file.valueString)
+      val symlink = DefaultPathBuilder.get(symLinkRequest.file.valueString)
 
       symlink.isSymbolicLink shouldBe true
       DigestUtils.md5Hex(symlink.pathAsString) should not be expected
@@ -141,7 +140,7 @@ class ConfigHashingStrategySpec extends FlatSpec with Matchers with TableDrivenP
       checkSibling.getHash(mockRequest(withMd5, symlink = false), mock[LoggingAdapter]) shouldBe Success(expected)
 
       val symLinkRequest: SingleFileHashRequest = mockRequest(withMd5, symlink = true)
-      val symlink = File(symLinkRequest.file.valueString)
+      val symlink = DefaultPathBuilder.get(symLinkRequest.file.valueString)
 
       symlink.isSymbolicLink shouldBe true
       checkSibling.getHash(symLinkRequest, mock[LoggingAdapter]) shouldBe Success(expected)
diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala
index cfbd2348d..86deb9649 100644
--- a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala
+++ b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala
@@ -1,21 +1,18 @@
 package cromwell.backend.sfs
 
-import java.nio.file.{Files, Paths}
-
 import akka.testkit.TestDuration
-import better.files._
 import com.typesafe.config.ConfigFactory
 import cromwell.backend.BackendJobExecutionActor.{AbortedResponse, JobFailedNonRetryableResponse, JobSucceededResponse}
 import cromwell.backend.BackendLifecycleActor.AbortJobCommand
+import cromwell.backend._
 import cromwell.backend.async.WrongReturnCode
 import cromwell.backend.io.TestWorkflows._
 import cromwell.backend.io.{JobPathsWithDocker, TestWorkflows}
 import cromwell.backend.sfs.TestLocalAsyncJobExecutionActor._
 import cromwell.backend.standard.StandardValidatedRuntimeAttributesBuilder
-import cromwell.backend._
 import cromwell.core.Tags._
 import cromwell.core._
-import cromwell.core.path.FileImplicits._
+import cromwell.core.path.{DefaultPathBuilder, Path}
 import cromwell.services.keyvalue.KeyValueServiceActor.{KvJobKey, KvPair, ScopedKey}
 import lenthall.exception.AggregatedException
 import org.scalatest.concurrent.PatienceConfiguration.Timeout
@@ -65,7 +62,7 @@ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSyst
   def localizationSpec(docker: Boolean): Assertion = {
     def templateConf(localizers: String) = BackendConfigurationDescriptor(
       ConfigFactory.parseString(
-        s"""{
+        s"""|{
             |  root = "local-cromwell-executions"
             |  filesystems {
             |    local {
@@ -75,7 +72,7 @@ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSyst
             |    }
             |  }
             |}
-        """.stripMargin),
+            |""".stripMargin),
       ConfigFactory.parseString("{}")
     )
 
@@ -117,17 +114,17 @@ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSyst
 
       whenReady(backend.execute) { executionResponse =>
         assertResponse(executionResponse, expectedResponse)
-        val localizedJsonInputFile = Paths.get(jobPaths.callInputsRoot.toString, jsonInputFile)
-        val localizedCallInputFile = Paths.get(jobPaths.callInputsRoot.toString, callInputFile)
+        val localizedJsonInputFile = DefaultPathBuilder.get(jobPaths.callInputsRoot.pathAsString, jsonInputFile)
+        val localizedCallInputFile = DefaultPathBuilder.get(jobPaths.callInputsRoot.pathAsString, callInputFile)
 
-        Files.isSymbolicLink(localizedJsonInputFile) shouldBe isSymlink
+        localizedJsonInputFile.isSymbolicLink shouldBe isSymlink
         val realJsonInputFile =
-          if (isSymlink) Files.readSymbolicLink(localizedJsonInputFile) else localizedJsonInputFile
+          if (isSymlink) localizedJsonInputFile.symbolicLink.get else localizedJsonInputFile
         realJsonInputFile.toFile should exist
 
-        Files.isSymbolicLink(localizedCallInputFile) shouldBe isSymlink
+        localizedCallInputFile.isSymbolicLink shouldBe isSymlink
         val realCallInputFile =
-          if (isSymlink) Files.readSymbolicLink(localizedJsonInputFile) else localizedCallInputFile
+          if (isSymlink) localizedCallInputFile.symbolicLink.get else localizedCallInputFile
         realCallInputFile.toFile should exist
       }
     }
@@ -162,14 +159,14 @@ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSyst
     val backend = backendRef.underlyingActor
 
     val jobPaths = new JobPathsWithDocker(jobDescriptor.key, workflowDescriptor, ConfigFactory.empty)
-    File(jobPaths.callExecutionRoot).createPermissionedDirectories()
-    File(jobPaths.stdout).write("Hello stubby ! ")
-    File(jobPaths.stderr).touch()
+    jobPaths.callExecutionRoot.createPermissionedDirectories()
+    jobPaths.stdout.write("Hello stubby ! ")
+    jobPaths.stderr.touch()
 
     val pid =
       if (completed) {
         if (writeReturnCode)
-          File(jobPaths.returnCode).write("0")
+          jobPaths.returnCode.write("0")
         "0"
       } else {
         import sys.process._
@@ -250,8 +247,8 @@ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSyst
     val jobDescriptor: BackendJobDescriptor = jobDescriptorFromSingleCallWorkflow(workflowDescriptor, inputs, WorkflowOptions.empty, runtimeAttributeDefinitions)
     val backend = createBackend(jobDescriptor, emptyBackendConfig)
     val jobPaths = new JobPathsWithDocker(jobDescriptor.key, workflowDescriptor, emptyBackendConfig.backendConfig)
-    val expectedA = WdlFile(jobPaths.callExecutionRoot.resolve("a").toAbsolutePath.toString)
-    val expectedB = WdlFile(jobPaths.callExecutionRoot.resolve("dir").toAbsolutePath.resolve("b").toString)
+    val expectedA = WdlFile(jobPaths.callExecutionRoot.resolve("a").toAbsolutePath.pathAsString)
+    val expectedB = WdlFile(jobPaths.callExecutionRoot.resolve("dir").toAbsolutePath.resolve("b").pathAsString)
     val expectedOutputs = Map(
       "o1" -> JobOutput(expectedA),
       "o2" -> JobOutput(WdlArray(WdlArrayType(WdlFileType), Seq(expectedA, expectedB))),
@@ -270,8 +267,7 @@ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSyst
     testWorkflow(workflow, backend)
   }
 
-  def createCannedFile(prefix: String, contents: String): File = {
-    val suffix = ".out"
-    File.newTemporaryFile(prefix, suffix).write(contents)
+  def createCannedFile(prefix: String, contents: String): Path = {
+    DefaultPathBuilder.createTempFile(prefix, ".out").write(contents)
   }
 }
diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemSpec.scala b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemSpec.scala
index 16ac18836..ec1db2314 100644
--- a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemSpec.scala
+++ b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemSpec.scala
@@ -1,13 +1,9 @@
 package cromwell.backend.sfs
 
-import java.nio.file.Files
-
-import better.files._
 import com.typesafe.config.{Config, ConfigFactory}
 import cromwell.backend.BackendSpec
 import cromwell.core.CromwellFatalExceptionMarker
-import cromwell.core.path.DefaultPathBuilder
-import cromwell.core.path.FileImplicits._
+import cromwell.core.path.{DefaultPathBuilder, Path}
 import org.scalatest.prop.TableDrivenPropertyChecks
 import org.scalatest.{FlatSpec, Matchers}
 import org.specs2.mock.Mockito
@@ -29,8 +25,8 @@ class SharedFileSystemSpec extends FlatSpec with Matchers with Mockito with Tabl
                        fileAlreadyExists: Boolean = false,
                        symlink: Boolean = false,
                        linkNb: Int = 1) = {
-    val callDir = File.newTemporaryDirectory("SharedFileSystem")
-    val orig = if (fileInCallDir) callDir.createChild("inputFile") else File.newTemporaryFile("inputFile")
+    val callDir = DefaultPathBuilder.createTempDirectory("SharedFileSystem")
+    val orig = if (fileInCallDir) callDir.createChild("inputFile") else DefaultPathBuilder.createTempFile("inputFile")
     val dest = if (fileInCallDir) orig else callDir./(orig.pathAsString.drop(1))
     orig.touch()
     if (fileAlreadyExists) {
@@ -44,7 +40,7 @@ class SharedFileSystemSpec extends FlatSpec with Matchers with Mockito with Tabl
       override val sharedFileSystemConfig = config
     }
     val localizedinputs = Map(inputs.head._1 -> WdlFile(dest.pathAsString))
-    val result = sharedFS.localizeInputs(callDir.path, docker = docker)(inputs)
+    val result = sharedFS.localizeInputs(callDir, docker = docker)(inputs)
 
     result.isSuccess shouldBe true
     result.get should contain theSameElementsAs localizedinputs
@@ -82,20 +78,20 @@ class SharedFileSystemSpec extends FlatSpec with Matchers with Mockito with Tabl
   }
   
   it should "throw a fatal exception if localization fails" in {
-    val callDir = File.newTemporaryDirectory("SharedFileSystem")
-    val orig = File("/made/up/origin")
+    val callDir = DefaultPathBuilder.createTempDirectory("SharedFileSystem")
+    val orig = DefaultPathBuilder.get("/made/up/origin")
 
     val inputs = fqnMapToDeclarationMap(Map("input" -> WdlFile(orig.pathAsString)))
     val sharedFS = new SharedFileSystem {
       override val pathBuilders = localPathBuilder
       override val sharedFileSystemConfig = defaultLocalization
     }
-    val result = sharedFS.localizeInputs(callDir.path, docker = false)(inputs)
+    val result = sharedFS.localizeInputs(callDir, docker = false)(inputs)
     result.isFailure shouldBe true
     result.failed.get.isInstanceOf[CromwellFatalExceptionMarker] shouldBe true
   }
 
-  private[this] def countLinks(file: File): Int = Files.getAttribute(file.path, "unix:nlink").asInstanceOf[Int]
+  private[this] def countLinks(file: Path): Int = file.getAttribute("unix:nlink").asInstanceOf[Int]
 
-  private[this] def isSymLink(file: File): Boolean = Files.isSymbolicLink(file.path)
+  private[this] def isSymLink(file: Path): Boolean = file.isSymbolicLink
 }
diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/TestLocalAsyncJobExecutionActor.scala b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/TestLocalAsyncJobExecutionActor.scala
index f57aa093d..3d9f63e54 100644
--- a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/TestLocalAsyncJobExecutionActor.scala
+++ b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/TestLocalAsyncJobExecutionActor.scala
@@ -10,11 +10,11 @@ import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptor}
 class TestLocalAsyncJobExecutionActor(override val standardParams: StandardAsyncExecutionActorParams)
   extends BackgroundAsyncJobExecutionActor {
   override lazy val processArgs: SharedFileSystemCommand = {
-    val script = jobPaths.script.toString
+    val script = jobPaths.script.pathAsString
     if (isDockerRun) {
       val docker = RuntimeAttributesValidation.extract(DockerValidation.instance, validatedRuntimeAttributes)
-      val cwd = jobPaths.callRoot.toString
-      val dockerCwd = jobPathsWithDocker.callDockerRoot.toString
+      val cwd = jobPaths.callRoot.pathAsString
+      val dockerCwd = jobPathsWithDocker.callDockerRoot.pathAsString
       SharedFileSystemCommand("/bin/bash", "-c",
         s"docker run --rm -v $cwd:$dockerCwd -i $docker /bin/bash < $script")
     } else {
diff --git a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkClusterProcess.scala b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkClusterProcess.scala
index f731c990f..0edffad09 100644
--- a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkClusterProcess.scala
+++ b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkClusterProcess.scala
@@ -1,18 +1,17 @@
 package cromwell.backend.impl.spark
 
-import java.nio.file.Path
-
 import akka.actor.ActorSystem
+import com.typesafe.scalalogging.Logger
 import cromwell.backend.impl.spark.SparkClusterProcess.{SparkJobSubmissionResponse, TerminalStatus}
+import cromwell.core.path.Obsolete._
+import cromwell.core.path.Path
+import org.slf4j.LoggerFactory
+import spray.client.pipelining._
 import spray.http.{HttpRequest, HttpResponse, StatusCodes}
 import spray.json.{DefaultJsonProtocol, JsonParser}
-import spray.client.pipelining._
 
-import scala.concurrent.{ExecutionContext, Future, Promise}
-import better.files._
-import com.typesafe.scalalogging.Logger
-import org.slf4j.LoggerFactory
 import scala.concurrent.duration._
+import scala.concurrent.{ExecutionContext, Future, Promise}
 import scala.util.{Failure, Success, Try}
 
 object SparkClusterProcess {
@@ -59,8 +58,8 @@ class SparkClusterProcess(implicit system: ActorSystem) extends SparkProcess
   with SparkClusterRestClient with SparkClusterJobParser with SparkClusterProcessMonitor {
 
   import SparkClusterProcess._
-  import spray.httpx.SprayJsonSupport._
   import SparkClusterJsonProtocol._
+  import spray.httpx.SprayJsonSupport._
 
   implicit lazy val ec: ExecutionContext = system.dispatcher
   lazy val completionPromise = Promise[TerminalStatus]()
diff --git a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkJobExecutionActor.scala b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkJobExecutionActor.scala
index 7deb08d15..80c73baa3 100644
--- a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkJobExecutionActor.scala
+++ b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkJobExecutionActor.scala
@@ -10,6 +10,7 @@ import cromwell.backend.sfs.{SharedFileSystem, SharedFileSystemExpressionFunctio
 import cromwell.backend.wdl.Command
 import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptor, BackendJobExecutionActor}
 import cromwell.core.path.JavaWriterImplicits._
+import cromwell.core.path.Obsolete._
 import cromwell.core.path.{DefaultPathBuilder, TailedWriter, UntailedWriter}
 import lenthall.util.TryUtil
 import wdl4s.parser.MemoryUnit
@@ -29,7 +30,6 @@ class SparkJobExecutionActor(override val jobDescriptor: BackendJobDescriptor,
                              override val configurationDescriptor: BackendConfigurationDescriptor) extends BackendJobExecutionActor with SharedFileSystem {
 
   import SparkJobExecutionActor._
-  import better.files._
 
   override val pathBuilders = DefaultPathBuilders
   private val tag = s"SparkJobExecutionActor-${jobDescriptor.key.tag}:"
diff --git a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkProcess.scala b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkProcess.scala
index 01d221161..74ac26560 100644
--- a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkProcess.scala
+++ b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkProcess.scala
@@ -1,11 +1,8 @@
 package cromwell.backend.impl.spark
 
-import java.nio.file.Path
-
-import better.files._
 import com.typesafe.scalalogging.StrictLogging
-import cromwell.core.path.PathImplicits._
-import cromwell.core.path.{TailedWriter, UntailedWriter}
+import cromwell.core.path.Obsolete._
+import cromwell.core.path.{Path, TailedWriter, UntailedWriter}
 
 import scala.sys.process._
 import scala.util.{Failure, Success, Try}
diff --git a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkClusterProcessSpec.scala b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkClusterProcessSpec.scala
index f4dbbe1d4..d03779f48 100644
--- a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkClusterProcessSpec.scala
+++ b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkClusterProcessSpec.scala
@@ -1,26 +1,23 @@
 package cromwell.backend.impl.spark
 
-import java.nio.file.Path
-
 import akka.testkit.ImplicitSender
-import better.files._
-import Cmds._
-
-import scala.concurrent.{Future, Promise}
+import cromwell.backend.impl.spark.SparkClusterProcess.SparkClusterJsonProtocol._
+import cromwell.backend.impl.spark.SparkClusterProcess._
 import cromwell.core.TestKitSuite
+import cromwell.core.path.Obsolete._
+import cromwell.core.path.Path
 import org.mockito.Matchers._
 import org.mockito.Mockito
 import org.mockito.Mockito._
 import org.scalatest.concurrent.PatienceConfiguration.Timeout
+import org.scalatest.concurrent.ScalaFutures
 import org.scalatest.mockito.MockitoSugar
 import org.scalatest.{BeforeAndAfter, Matchers, WordSpecLike}
+import spray.http._
 import spray.json._
 
 import scala.concurrent.duration._
-import cromwell.backend.impl.spark.SparkClusterProcess.{Failed, _}
-import org.scalatest.concurrent.ScalaFutures
-import spray.http._
-import SparkClusterJsonProtocol._
+import scala.concurrent.{Future, Promise}
 
 class SparkClusterProcessSpec extends TestKitSuite("SparkClusterProcess")
   with WordSpecLike
diff --git a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkJobExecutionActorSpec.scala b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkJobExecutionActorSpec.scala
index 3a7873fc5..0020977f8 100644
--- a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkJobExecutionActorSpec.scala
+++ b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkJobExecutionActorSpec.scala
@@ -1,17 +1,16 @@
 package cromwell.backend.impl.spark
 
 import java.io.Writer
-import java.nio.file.Path
 
 import akka.testkit.{ImplicitSender, TestActorRef}
-import better.files._
 import com.typesafe.config.ConfigFactory
 import cromwell.backend.BackendJobExecutionActor.{JobFailedNonRetryableResponse, JobSucceededResponse}
 import cromwell.backend.impl.spark.SparkClusterProcess._
 import cromwell.backend.io._
 import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptor, BackendSpec}
+import cromwell.core.path.Obsolete._
+import cromwell.core.path.{Path, PathWriter, TailedWriter, UntailedWriter}
 import cromwell.core.{TestKitSuite, WorkflowOptions}
-import cromwell.core.path.{PathWriter, TailedWriter, UntailedWriter}
 import org.mockito.Mockito
 import org.mockito.Mockito._
 import org.scalatest.concurrent.PatienceConfiguration.Timeout

From 78bad29fe490b426ec5448cc8d5fb64ae4671374 Mon Sep 17 00:00:00 2001
From: Chris Llanwarne <cjllanwarne@gmail.com>
Date: Mon, 6 Feb 2017 13:49:57 -0500
Subject: [PATCH 47/84] New wdl4s for array construction

---
 .../cromwell/backend/wdl/WriteFunctions.scala      |  4 +-
 core/src/test/scala/cromwell/util/SampleWdl.scala  | 59 ++--------------------
 .../workflow/lifecycle/execution/OutputStore.scala |  2 +-
 .../test/scala/cromwell/ArrayWorkflowSpec.scala    | 16 +-----
 project/Dependencies.scala                         |  2 +-
 5 files changed, 8 insertions(+), 75 deletions(-)

diff --git a/backend/src/main/scala/cromwell/backend/wdl/WriteFunctions.scala b/backend/src/main/scala/cromwell/backend/wdl/WriteFunctions.scala
index dd467bcf6..b8568f840 100644
--- a/backend/src/main/scala/cromwell/backend/wdl/WriteFunctions.scala
+++ b/backend/src/main/scala/cromwell/backend/wdl/WriteFunctions.scala
@@ -20,7 +20,7 @@ trait WriteFunctions { this: WdlStandardLibraryFunctions =>
   def writeTempFile(path: String,prefix: String,suffix: String,content: String): String = throw new NotImplementedError("This method is not used anywhere and should be removed")
 
   private def writeContent(baseName: String, content: String): Try[WdlFile] = {
-    val tmpFile = _writeDirectory / s"$baseName-${content.md5Sum}.tmp"
+    val tmpFile = _writeDirectory / s"${baseName}_${content.md5Sum}.tmp"
 
     Try {
       if (tmpFile.notExists) tmpFile.write(content)
@@ -41,7 +41,7 @@ trait WriteFunctions { this: WdlStandardLibraryFunctions =>
       singleArgument <- extractSingleArgument(functionName, params)
       downcast <- Try(castOrDefault(singleArgument))
       tsvSerialized <- downcast.tsvSerialize
-      file <- writeContent(wdlClass.getSimpleName.toLowerCase, tsvSerialized)
+      file <- writeContent(functionName, tsvSerialized)
     } yield file
   }
 
diff --git a/core/src/test/scala/cromwell/util/SampleWdl.scala b/core/src/test/scala/cromwell/util/SampleWdl.scala
index 0a685fdac..5a3940b81 100644
--- a/core/src/test/scala/cromwell/util/SampleWdl.scala
+++ b/core/src/test/scala/cromwell/util/SampleWdl.scala
@@ -4,6 +4,7 @@ import java.util.UUID
 
 import cromwell.core.WorkflowSourceFilesWithoutImports
 import cromwell.core.path.{DefaultPathBuilder, Path}
+import cromwell.core.WorkflowSourceFilesWithoutImports
 import spray.json._
 import wdl4s._
 import wdl4s.types.{WdlArrayType, WdlStringType}
@@ -520,41 +521,7 @@ object SampleWdl {
 
   object ArrayIO extends SampleWdl {
     override def wdlSource(runtime: String = "") =
-      s"""task concat_files {
-        |  String? flags
-        |  Array[File]+ files
-        |  command {
-        |    cat $${default = "-s" flags} $${sep = " " files}
-        |  }
-        |  output {
-        |    File concatenated = stdout()
-        |  }
-        |  RUNTIME
-        |}
-        |
-        |task find {
-        |  String pattern
-        |  File root
-        |  command {
-        |    find $${root} $${"-name " + pattern}
-        |  }
-        |  output {
-        |    Array[String] results = read_lines(stdout())
-        |  }
-        |  RUNTIME
-        |}
-        |
-        |task count_lines {
-        |  Array[File]+ files
-        |  command {
-        |    cat $${sep = ' ' files} | wc -l
-        |  }
-        |  output {
-        |    Int count = read_int(stdout())
-        |  }
-        |  RUNTIME
-        |}
-        |
+      s"""
         |task serialize {
         |  Array[String] strs
         |  command {
@@ -567,33 +534,13 @@ object SampleWdl {
         |}
         |
         |workflow wf {
-        |  Array[File] files
         |  Array[String] strings = ["str1", "str2", "str3"]
         |  call serialize {
         |    input: strs = strings
         |  }
-        |  call concat_files as concat {
-        |    input: files = files
-        |  }
-        |  call count_lines {
-        |    input: files = [concat.concatenated]
-        |  }
-        |  call find
-        |  call count_lines as count_lines_array {
-        |    input: files = find.results
-        |  }
         |}
       """.stripMargin.replaceAll("RUNTIME", runtime)
-
-    val tempDir = DefaultPathBuilder.createTempDirectory("ArrayIO")
-    val firstFile = createCannedFile(prefix = "first", contents = "foo\n", dir = Option(tempDir))
-    val secondFile = createCannedFile(prefix = "second", contents = "bar\nbaz\n", dir = Option(tempDir))
-
-    override val rawInputs = Map(
-      "wf.find.root" -> tempDir,
-      "wf.find.pattern" -> "*.out", // createCannedFile makes files that have .out extension
-      "wf.files" -> Seq(firstFile.pathAsString, secondFile.pathAsString)
-    )
+    override val rawInputs: Map[String, Any] = Map.empty
   }
 
   case class ArrayLiteral(catRootDir: Path) extends SampleWdl {
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/OutputStore.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/OutputStore.scala
index 4a40a538f..9223ec1c6 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/OutputStore.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/OutputStore.scala
@@ -70,7 +70,7 @@ case class OutputStore(store: Map[OutputCallKey, List[OutputEntry]]) {
       val wdlValues = shardsOutputs.map(
         _.getOrElse(taskOutput.unqualifiedName, throw new RuntimeException(s"Could not retrieve output ${taskOutput.unqualifiedName}")))
       val arrayType = taskOutput.relativeWdlType(scatter).asInstanceOf[WdlArrayType]
-      val arrayOfValues = new WdlArray(arrayType, wdlValues)
+      val arrayOfValues = WdlArray(arrayType, wdlValues)
       taskOutput.unqualifiedName -> JobOutput(arrayOfValues)
     } toMap
   }
diff --git a/engine/src/test/scala/cromwell/ArrayWorkflowSpec.scala b/engine/src/test/scala/cromwell/ArrayWorkflowSpec.scala
index 5c97e856e..a4ca15bac 100644
--- a/engine/src/test/scala/cromwell/ArrayWorkflowSpec.scala
+++ b/engine/src/test/scala/cromwell/ArrayWorkflowSpec.scala
@@ -5,7 +5,7 @@ import cromwell.core.path.DefaultPathBuilder
 import cromwell.util.SampleWdl
 import wdl4s.expression.NoFunctions
 import wdl4s.types.{WdlArrayType, WdlFileType, WdlStringType}
-import wdl4s.values.{WdlArray, WdlFile, WdlInteger, WdlString}
+import wdl4s.values.{WdlArray, WdlFile, WdlString}
 import wdl4s.{ImportResolver, WdlNamespaceWithWorkflow}
 
 class ArrayWorkflowSpec extends CromwellTestKitSpec {
@@ -13,20 +13,6 @@ class ArrayWorkflowSpec extends CromwellTestKitSpec {
   val ns = WdlNamespaceWithWorkflow.load(SampleWdl.ArrayLiteral(tmpDir).wdlSource(), Seq.empty[ImportResolver])
   val expectedArray = WdlArray(WdlArrayType(WdlFileType), Seq(WdlFile("f1"), WdlFile("f2"), WdlFile("f3")))
 
-  "A task which contains a parameter " should {
-    "accept an array for the value" in {
-      runWdlAndAssertOutputs(
-        sampleWdl = SampleWdl.ArrayIO,
-        eventFilter = EventFilter.info(pattern = "Workflow complete", occurrences = 1),
-        expectedOutputs = Map(
-          "wf.count_lines.count" -> WdlInteger(3),
-          "wf.count_lines_array.count" -> WdlInteger(3),
-          "wf.serialize.contents" -> WdlString("str1\nstr2\nstr3")
-        )
-      )
-    }
-  }
-
   "A static Array[File] declaration" should {
     "be a valid declaration" in {
       val declaration = ns.workflow.declarations.find {_.unqualifiedName == "arr"}.getOrElse {
diff --git a/project/Dependencies.scala b/project/Dependencies.scala
index 0440159ee..b631640c6 100644
--- a/project/Dependencies.scala
+++ b/project/Dependencies.scala
@@ -2,7 +2,7 @@ import sbt._
 
 object Dependencies {
   lazy val lenthallV = "0.21-e1b7822-SNAP"
-  lazy val wdl4sV = "0.10-76bf06b-SNAP"
+  lazy val wdl4sV = "0.10-cc58e9b-SNAP"
   lazy val sprayV = "1.3.3"
   /*
   spray-json is an independent project from the "spray suite"

From feaf5f275ee36be6774f947883f5f5a94092ddab Mon Sep 17 00:00:00 2001
From: Khalid Shakir <kshakir@broadinstitute.org>
Date: Wed, 8 Feb 2017 15:53:35 -0500
Subject: [PATCH 48/84] Added katevoss and alphabetized pullapprove

---
 .pullapprove.yml | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/.pullapprove.yml b/.pullapprove.yml
index 8e5e37ffa..d9d70c225 100644
--- a/.pullapprove.yml
+++ b/.pullapprove.yml
@@ -5,13 +5,14 @@ author_approval: ignored
 reviewers:
     required: 2
     members:
-        - cjllanwarne
         - Horneth
-        - mcovarr
-        - geoffjentry
-        - kshakir
+        - cjllanwarne
         - francares
         - gauravs90
+        - geoffjentry
         - jainh
+        - katevoss
         - kcibul
+        - kshakir
+        - mcovarr
         - ruchim

From f7576264a1c0fe138dc4996139c37db7738cdb84 Mon Sep 17 00:00:00 2001
From: Jeff Gentry <geoffjentry@gmail.com>
Date: Thu, 9 Feb 2017 18:26:28 -0500
Subject: [PATCH 49/84] update logback (#1967)

---
 project/Dependencies.scala | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/project/Dependencies.scala b/project/Dependencies.scala
index b631640c6..36ff9be45 100644
--- a/project/Dependencies.scala
+++ b/project/Dependencies.scala
@@ -46,8 +46,8 @@ object Dependencies {
 
   private val slf4jBindingDependencies = List(
     // http://logback.qos.ch/dependencies.html
-    "ch.qos.logback" % "logback-classic" % "1.1.7",
-    "ch.qos.logback" % "logback-access" % "1.1.7",
+    "ch.qos.logback" % "logback-classic" % "1.2.1",
+    "ch.qos.logback" % "logback-access" % "1.2.1",
     "org.codehaus.janino" % "janino" % "3.0.1"
   )
 

From 9fbde21cce3143353d25cdd2a9508489ec5ea9de Mon Sep 17 00:00:00 2001
From: Chris Llanwarne <cjllanwarne@gmail.com>
Date: Fri, 10 Feb 2017 14:49:53 -0500
Subject: [PATCH 50/84] Swagger fix for wdlDependencies description

---
 engine/src/main/resources/swagger/cromwell.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/engine/src/main/resources/swagger/cromwell.yaml b/engine/src/main/resources/swagger/cromwell.yaml
index f634f81a0..962711bde 100644
--- a/engine/src/main/resources/swagger/cromwell.yaml
+++ b/engine/src/main/resources/swagger/cromwell.yaml
@@ -99,7 +99,7 @@ paths:
           type: file
           in: formData
         - name: wdlDependencies
-          description: Workflow Options JSON
+          description: Zip file containing WDL dependencies
           required: false
           type: file
           in: formData
@@ -150,7 +150,7 @@ paths:
           type: file
           in: formData
         - name: wdlDependencies
-          description: Workflow Options JSON
+          description: Zip file containing WDL dependencies
           required: false
           type: file
           in: formData

From 5a3f90676623095b1271951d542bc56aca2904b2 Mon Sep 17 00:00:00 2001
From: Thib <Horneth@users.noreply.github.com>
Date: Fri, 10 Feb 2017 15:43:17 -0500
Subject: [PATCH 51/84] Add sync command to force flush of disk writes (#1974)

* add sync command to force flush of disk writes

* also wrap a metadata test around eventually
---
 .../standard/StandardAsyncExecutionActor.scala     |  1 +
 .../workflow/SingleWorkflowRunnerActorSpec.scala   | 68 +++++++++++-----------
 2 files changed, 36 insertions(+), 33 deletions(-)

diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala b/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala
index 5c2416009..0b591f40a 100644
--- a/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala
+++ b/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala
@@ -185,6 +185,7 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
         |cd $cwd
         |${globManipulations(globFiles)}
         |)
+        |sync
         |mv $rcTmpPath $rcPath
         |""".stripMargin.replace("INSTANTIATED_COMMAND", instantiatedCommand)
   }
diff --git a/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala
index f3b68539a..fbe1880b1 100644
--- a/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala
+++ b/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala
@@ -118,40 +118,42 @@ class SingleWorkflowRunnerActorWithMetadataSpec extends SingleWorkflowRunnerActo
         outputFile = Option(metadataFile))
         TestKit.shutdownActorSystem(system, TimeoutDuration)
     }
-    val metadataFileContent = metadataFile.contentAsString
-    val metadata = metadataFileContent.parseJson.asJsObject.fields
-    metadata.get("id") shouldNot be(empty)
-    metadata.get("status").toStringValue should be("Succeeded")
-    metadata.get("submission").toOffsetDateTime should be >= testStart
-    val workflowStart = metadata.get("start").toOffsetDateTime
-    workflowStart should be >= metadata.get("submission").toOffsetDateTime
-    val workflowEnd = metadata.get("end").toOffsetDateTime
-    workflowEnd should be >= metadata.get("start").toOffsetDateTime
-    metadata.get("inputs").toFields should have size workflowInputs
-    metadata.get("outputs").toFields should have size workflowOutputs
-    val calls = metadata.get("calls").toFields
-    calls should not be empty
+    eventually {
+      val metadataFileContent = metadataFile.contentAsString
+      val metadata = metadataFileContent.parseJson.asJsObject.fields
+      metadata.get("id") shouldNot be(empty)
+      metadata.get("status").toStringValue should be("Succeeded")
+      metadata.get("submission").toOffsetDateTime should be >= testStart
+      val workflowStart = metadata.get("start").toOffsetDateTime
+      workflowStart should be >= metadata.get("submission").toOffsetDateTime
+      val workflowEnd = metadata.get("end").toOffsetDateTime
+      workflowEnd should be >= metadata.get("start").toOffsetDateTime
+      metadata.get("inputs").toFields should have size workflowInputs
+      metadata.get("outputs").toFields should have size workflowOutputs
+      val calls = metadata.get("calls").toFields
+      calls should not be empty
 
-    forAll(expectedCalls) { (callName, numInputs, numOutputs) =>
-      val callSeq = calls(callName).asInstanceOf[JsArray].elements
-      callSeq should have size 1
-      val call = callSeq.head.asJsObject.fields
-      val inputs = call.get("inputs").toFields
-      inputs should have size numInputs
-      call.get("executionStatus").toStringValue should be("Done")
-      call.get("backend").toStringValue should be("Local")
-      call.get("backendStatus").toStringValue should be("Done")
-      call.get("outputs").toFields should have size numOutputs
-      val callStart = call.get("start").toOffsetDateTime
-      callStart should be >= workflowStart
-      val callEnd = call.get("end").toOffsetDateTime
-      callEnd should be >= callStart
-      callEnd should be <= workflowEnd
-      call.get("jobId") shouldNot be(empty)
-      call("returnCode").asInstanceOf[JsNumber].value should be (0)
-      call.get("stdout") shouldNot be(empty)
-      call.get("stderr") shouldNot be(empty)
-      call("attempt").asInstanceOf[JsNumber].value should be (1)
+      forAll(expectedCalls) { (callName, numInputs, numOutputs) =>
+        val callSeq = calls(callName).asInstanceOf[JsArray].elements
+        callSeq should have size 1
+        val call = callSeq.head.asJsObject.fields
+        val inputs = call.get("inputs").toFields
+        inputs should have size numInputs
+        call.get("executionStatus").toStringValue should be("Done")
+        call.get("backend").toStringValue should be("Local")
+        call.get("backendStatus").toStringValue should be("Done")
+        call.get("outputs").toFields should have size numOutputs
+        val callStart = call.get("start").toOffsetDateTime
+        callStart should be >= workflowStart
+        val callEnd = call.get("end").toOffsetDateTime
+        callEnd should be >= callStart
+        callEnd should be <= workflowEnd
+        call.get("jobId") shouldNot be(empty)
+        call("returnCode").asInstanceOf[JsNumber].value should be(0)
+        call.get("stdout") shouldNot be(empty)
+        call.get("stderr") shouldNot be(empty)
+        call("attempt").asInstanceOf[JsNumber].value should be(1)
+      }
     }
   }
 

From 7252d68e72c759f9bb3c64f6043a9d2678197690 Mon Sep 17 00:00:00 2001
From: Chris Llanwarne <cjllanwarne@gmail.com>
Date: Fri, 10 Feb 2017 14:01:23 -0500
Subject: [PATCH 52/84] Prevent stack overflow and Cromwell crash for bad WDL

---
 .../test/scala/cromwell/backend/BackendSpec.scala  |  2 +-
 .../MaterializeWorkflowDescriptorActor.scala       | 28 +++++++++++++---------
 .../test/scala/cromwell/ArrayWorkflowSpec.scala    |  2 +-
 .../scala/cromwell/DeclarationWorkflowSpec.scala   |  2 +-
 .../src/test/scala/cromwell/MapWorkflowSpec.scala  |  2 +-
 .../scala/cromwell/OptionalParamWorkflowSpec.scala |  2 +-
 project/Dependencies.scala                         |  2 +-
 .../jes/JesAsyncBackendJobExecutionActorSpec.scala | 22 ++++++++---------
 .../impl/sfs/config/ConfigWdlNamespace.scala       | 12 +++++-----
 .../impl/spark/SparkRuntimeAttributesSpec.scala    |  2 +-
 10 files changed, 41 insertions(+), 35 deletions(-)

diff --git a/backend/src/test/scala/cromwell/backend/BackendSpec.scala b/backend/src/test/scala/cromwell/backend/BackendSpec.scala
index 810726822..a80a9496d 100644
--- a/backend/src/test/scala/cromwell/backend/BackendSpec.scala
+++ b/backend/src/test/scala/cromwell/backend/BackendSpec.scala
@@ -29,7 +29,7 @@ trait BackendSpec extends ScalaFutures with Matchers with Mockito {
                               runtime: String = "") = {
     BackendWorkflowDescriptor(
       WorkflowId.randomId(),
-      WdlNamespaceWithWorkflow.load(wdl.replaceAll("RUNTIME", runtime), Seq.empty[ImportResolver]).workflow,
+      WdlNamespaceWithWorkflow.load(wdl.replaceAll("RUNTIME", runtime), Seq.empty[ImportResolver]).get.workflow, // Get ok, this is a test!
       inputs,
       options,
       Labels.empty
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala
index 6d3889a87..270140d06 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala
@@ -351,18 +351,23 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef,
       serviceRegistryActor ! PutMetadataAction(wfImportEvents)
     }
 
-    validateImportsDirectory(w.importsZip) flatMap { importsDir =>
-      writeMetadatae(importsDir)
-      val importsDirFile = better.files.File(importsDir.pathAsString) // For wdl4s better file compatibility
+    def importsAsNamespace(importsDir: Path): ErrorOr[WdlNamespaceWithWorkflow] = {
+        writeMetadatae(importsDir)
+        val importsDirFile = better.files.File(importsDir.pathAsString) // For wdl4s better file compatibility
       val importResolvers: Seq[ImportResolver] = if (importLocalFilesystem) {
-        List(WdlNamespace.directoryResolver(importsDirFile), WdlNamespace.fileResolver)
-      } else {
-        List(WdlNamespace.directoryResolver(importsDirFile))
-      }
-      val results = WdlNamespaceWithWorkflow.load(w.wdlSource, importResolvers)
-      importsDir.delete(swallowIOExceptions = true)
-      results.validNel
+          List(WdlNamespace.directoryResolver(importsDirFile), WdlNamespace.fileResolver)
+        } else {
+          List(WdlNamespace.directoryResolver(importsDirFile))
+        }
+        val results = WdlNamespaceWithWorkflow.load(w.wdlSource, importResolvers)
+        importsDir.delete(swallowIOExceptions = true)
+        results match {
+          case Success(ns) => ns.validNel
+          case Failure(f) => f.getMessage.invalidNel
+        }
     }
+
+    validateImportsDirectory(w.importsZip) flatMap importsAsNamespace
   }
 
   private def validateNamespace(source: WorkflowSourceFilesCollection): ErrorOr[WdlNamespaceWithWorkflow] = {
@@ -375,7 +380,8 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef,
           } else {
             List.empty
           }
-          WdlNamespaceWithWorkflow.load(w.wdlSource, importResolvers).validNel
+          // This .get is ok because we're already in a try/catch.
+          WdlNamespaceWithWorkflow.load(w.wdlSource, importResolvers).get.validNel
       }
     } catch {
       case e: Exception => s"Unable to load namespace from workflow: ${e.getMessage}".invalidNel
diff --git a/engine/src/test/scala/cromwell/ArrayWorkflowSpec.scala b/engine/src/test/scala/cromwell/ArrayWorkflowSpec.scala
index a4ca15bac..0dcf76cee 100644
--- a/engine/src/test/scala/cromwell/ArrayWorkflowSpec.scala
+++ b/engine/src/test/scala/cromwell/ArrayWorkflowSpec.scala
@@ -10,7 +10,7 @@ import wdl4s.{ImportResolver, WdlNamespaceWithWorkflow}
 
 class ArrayWorkflowSpec extends CromwellTestKitSpec {
   val tmpDir = DefaultPathBuilder.createTempDirectory("ArrayWorkflowSpec")
-  val ns = WdlNamespaceWithWorkflow.load(SampleWdl.ArrayLiteral(tmpDir).wdlSource(), Seq.empty[ImportResolver])
+  val ns = WdlNamespaceWithWorkflow.load(SampleWdl.ArrayLiteral(tmpDir).wdlSource(), Seq.empty[ImportResolver]).get
   val expectedArray = WdlArray(WdlArrayType(WdlFileType), Seq(WdlFile("f1"), WdlFile("f2"), WdlFile("f3")))
 
   "A static Array[File] declaration" should {
diff --git a/engine/src/test/scala/cromwell/DeclarationWorkflowSpec.scala b/engine/src/test/scala/cromwell/DeclarationWorkflowSpec.scala
index aeca4777f..ca2b56de7 100644
--- a/engine/src/test/scala/cromwell/DeclarationWorkflowSpec.scala
+++ b/engine/src/test/scala/cromwell/DeclarationWorkflowSpec.scala
@@ -9,7 +9,7 @@ import org.scalatest.{Matchers, WordSpecLike}
 class DeclarationWorkflowSpec extends Matchers with WordSpecLike {
   "A workflow with declarations in it" should {
     "compute inputs properly" in {
-      WdlNamespaceWithWorkflow.load(SampleWdl.DeclarationsWorkflow.wdlSource(runtime=""), Seq.empty[ImportResolver]).workflow.inputs shouldEqual Map(
+      WdlNamespaceWithWorkflow.load(SampleWdl.DeclarationsWorkflow.wdlSource(runtime=""), Seq.empty[ImportResolver]).get.workflow.inputs shouldEqual Map(
         "two_step.cat.file" -> WorkflowInput("two_step.cat.file", WdlFileType),
         "two_step.cgrep.str_decl" -> WorkflowInput("two_step.cgrep.str_decl", WdlStringType),
         "two_step.cgrep.pattern" -> WorkflowInput("two_step.cgrep.pattern", WdlStringType),
diff --git a/engine/src/test/scala/cromwell/MapWorkflowSpec.scala b/engine/src/test/scala/cromwell/MapWorkflowSpec.scala
index d66559a30..04031fef5 100644
--- a/engine/src/test/scala/cromwell/MapWorkflowSpec.scala
+++ b/engine/src/test/scala/cromwell/MapWorkflowSpec.scala
@@ -13,7 +13,7 @@ import scala.util.{Success, Try}
 class MapWorkflowSpec extends CromwellTestKitSpec {
   private val pwd = DefaultPathBuilder.get(".")
   private val sampleWdl = SampleWdl.MapLiteral(pwd)
-  val ns = WdlNamespaceWithWorkflow.load(sampleWdl.wdlSource(), Seq.empty[ImportResolver])
+  val ns = WdlNamespaceWithWorkflow.load(sampleWdl.wdlSource(), Seq.empty[ImportResolver]).get
   val expectedMap = WdlMap(WdlMapType(WdlFileType, WdlStringType), Map(
     WdlFile("f1") -> WdlString("alice"),
     WdlFile("f2") -> WdlString("bob"),
diff --git a/engine/src/test/scala/cromwell/OptionalParamWorkflowSpec.scala b/engine/src/test/scala/cromwell/OptionalParamWorkflowSpec.scala
index 29035c00b..4cbfb9ac5 100644
--- a/engine/src/test/scala/cromwell/OptionalParamWorkflowSpec.scala
+++ b/engine/src/test/scala/cromwell/OptionalParamWorkflowSpec.scala
@@ -22,7 +22,7 @@ class OptionalParamWorkflowSpec extends Matchers with WordSpecLike {
          |  call find
          |}
        """.stripMargin
-      val ns = WdlNamespace.loadUsingSource(wf, None, None)
+      val ns = WdlNamespace.loadUsingSource(wf, None, None).get
       val findTask = ns.findTask("find") getOrElse {
         fail("Expected to find task 'find'")
       }
diff --git a/project/Dependencies.scala b/project/Dependencies.scala
index b631640c6..036d13910 100644
--- a/project/Dependencies.scala
+++ b/project/Dependencies.scala
@@ -2,7 +2,7 @@ import sbt._
 
 object Dependencies {
   lazy val lenthallV = "0.21-e1b7822-SNAP"
-  lazy val wdl4sV = "0.10-cc58e9b-SNAP"
+  lazy val wdl4sV = "0.10-9ad47e0-SNAP"
   lazy val sprayV = "1.3.3"
   /*
   spray-json is an independent project from the "spray suite"
diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala
index 15d5395fa..592d5b432 100644
--- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala
+++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala
@@ -130,7 +130,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
   private def buildPreemptibleJobDescriptor(attempt: Int, preemptible: Int): BackendJobDescriptor = {
     val workflowDescriptor = BackendWorkflowDescriptor(
       WorkflowId.randomId(),
-      WdlNamespaceWithWorkflow.load(YoSup.replace("[PREEMPTIBLE]", s"preemptible: $preemptible"), Seq.empty[ImportResolver]).workflow,
+      WdlNamespaceWithWorkflow.load(YoSup.replace("[PREEMPTIBLE]", s"preemptible: $preemptible"), Seq.empty[ImportResolver]).get.workflow,
       Inputs,
       NoOptions,
       Labels.empty
@@ -297,7 +297,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
 
     val workflowDescriptor = BackendWorkflowDescriptor(
       WorkflowId.randomId(),
-      WdlNamespaceWithWorkflow.load(YoSup.replace("[PREEMPTIBLE]", ""), Seq.empty[ImportResolver]).workflow,
+      WdlNamespaceWithWorkflow.load(YoSup.replace("[PREEMPTIBLE]", ""), Seq.empty[ImportResolver]).get.workflow,
       inputs,
       NoOptions,
       Labels.empty
@@ -357,7 +357,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
 
     val workflowDescriptor = BackendWorkflowDescriptor(
       WorkflowId.randomId(),
-      WdlNamespaceWithWorkflow.load(SampleWdl.CurrentDirectory.asWorkflowSources(DockerAndDiskRuntime).wdlSource, Seq.empty[ImportResolver]).workflow,
+      WdlNamespaceWithWorkflow.load(SampleWdl.CurrentDirectory.asWorkflowSources(DockerAndDiskRuntime).wdlSource, Seq.empty[ImportResolver]).get.workflow,
       inputs,
       NoOptions,
       Labels.empty
@@ -397,7 +397,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
   TestActorRef[TestableJesJobExecutionActor] = {
     val workflowDescriptor = BackendWorkflowDescriptor(
       WorkflowId.randomId(),
-      WdlNamespaceWithWorkflow.load(sampleWdl.asWorkflowSources(DockerAndDiskRuntime).wdlSource, Seq.empty[ImportResolver]).workflow,
+      WdlNamespaceWithWorkflow.load(sampleWdl.asWorkflowSources(DockerAndDiskRuntime).wdlSource, Seq.empty[ImportResolver]).get.workflow,
       inputs,
       NoOptions,
       Labels.empty
@@ -458,7 +458,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
 
     val workflowDescriptor = BackendWorkflowDescriptor(
       WorkflowId.randomId(),
-      WdlNamespaceWithWorkflow.load(SampleWdl.CurrentDirectory.asWorkflowSources(DockerAndDiskRuntime).wdlSource, Seq.empty[ImportResolver]).workflow,
+      WdlNamespaceWithWorkflow.load(SampleWdl.CurrentDirectory.asWorkflowSources(DockerAndDiskRuntime).wdlSource, Seq.empty[ImportResolver]).get.workflow,
       inputs,
       NoOptions,
       Labels.empty
@@ -487,7 +487,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
 
     val workflowDescriptor = BackendWorkflowDescriptor(
       WorkflowId.randomId(),
-      WdlNamespaceWithWorkflow.load(SampleWdl.CurrentDirectory.asWorkflowSources(DockerAndDiskRuntime).wdlSource, Seq.empty[ImportResolver]).workflow,
+      WdlNamespaceWithWorkflow.load(SampleWdl.CurrentDirectory.asWorkflowSources(DockerAndDiskRuntime).wdlSource, Seq.empty[ImportResolver]).get.workflow,
       inputs,
       NoOptions,
       Labels.empty
@@ -532,7 +532,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
 
     val workflowDescriptor = BackendWorkflowDescriptor(
       WorkflowId.randomId(),
-      WdlNamespaceWithWorkflow.load(SampleWdl.EmptyString.asWorkflowSources(DockerAndDiskRuntime).wdlSource, Seq.empty[ImportResolver]).workflow,
+      WdlNamespaceWithWorkflow.load(SampleWdl.EmptyString.asWorkflowSources(DockerAndDiskRuntime).wdlSource, Seq.empty[ImportResolver]).get.workflow,
       Map.empty,
       NoOptions,
       Labels.empty
@@ -565,7 +565,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
   it should "create a JesFileInput for the monitoring script, when specified" in {
     val workflowDescriptor = BackendWorkflowDescriptor(
       WorkflowId.randomId(),
-      WdlNamespaceWithWorkflow.load(SampleWdl.EmptyString.asWorkflowSources(DockerAndDiskRuntime).wdlSource, Seq.empty[ImportResolver]).workflow,
+      WdlNamespaceWithWorkflow.load(SampleWdl.EmptyString.asWorkflowSources(DockerAndDiskRuntime).wdlSource, Seq.empty[ImportResolver]).get.workflow,
       Map.empty,
       WorkflowOptions.fromJsonString("""{"monitoring_script": "gs://path/to/script"}""").get,
       Labels.empty
@@ -587,7 +587,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
   it should "not create a JesFileInput for the monitoring script, when not specified" in {
     val workflowDescriptor = BackendWorkflowDescriptor(
       WorkflowId.randomId(),
-      WdlNamespaceWithWorkflow.load(SampleWdl.EmptyString.asWorkflowSources(DockerAndDiskRuntime).wdlSource, Seq.empty[ImportResolver]).workflow,
+      WdlNamespaceWithWorkflow.load(SampleWdl.EmptyString.asWorkflowSources(DockerAndDiskRuntime).wdlSource, Seq.empty[ImportResolver]).get.workflow,
       Map.empty,
       NoOptions,
       Labels.empty
@@ -609,7 +609,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
     val workflowDescriptor = BackendWorkflowDescriptor(
       WorkflowId(UUID.fromString("e6236763-c518-41d0-9688-432549a8bf7c")),
       WdlNamespaceWithWorkflow.load(
-        SampleWdl.HelloWorld.asWorkflowSources(""" runtime {docker: "ubuntu:latest"} """).wdlSource, Seq.empty[ImportResolver]).workflow,
+        SampleWdl.HelloWorld.asWorkflowSources(""" runtime {docker: "ubuntu:latest"} """).wdlSource, Seq.empty[ImportResolver]).get.workflow,
       Map.empty,
       WorkflowOptions.fromJsonString(""" {"jes_gcs_root": "gs://path/to/gcs_root"} """).get,
       Labels.empty
@@ -641,7 +641,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
     val workflowDescriptor = BackendWorkflowDescriptor(
       WorkflowId(UUID.fromString("e6236763-c518-41d0-9688-432549a8bf7d")),
       WdlNamespaceWithWorkflow.load(
-        new SampleWdl.ScatterWdl().asWorkflowSources(""" runtime {docker: "ubuntu:latest"} """).wdlSource, Seq.empty[ImportResolver]).workflow,
+        new SampleWdl.ScatterWdl().asWorkflowSources(""" runtime {docker: "ubuntu:latest"} """).wdlSource, Seq.empty[ImportResolver]).get.workflow,
       Map.empty,
       WorkflowOptions.fromJsonString(""" {"jes_gcs_root": "gs://path/to/gcs_root"} """).get,
       Labels.empty
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigWdlNamespace.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigWdlNamespace.scala
index 161768da1..565819e48 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigWdlNamespace.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigWdlNamespace.scala
@@ -5,6 +5,8 @@ import cromwell.backend.impl.sfs.config.ConfigConstants._
 import net.ceedubs.ficus.Ficus._
 import wdl4s._
 
+import scala.util.{Failure, Success}
+
 /**
   * Builds a wdl namespace from the config.
   *
@@ -43,11 +45,9 @@ class ConfigWdlNamespace(backendConfig: Config) {
     * The wdl namespace containing the submit, kill, and check alive tasks.
     */
   val wdlNamespace = {
-    try {
-      WdlNamespace.loadUsingSource(wdlSource, None, None)
-    } catch {
-      case exception: Exception =>
-        throw new RuntimeException(s"Error parsing generated wdl:\n$wdlSource".stripMargin, exception)
+    WdlNamespace.loadUsingSource(wdlSource, None, None) match {
+      case Success(ns) => ns
+      case Failure(f) => throw new RuntimeException(s"Error parsing generated wdl:\n$wdlSource".stripMargin, f)
     }
   }
 
@@ -74,7 +74,7 @@ object ConfigWdlNamespace {
 
   private def makeTask(taskName: String, command: String, declarations: String): Task = {
     val wdlSource = makeWdlSource(taskName, command, declarations)
-    val wdlNamespace = WdlNamespace.loadUsingSource(wdlSource, None, None)
+    val wdlNamespace = WdlNamespace.loadUsingSource(wdlSource, None, None).get
     wdlNamespace.findTask(taskName).getOrElse(throw new RuntimeException(s"Couldn't find task $taskName"))
   }
 
diff --git a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkRuntimeAttributesSpec.scala b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkRuntimeAttributesSpec.scala
index 0eb0ddcec..d7c0340d8 100644
--- a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkRuntimeAttributesSpec.scala
+++ b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkRuntimeAttributesSpec.scala
@@ -91,7 +91,7 @@ class SparkRuntimeAttributesSpec extends WordSpecLike with Matchers {
                                       runtime: String) = {
     BackendWorkflowDescriptor(
       WorkflowId.randomId(),
-      WdlNamespaceWithWorkflow.load(wdl.replaceAll("RUNTIME", runtime.format("appMainClass", "com.test.spark")), Seq.empty[ImportResolver]).workflow,
+      WdlNamespaceWithWorkflow.load(wdl.replaceAll("RUNTIME", runtime.format("appMainClass", "com.test.spark")), Seq.empty[ImportResolver]).get.workflow,
       inputs,
       options,
       Labels.empty

From a1d336c7ea107d8215bb5470748081fa55827830 Mon Sep 17 00:00:00 2001
From: Miguel Covarrubias <mcovarr@broadinstitute.org>
Date: Tue, 7 Feb 2017 18:43:42 -0500
Subject: [PATCH 53/84] Encapsulate `ExecutionStore`, optimize a bit.

---
 .../lifecycle/execution/ExecutionStore.scala       | 65 +++++++++++++++-------
 .../execution/WorkflowExecutionActor.scala         | 24 ++------
 .../execution/WorkflowExecutionActorData.scala     | 28 ++--------
 3 files changed, 56 insertions(+), 61 deletions(-)

diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/ExecutionStore.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/ExecutionStore.scala
index e35f0e004..64b7c589a 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/ExecutionStore.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/ExecutionStore.scala
@@ -19,16 +19,51 @@ object ExecutionStore {
       case scatter: Scatter => Option(ScatterKey(scatter))
       case conditional: If => Option(ConditionalKey(conditional, None))
       case declaration: Declaration => Option(DeclarationKey(declaration, None, workflowCoercedInputs))
-      case _ => None // Ifs will need to be added here when supported
+      case _ => None
     }
 
     new ExecutionStore(keys.flatten.map(_ -> NotStarted).toMap)
   }
 }
 
-case class ExecutionStore(store: Map[JobKey, ExecutionStatus]) {
+case class ExecutionStore(private val store: Map[JobKey, ExecutionStatus]) {
 
-  override def toString = store.map { case (j, s) => s"$j -> $s" } mkString(System.lineSeparator())
+  // This is hugely ripe for optimization, if it becomes a bottleneck
+  def isBypassedConditional(jobKey: JobKey, conditional: If): Boolean = {
+    store.exists {
+      case (key: ConditionalKey, status) =>
+        key.scope.fullyQualifiedName.equals(conditional.fullyQualifiedName) &&
+          key.index.equals(jobKey.index) &&
+          status.equals(ExecutionStatus.Bypassed)
+      case _ => false
+    }
+  }
+
+  def hasActiveJob: Boolean = {
+    def upstreamFailed(scope: Scope): Boolean = scope match {
+      case node: GraphNode => node.upstreamAncestry exists hasFailedScope
+    }
+
+    store exists {
+      case (jobKey, jobStatus) => (jobStatus == NotStarted && !upstreamFailed(jobKey.scope)) || jobStatus == QueuedInCromwell || jobStatus == Starting || jobStatus == Running
+    }
+  }
+
+  def jobStatus(jobKey: JobKey): Option[ExecutionStatus] = store.get(jobKey)
+
+  def unstartedJobs: List[BackendJobDescriptorKey] = {
+    store.collect({case (k: BackendJobDescriptorKey, status) if status != ExecutionStatus.NotStarted => k }).toList
+  }
+
+  private def hasFailedScope(s: GraphNode): Boolean = {
+    store.exists({ case (key, status) => status == Failed && key.scope == s })
+  }
+
+  def hasFailedJob: Boolean = {
+    store.values.exists(_ == ExecutionStatus.Failed)
+  }
+
+  override def toString = store.map { case (j, s) => s"$j -> $s" } mkString System.lineSeparator()
 
   def add(values: Map[JobKey, ExecutionStatus]) = this.copy(store = store ++ values)
 
@@ -36,8 +71,6 @@ case class ExecutionStore(store: Map[JobKey, ExecutionStatus]) {
   // assembling the result.
   def runnableScopes = store.toList collect { case entry if isRunnable(entry) => entry._1 }
 
-  def backendJobKeys = store.keys.toList collect { case k: BackendJobDescriptorKey => k }
-
   private def isRunnable(entry: ExecutionStoreEntry) = {
     entry match {
       case (key, ExecutionStatus.NotStarted) => arePrerequisitesDone(key)
@@ -45,18 +78,17 @@ case class ExecutionStore(store: Map[JobKey, ExecutionStatus]) {
     }
   }
 
-  def findShardEntries(key: CollectorKey): List[ExecutionStoreEntry] = store.toList filter {
-    case (k: CallKey, v) => k.scope == key.scope && k.isShard
-    case (k: DeclarationKey, v) => k.scope == key.scope && k.isShard
-    case _ => false
+  def findCompletedShardsForOutput(key: CollectorKey): List[JobKey] = store.toList collect {
+    case (k: CallKey, v) if v.isDoneOrBypassed && k.scope == key.scope && k.isShard => k
+    case (k: DynamicDeclarationKey, v) if v.isDoneOrBypassed && k.scope == key.scope && k.isShard => k
   }
 
   // Just used to decide whether a collector can be run. In case the shard entries haven't been populated into the
   // execution store yet.
   case class TempJobKey(scope: Scope with GraphNode, index: Option[Int]) extends JobKey {
     // If these are ever used, we've done something wrong...
-    override def attempt: Int = ???
-    override def tag: String = ???
+    override def attempt: Int = throw new NotImplementedError("We've done something wrong.")
+    override def tag: String = throw new NotImplementedError("We've done something wrong.")
   }
   def emulateShardEntries(key: CollectorKey): List[JobKey] = {
     (0 until key.scatterWidth).toList flatMap { i => key.scope match {
@@ -66,13 +98,6 @@ case class ExecutionStore(store: Map[JobKey, ExecutionStatus]) {
     }}
   }
 
-
-//    store.toList filter {
-//    case (k: CallKey, v) => k.scope == key.scope && k.isShard
-//    case (k: DeclarationKey, v) => k.scope == key.scope && k.isShard
-//    case _ => false
-//  }
-
   private def arePrerequisitesDone(key: JobKey): Boolean = {
     val upstream = key.scope.upstream collect {
       case n: Call => upstreamEntry(key, n)
@@ -80,7 +105,7 @@ case class ExecutionStore(store: Map[JobKey, ExecutionStatus]) {
       case n: Declaration => upstreamEntry(key, n)
     }
 
-    val downstream: List[JobKey] = key match {
+    val shardEntriesForCollector: List[JobKey] = key match {
       case collector: CollectorKey => emulateShardEntries(collector)
       case _ => Nil
     }
@@ -94,7 +119,7 @@ case class ExecutionStore(store: Map[JobKey, ExecutionStatus]) {
       case (k, s) => k.scope.fullyQualifiedName == e.scope.fullyQualifiedName && k.index == e.index && s.isDoneOrBypassed
     }
 
-    val dependencies = upstream.flatten.map(_._1) ++ downstream
+    val dependencies = upstream.flatten.map(_._1) ++ shardEntriesForCollector
     val dependenciesResolved = dependencies forall { isDone }
 
     /*
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala
index 8a76b20a7..02114ceb4 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala
@@ -175,7 +175,7 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor,
     // Both of these Should Never Happen (tm), assuming the state data is set correctly on EJEA creation.
     // If they do, it's a big programmer error and the workflow execution fails.
     val jobKey = stateData.engineCallExecutionActors.getOrElse(actorRef, throw new RuntimeException("Programmer Error: An EJEA has terminated but was not assigned a jobKey"))
-    val jobStatus = stateData.executionStore.store.getOrElse(jobKey, throw new RuntimeException("Programmer Error: An EJEA representing a jobKey which this workflow is not running has sent up a terminated message."))
+    val jobStatus = stateData.executionStore.jobStatus(jobKey).getOrElse(throw new RuntimeException("Programmer Error: An EJEA representing a jobKey which this workflow is not running has sent up a terminated message."))
 
     if (!jobStatus.isTerminal) {
       val terminationException = getFailureCause(actorRef) match {
@@ -386,19 +386,8 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor,
   }
 
   private def isInBypassedScope(jobKey: JobKey, data: WorkflowExecutionActorData) = {
-    // This is hugely ripe for optimization, if it becomes a bottleneck
-    def isBypassedConditional(conditional: If, executionStore: ExecutionStore): Boolean = {
-      executionStore.store.exists { case (jobKeyUnderExamination, executionStatus) =>
-        if (jobKeyUnderExamination.isInstanceOf[ConditionalKey]) {
-          if (jobKeyUnderExamination.scope.fullyQualifiedName.equals(conditional.fullyQualifiedName) && jobKeyUnderExamination.index.equals(jobKey.index)) {
-            executionStatus.equals(ExecutionStatus.Bypassed)
-          } else false
-        } else false
-      }
-    }
-
     val result = jobKey.scope.ancestry.exists {
-      case i: If => isBypassedConditional(i, data.executionStore)
+      case i: If => data.executionStore.isBypassedConditional(jobKey, i)
       case _ => false
     }
     result
@@ -526,11 +515,10 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor,
   }
 
   private def processRunnableCollector(collector: CollectorKey, data: WorkflowExecutionActorData, isInBypassed: Boolean): Try[WorkflowExecutionDiff] = {
-    def shards(collectorKey: CollectorKey) = data.executionStore.findShardEntries(collectorKey) collect {
-      case (k: CallKey, v) if v.isDoneOrBypassed => k
-      case (k: DynamicDeclarationKey, v) if v.isDoneOrBypassed => k
-    }
-    data.outputStore.generateCollectorOutput(collector, shards(collector)) match {
+
+    val shards = data.executionStore.findCompletedShardsForOutput(collector)
+
+    data.outputStore.generateCollectorOutput(collector, shards) match {
       case Failure(e) => Failure(new RuntimeException(s"Failed to collect output shards for call ${collector.tag}", e))
       case Success(outputs) =>
         val adjustedOutputs: CallOutputs = if (isInBypassed) {
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala
index 8cf1c7abc..9f4c486ea 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala
@@ -8,7 +8,6 @@ import cromwell.engine.workflow.lifecycle.execution.OutputStore.{OutputCallKey,
 import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor.{DeclarationKey, SubWorkflowKey}
 import cromwell.engine.{EngineWorkflowDescriptor, WdlFunctions}
 import wdl4s.values.WdlValue
-import wdl4s.{GraphNode, Scope}
 
 object WorkflowExecutionDiff {
   def empty = WorkflowExecutionDiff(Map.empty)
@@ -78,32 +77,16 @@ case class WorkflowExecutionActorData(workflowDescriptor: EngineWorkflowDescript
     Map(OutputCallKey(jobKey.scope, jobKey.index) -> newOutputEntries.toList)
   }
 
-  /** Checks if the workflow is completed by scanning through the executionStore.
+  /** Checks if the workflow is completed.
     * If complete, this will return Some(finalStatus).  Otherwise, returns None */
   def workflowCompletionStatus: Option[ExecutionStatus] = {
-    // `List`ify the `prerequisiteScopes` to avoid expensive hashing of `Scope`s when assembling the result.
-
-    def upstreamFailed(scope: Scope): List[GraphNode] = scope match {
-      case node: GraphNode => node.upstreamAncestry.toList filter { s =>
-        executionStore.store.exists({ case (key, status) => status == Failed && key.scope == s })
-      }
-    }
-    // activeJobs is the subset of the executionStore that are either running or will run in the future.
-    val activeJobs = executionStore.store.toList filter {
-      case (jobKey, jobStatus) => (jobStatus == NotStarted && upstreamFailed(jobKey.scope).isEmpty) || jobStatus == QueuedInCromwell || jobStatus == Starting || jobStatus == Running
-    }
-
-    activeJobs match {
-      case jobs if jobs.isEmpty && hasFailedJob => Option(Failed)
-      case jobs if jobs.isEmpty && !hasFailedJob => Option(Done)
+    (executionStore.hasActiveJob, executionStore.hasFailedJob) match {
+      case (false, true) => Option(Failed)
+      case (false, false) => Option(Done)
       case _ => None
     }
   }
 
-  def hasFailedJob: Boolean = {
-    executionStore.store.values.exists(_ == ExecutionStatus.Failed)
-  }
-
   def removeEngineJobExecutionActor(actorRef: ActorRef) = {
     this.copy(engineCallExecutionActors = engineCallExecutionActors - actorRef)
   }
@@ -141,8 +124,7 @@ case class WorkflowExecutionActorData(workflowDescriptor: EngineWorkflowDescript
   }
   
   def jobExecutionMap: JobExecutionMap = {
-    val keys = executionStore.store.collect({case (k: BackendJobDescriptorKey, status) if status != ExecutionStatus.NotStarted => k }).toList
-    downstreamExecutionMap updated (workflowDescriptor.backendDescriptor, keys)
+    downstreamExecutionMap updated (workflowDescriptor.backendDescriptor, executionStore.unstartedJobs)
   }
   
   def hasRunningActors = backendJobExecutionActors.nonEmpty || subWorkflowExecutionActors.nonEmpty

From 35d2db02c5e43d506c8af3e2657b11fbad2162aa Mon Sep 17 00:00:00 2001
From: Chris Llanwarne <cjllanwarne@gmail.com>
Date: Mon, 13 Feb 2017 15:49:14 -0500
Subject: [PATCH 54/84] Added the function 'defined' for optionals

---
 CHANGELOG.md               | 1 +
 project/Dependencies.scala | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b2efbc431..39a058e17 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,6 +12,7 @@
   * `prefix: (String, Array[X]) => Array[String]` - generate an array consisting of each element of the input array prefixed
      by a specified `String`.  The input array can have elements of any primitive type, the return array will always have
      type `Array[String]`.
+  * `defined: (Any) => Boolean` - Will return false if the provided value is an optional that is not defined. Returns true in all other cases.
 * The cromwell server TCP binding timeout is now configurable via the config key `webservice.binding-timeout`, defaulted
   to the previous value `5s` (five seconds) via the reference.conf.
 * Cromwell's Config (Shared Filesystem) backend now supports invocation of commands which run in a Docker image as a non-root user.
diff --git a/project/Dependencies.scala b/project/Dependencies.scala
index 3da054794..53e51ff36 100644
--- a/project/Dependencies.scala
+++ b/project/Dependencies.scala
@@ -2,7 +2,7 @@ import sbt._
 
 object Dependencies {
   lazy val lenthallV = "0.21-e1b7822-SNAP"
-  lazy val wdl4sV = "0.10-9ad47e0-SNAP"
+  lazy val wdl4sV = "0.10-e040cf8-SNAP"
   lazy val sprayV = "1.3.3"
   /*
   spray-json is an independent project from the "spray suite"

From 25c6573bb9342a859fee735f4190b6b6d9f2dec4 Mon Sep 17 00:00:00 2001
From: Thib <Horneth@users.noreply.github.com>
Date: Wed, 15 Feb 2017 14:53:37 -0500
Subject: [PATCH 55/84] Allow for relative symlink localization (#1984)

* allow for relative symlink localization

* PR comments
---
 .travis.yml                                         |  2 +-
 .../standard/StandardAsyncExecutionActor.scala      |  8 ++++++--
 .../cromwell/core/path/EvenBetterPathMethods.scala  | 21 +++++++++++++++------
 3 files changed, 22 insertions(+), 9 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index cfd59b202..5b926d383 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -17,7 +17,7 @@ before_cache:
   - find $HOME/.sbt -name "*.lock" -delete
 env:
   global:
-    - CENTAUR_BRANCH=develop
+    - CENTAUR_BRANCH=symlink_localization
   matrix:
     # Setting this variable twice will cause the 'script' section to run twice with the respective env var invoked
     - BUILD_TYPE=sbt
diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala b/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala
index 0b591f40a..fa9ffffd3 100644
--- a/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala
+++ b/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala
@@ -1,5 +1,7 @@
 package cromwell.backend.standard
 
+import java.io.IOException
+
 import akka.actor.{Actor, ActorLogging, ActorRef}
 import akka.event.LoggingReceive
 import cromwell.backend.BackendJobExecutionActor.{AbortedResponse, BackendJobExecutionResponse}
@@ -10,7 +12,7 @@ import cromwell.backend.validation._
 import cromwell.backend.wdl.{Command, OutputEvaluator, WdlFileMapper}
 import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendJobDescriptor, BackendJobLifecycleActor}
 import cromwell.core.path.Path
-import cromwell.core.{CallOutputs, CromwellAggregatedException, ExecutionEvent}
+import cromwell.core.{CallOutputs, CromwellAggregatedException, CromwellFatalExceptionMarker, ExecutionEvent}
 import cromwell.services.keyvalue.KeyValueServiceActor._
 import cromwell.services.metadata.CallMetadataKeys
 import lenthall.util.TryUtil
@@ -100,7 +102,9 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
 
   /** @see [[Command.instantiate]] */
   final lazy val commandLinePreProcessor: EvaluatedTaskInputs => Try[EvaluatedTaskInputs] = {
-    inputs => TryUtil.sequenceMap(inputs mapValues WdlFileMapper.mapWdlFiles(preProcessWdlFile))
+    inputs => TryUtil.sequenceMap(inputs mapValues WdlFileMapper.mapWdlFiles(preProcessWdlFile)) recoverWith {
+      case e => Failure(new IOException(e.getMessage) with CromwellFatalExceptionMarker)
+    }
   }
 
   /**
diff --git a/core/src/main/scala/cromwell/core/path/EvenBetterPathMethods.scala b/core/src/main/scala/cromwell/core/path/EvenBetterPathMethods.scala
index 8ca7ba12c..1fc6bf4df 100644
--- a/core/src/main/scala/cromwell/core/path/EvenBetterPathMethods.scala
+++ b/core/src/main/scala/cromwell/core/path/EvenBetterPathMethods.scala
@@ -1,9 +1,8 @@
 package cromwell.core.path
 
-import java.nio.file.FileAlreadyExistsException
+import java.nio.file.{FileAlreadyExistsException, Files}
 import java.nio.file.attribute.{PosixFilePermission, PosixFilePermissions}
 
-import scala.annotation.tailrec
 import scala.collection.JavaConverters._
 
 /**
@@ -37,10 +36,20 @@ trait EvenBetterPathMethods {
     this
   }
 
-  @tailrec
-  final def followSymbolicLinks: Path = symbolicLink match {
-    case Some(target) => target.followSymbolicLinks
-    case None => this
+  // betterFile.symbolicLink calls Files.readSymbolicLink, but then implicitly converts the java.nio.Path returned to a better.File
+  // which calls toAbsolutePath. Consequently, if the path was relative, the current directory is used to make it absolute.
+  // This is not the desired behaviour to be able to follow relative symbolic links, so bypass better files method and directly use the java one.
+  final def symbolicLinkRelative: Option[Path] = {
+    if (betterFile.isSymbolicLink) {
+      Option(newPath(Files.readSymbolicLink(betterFile.path)))
+    } else None
+  }
+
+  final def followSymbolicLinks: Path = {
+    symbolicLinkRelative match {
+      case Some(target) => parent.resolve(target.followSymbolicLinks)
+      case None => this
+    }
   }
 
   final def createPermissionedDirectories(): this.type = {

From c398490a3ab4399ebfe9be839bd0d701af0234c8 Mon Sep 17 00:00:00 2001
From: Thibault Jeandet <tjeandet@broadinstitute.org>
Date: Wed, 15 Feb 2017 16:27:33 -0500
Subject: [PATCH 56/84] back to centaur develop

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 5b926d383..cfd59b202 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -17,7 +17,7 @@ before_cache:
   - find $HOME/.sbt -name "*.lock" -delete
 env:
   global:
-    - CENTAUR_BRANCH=symlink_localization
+    - CENTAUR_BRANCH=develop
   matrix:
     # Setting this variable twice will cause the 'script' section to run twice with the respective env var invoked
     - BUILD_TYPE=sbt

From ca9bcefaf621b287021adc435cf48f3838aeb1f5 Mon Sep 17 00:00:00 2001
From: Chris Llanwarne <cjllanwarne@gmail.com>
Date: Thu, 16 Feb 2017 09:45:54 -0500
Subject: [PATCH 57/84] No call names needed in CC

---
 .../lifecycle/execution/callcaching/EngineJobHashingActor.scala       | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActor.scala
index 667e5b513..4830951c0 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActor.scala
@@ -66,7 +66,9 @@ case class EngineJobHashingActor(receiver: ActorRef,
 
     import cromwell.core.simpleton.WdlValueSimpleton._
 
-    val inputSimpletons = jobDescriptor.fullyQualifiedInputs.simplify
+    val unqualifiedInputs = jobDescriptor.inputDeclarations map { case (declaration, value) => declaration.unqualifiedName -> value }
+
+    val inputSimpletons = unqualifiedInputs.simplify
     val (fileInputSimpletons, nonFileInputSimpletons) = inputSimpletons partition {
       case WdlValueSimpleton(_, f: WdlFile) => true
       case _ => false

From 869dfdb2b9f2ea9ece66d40e624062cf2bf39560 Mon Sep 17 00:00:00 2001
From: Miguel Covarrubias <mcovarr@broadinstitute.org>
Date: Fri, 17 Feb 2017 13:07:58 -0500
Subject: [PATCH 58/84] Disambiguate actor name by including source cache entry
 ID.

---
 .../workflow/lifecycle/execution/EngineJobExecutionActor.scala | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala
index 796def7b2..49ac5610a 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala
@@ -140,7 +140,7 @@ class EngineJobExecutionActor(replyTo: ActorRef,
     case Event(CachedOutputLookupSucceeded(wdlValueSimpletons, jobDetritus, returnCode, cacheResultId, cacheHitDetails), data: ResponsePendingData) =>
       writeToMetadata(Map(callCachingReadResultMetadataKey -> s"Cache Hit: $cacheHitDetails"))
       log.debug("Cache hit for {}! Fetching cached result {}", jobTag, cacheResultId)
-      makeBackendCopyCacheHit(wdlValueSimpletons, jobDetritus, returnCode, data)
+      makeBackendCopyCacheHit(wdlValueSimpletons, jobDetritus, returnCode, data, cacheResultId)
     case Event(CachedOutputLookupFailed(callCachingEntryId, error), data: ResponsePendingData) =>
       log.warning("Can't make a copy of the cached job outputs for {} due to {}. Running job.", jobTag, error)
       runJob(data)
@@ -328,11 +328,11 @@ class EngineJobExecutionActor(replyTo: ActorRef,
     goto(FetchingCachedOutputsFromDatabase) using data
   }
 
-  private def makeBackendCopyCacheHit(wdlValueSimpletons: Seq[WdlValueSimpleton], jobDetritusFiles: Map[String,String], returnCode: Option[Int], data: ResponsePendingData) = {
+  private def makeBackendCopyCacheHit(wdlValueSimpletons: Seq[WdlValueSimpleton], jobDetritusFiles: Map[String,String], returnCode: Option[Int], data: ResponsePendingData, cacheResultId: CallCachingEntryId) = {
     factory.cacheHitCopyingActorProps match {
       case Some(propsMaker) =>
         val backendCacheHitCopyingActorProps = propsMaker(data.jobDescriptor, initializationData, serviceRegistryActor)
-        val cacheHitCopyActor = context.actorOf(backendCacheHitCopyingActorProps, buildCacheHitCopyingActorName(data.jobDescriptor))
+        val cacheHitCopyActor = context.actorOf(backendCacheHitCopyingActorProps, buildCacheHitCopyingActorName(data.jobDescriptor, cacheResultId))
         cacheHitCopyActor ! CopyOutputsCommand(wdlValueSimpletons, jobDetritusFiles, returnCode)
         replyTo ! JobRunning(data.jobDescriptor.key, data.jobDescriptor.inputDeclarations, None)
         goto(BackendIsCopyingCachedOutputs)
@@ -373,8 +373,8 @@ class EngineJobExecutionActor(replyTo: ActorRef,
     s"$workflowIdForLogging-BackendJobExecutionActor-$jobTag"
   }
 
-  private def buildCacheHitCopyingActorName(jobDescriptor: BackendJobDescriptor) = {
-    s"$workflowIdForLogging-BackendCacheHitCopyingActor-$jobTag"
+  private def buildCacheHitCopyingActorName(jobDescriptor: BackendJobDescriptor, cacheResultId: CallCachingEntryId) = {
+    s"$workflowIdForLogging-BackendCacheHitCopyingActor-$jobTag-${cacheResultId.id}"
   }
 
   protected def createSaveCacheResultsActor(hashes: CallCacheHashes, success: JobSucceededResponse): Unit = {

From 16dec45e01b9832dd2fa0f6fceffb8edb404dd3b Mon Sep 17 00:00:00 2001
From: Chris Llanwarne <cjllanwarne@gmail.com>
Date: Thu, 16 Feb 2017 12:13:06 -0500
Subject: [PATCH 59/84] Bring sanity to the Call Caching metadata

---
 CHANGELOG.md                                       | 27 ++++++++++++++++++----
 .../standard/StandardCachingActorHelper.scala      |  3 ++-
 .../execution/EngineJobExecutionActor.scala        | 18 +++++++++++----
 .../services/metadata/CallMetadataKeys.scala       |  1 +
 src/bin/travis/resources/local_centaur.conf        |  1 +
 5 files changed, 39 insertions(+), 11 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 39a058e17..48ae8b1cb 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,23 +2,40 @@
 
 ## 25
 
+### Breaking Changes
+
+* Metadata keys for call caching are changed. All call caching keys are now in a `callCaching` stanze. `Call cache read result` has moved here and is now `result`. The `allowResultReuse` and `effectiveCallCachingMode` have moved here. The `hit` boolean is a simple indication of whether or not it was a hit, with no additional information. An example using the new format is:
+```
+"callCaching": {
+  "hit": false,
+  "effectiveCallCachingMode": "ReadAndWriteCache",
+  "result": "Cache Miss",
+  "allowResultReuse": true
+}
+```
+
+### Config Changes
+
 * Added a field `insert-batch-size` to the `database` stanza which defines how many values from a batch insert will be processed at a time. This value defaults to 2000. 
-* Cromwell's WDL parser now recognizes empty array literals correctly, e.g. `Array[String] emptyArray = []`.
 * Moved the config value `services.MetadataService.metadata-summary-refresh-interval` to `services.MetadataService.config.metadata-summary-refresh-interval`
-* Cromwell now applies default labels automatically to JES pipeline runs.
 * Added ability to override the default zone(s) used by JES via the config structure by setting `genomics.default-zones` in the JES configuration
+* The cromwell server TCP binding timeout is now configurable via the config key `webservice.binding-timeout`, defaulted
+  to the previous value `5s` (five seconds) via the reference.conf.
+* For MySQL users, a massive scalability improvement via batched DB writing of internal metadata events. Note that one must add `rewriteBatchedStatements=true` to their JDBC URL in their config in order to take advantage of this
+
+### General Changes
+
+* Cromwell's WDL parser now recognizes empty array literals correctly, e.g. `Array[String] emptyArray = []`.
+* Cromwell now applies default labels automatically to JES pipeline runs.
 * Added support for new WDL functions:
   * `length: (Array[X]) => Integer` - report the length of the specified array
   * `prefix: (String, Array[X]) => Array[String]` - generate an array consisting of each element of the input array prefixed
      by a specified `String`.  The input array can have elements of any primitive type, the return array will always have
      type `Array[String]`.
   * `defined: (Any) => Boolean` - Will return false if the provided value is an optional that is not defined. Returns true in all other cases.
-* The cromwell server TCP binding timeout is now configurable via the config key `webservice.binding-timeout`, defaulted
-  to the previous value `5s` (five seconds) via the reference.conf.
 * Cromwell's Config (Shared Filesystem) backend now supports invocation of commands which run in a Docker image as a non-root user.
   The non-root user could either be the default user for a given Docker image (e.g. specified in a Dockerfile via a `USER` directive),
   or the Config backend could pass an optional `"-u username"` as part of the `submit-docker` command.
-* For MySQL users, a massive scalability improvement via batched DB writing of internal metadata events. Note that one must add `rewriteBatchedStatements=true` to their JDBC URL in their config in order to take advantage of this
 * In some cases the SFS backend, used for Local, SGE, etc., coerced `WdlFile` to `WdlString` by using `.toUri`. This
 resulted in strings prepended with `file:///path/to/file`. Now absolute file paths will not contain the uri scheme.
 
diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardCachingActorHelper.scala b/backend/src/main/scala/cromwell/backend/standard/StandardCachingActorHelper.scala
index f2102ab58..52717df8b 100644
--- a/backend/src/main/scala/cromwell/backend/standard/StandardCachingActorHelper.scala
+++ b/backend/src/main/scala/cromwell/backend/standard/StandardCachingActorHelper.scala
@@ -7,6 +7,7 @@ import cromwell.backend.io.{JobPaths, WorkflowPaths}
 import cromwell.backend.validation.{RuntimeAttributesValidation, ValidatedRuntimeAttributes}
 import cromwell.core.logging.JobLogging
 import cromwell.core.path.Path
+import cromwell.services.metadata.{CallMetadataKeys, MetadataKey}
 import wdl4s.TaskCall
 
 import scala.util.Try
@@ -74,7 +75,7 @@ trait StandardCachingActorHelper extends JobCachingActorHelper {
 
     val fileMetadata = jobPaths.metadataPaths
 
-    val otherMetadata = Map("cache:allowResultReuse" -> true)
+    val otherMetadata = Map(CallMetadataKeys.CallCaching + MetadataKey.KeySeparator + "allowResultReuse" -> true)
 
     runtimeAttributesMetadata ++ fileMetadata ++ otherMetadata ++ nonStandardMetadata
   }
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala
index 796def7b2..044df92b8 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala
@@ -21,6 +21,7 @@ import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor.{JobExecu
 import cromwell.jobstore.JobStoreActor._
 import cromwell.jobstore.{Pending => _, _}
 import cromwell.services.SingletonServicesStore
+import cromwell.services.metadata.{CallMetadataKeys, MetadataKey}
 import wdl4s.TaskOutput
 
 import scala.concurrent.ExecutionContext
@@ -55,7 +56,7 @@ class EngineJobExecutionActor(replyTo: ActorRef,
 
   private[execution] var executionToken: Option[JobExecutionToken] = None
 
-  private val effectiveCallCachingKey = "Effective call caching mode"
+  private val effectiveCallCachingKey = CacheMetadataKeyPrefix + "effectiveCallCachingMode"
 
   implicit val ec: ExecutionContext = context.dispatcher
 
@@ -122,10 +123,13 @@ class EngineJobExecutionActor(replyTo: ActorRef,
       respondAndStop(JobFailedNonRetryableResponse(jobKey, throwable, None))
   }
 
-  private val callCachingReadResultMetadataKey = "Call caching read result"
+  private val callCachingReadResultMetadataKey = CacheMetadataKeyPrefix + "result"
+  private val callCachingHitResultMetadataKey = CacheMetadataKeyPrefix + "hit"
   when(CheckingCallCache) {
     case Event(CacheMiss, data: ResponsePendingData) =>
-      writeToMetadata(Map(callCachingReadResultMetadataKey -> "Cache Miss"))
+      writeToMetadata(Map(
+        callCachingHitResultMetadataKey -> false,
+        callCachingReadResultMetadataKey -> "Cache Miss"))
       log.debug("Cache miss for job {}", jobTag)
       runJob(data)
     case Event(hit: CacheHit, data: ResponsePendingData) =>
@@ -138,7 +142,9 @@ class EngineJobExecutionActor(replyTo: ActorRef,
 
   when(FetchingCachedOutputsFromDatabase) {
     case Event(CachedOutputLookupSucceeded(wdlValueSimpletons, jobDetritus, returnCode, cacheResultId, cacheHitDetails), data: ResponsePendingData) =>
-      writeToMetadata(Map(callCachingReadResultMetadataKey -> s"Cache Hit: $cacheHitDetails"))
+      writeToMetadata(Map(
+        callCachingHitResultMetadataKey -> true,
+        callCachingReadResultMetadataKey -> s"Cache Hit: $cacheHitDetails"))
       log.debug("Cache hit for {}! Fetching cached result {}", jobTag, cacheResultId)
       makeBackendCopyCacheHit(wdlValueSimpletons, jobDetritus, returnCode, data)
     case Event(CachedOutputLookupFailed(callCachingEntryId, error), data: ResponsePendingData) =>
@@ -434,7 +440,7 @@ class EngineJobExecutionActor(replyTo: ActorRef,
     jobStoreActor ! RegisterJobCompleted(jobStoreKey, jobStoreResult)
   }
 
-  private def writeToMetadata(keyValues: Map[String, String]) = {
+  private def writeToMetadata(keyValues: Map[String, Any]) = {
     import cromwell.services.metadata.MetadataService.implicits.MetadataAutoPutter
     serviceRegistryActor.putMetadata(workflowIdForLogging, Option(jobDescriptorKey), keyValues)
   }
@@ -464,6 +470,8 @@ object EngineJobExecutionActor {
   sealed trait EngineJobExecutionActorCommand
   case object Execute extends EngineJobExecutionActorCommand
 
+  val CacheMetadataKeyPrefix = CallMetadataKeys.CallCaching + MetadataKey.KeySeparator
+
   def props(replyTo: ActorRef,
             jobDescriptorKey: BackendJobDescriptorKey,
             executionData: WorkflowExecutionActorData,
diff --git a/services/src/main/scala/cromwell/services/metadata/CallMetadataKeys.scala b/services/src/main/scala/cromwell/services/metadata/CallMetadataKeys.scala
index b22bca0b8..ef0dc1091 100644
--- a/services/src/main/scala/cromwell/services/metadata/CallMetadataKeys.scala
+++ b/services/src/main/scala/cromwell/services/metadata/CallMetadataKeys.scala
@@ -20,4 +20,5 @@ object CallMetadataKeys {
   val CallRoot = "callRoot"
   val SubWorkflowId = "subWorkflowId"
   val SubWorkflowMetadata = "subWorkflowMetadata"
+  val CallCaching = "callCaching"
 }
diff --git a/src/bin/travis/resources/local_centaur.conf b/src/bin/travis/resources/local_centaur.conf
index ef6df3975..848780e82 100644
--- a/src/bin/travis/resources/local_centaur.conf
+++ b/src/bin/travis/resources/local_centaur.conf
@@ -17,5 +17,6 @@ call-caching {
   enabled = true
 }
 
+backend.providers.Local.config.filesystems.local.caching.duplication-strategy = ["copy"]
 backend.providers.Local.config.filesystems.local.localization = ["soft-link", "copy"]
 backend.providers.Local.config.concurrent-job-limit = 20

From bda6afe05bd295acafb88af021139548693f70b9 Mon Sep 17 00:00:00 2001
From: Thib <Horneth@users.noreply.github.com>
Date: Fri, 17 Feb 2017 16:38:00 -0500
Subject: [PATCH 60/84] don't return invalidated cache entries (#2004)

---
 .../database/slick/tables/CallCachingHashEntryComponent.scala         | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingHashEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingHashEntryComponent.scala
index 049f2a4f1..71140ce5f 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingHashEntryComponent.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingHashEntryComponent.scala
@@ -65,7 +65,7 @@ trait CallCachingHashEntryComponent {
   }
 
   /**
-    * Returns the callCachingEntryIds where all the hash keys and hash values exist.
+    * Returns the callCachingEntryIds where all the hash keys and hash values exist, and are allowed to be reused.
     *
     * @param hashKeyHashValues The hash keys and hash values as tuples.
     * @return The callCachingEntryIds with all of the hash keys and hash values.
@@ -74,7 +74,7 @@ trait CallCachingHashEntryComponent {
     for {
       callCachingEntry <- callCachingEntries
       if existsAllCallCachingEntryIdHashKeyHashValues(
-        callCachingEntry.callCachingEntryId, hashKeyHashValues)
+        callCachingEntry.callCachingEntryId, hashKeyHashValues) && callCachingEntry.allowResultReuse
     } yield callCachingEntry.callCachingEntryId
   }
 }

From d7adb191e8a38b84ab637e261a2e473de7163222 Mon Sep 17 00:00:00 2001
From: adamstruck <strucka@ohsu.edu>
Date: Mon, 13 Feb 2017 10:25:32 -0800
Subject: [PATCH 61/84] tes backend prototype with centuar setup

added -elocaldockertest to centaur command

print TES logs after run

fixed unmarshalling bug

inputs in read-only volume; matched TES case classes to schema

address final PR comments
---
 .travis.yml                                        |   1 +
 README.md                                          | 442 ++++++++++++---------
 build.sbt                                          |   8 +
 core/src/main/resources/reference.conf             |  13 +-
 project/Dependencies.scala                         |   4 +
 project/Settings.scala                             |   5 +
 src/bin/travis/resources/tes.conf                  |  13 +
 src/bin/travis/resources/tes_centaur.conf          |  32 ++
 src/bin/travis/test.sh                             |   2 +
 src/bin/travis/testCentaurTes.sh                   |  55 +++
 .../tes/TesAsyncBackendJobExecutionActor.scala     | 206 ++++++++++
 .../backend/impl/tes/TesBackendFileHashing.scala   |  21 +
 .../impl/tes/TesBackendInitializationData.scala    |  11 +
 .../impl/tes/TesBackendLifecycleActorFactory.scala |  34 ++
 .../backend/impl/tes/TesConfiguration.scala        |   7 +
 .../backend/impl/tes/TesInitializationActor.scala  |  40 ++
 .../impl/tes/TesJobCachingActorHelper.scala        |  22 +
 .../cromwell/backend/impl/tes/TesJobPaths.scala    |  58 +++
 .../impl/tes/TesResponseJsonFormatter.scala        |  25 ++
 .../backend/impl/tes/TesRuntimeAttributes.scala    | 133 +++++++
 .../scala/cromwell/backend/impl/tes/TesTask.scala  | 235 +++++++++++
 .../backend/impl/tes/TesWorkflowPaths.scala        |  24 ++
 .../impl/tes/TesInitializationActorSpec.scala      | 101 +++++
 .../backend/impl/tes/TesJobPathsSpec.scala         |  73 ++++
 .../impl/tes/TesRuntimeAttributesSpec.scala        | 178 +++++++++
 .../backend/impl/tes/TesWorkflowPathsSpec.scala    |  75 ++++
 .../cromwell/backend/impl/tes/TestWorkflows.scala  | 141 +++++++
 27 files changed, 1769 insertions(+), 190 deletions(-)
 create mode 100644 src/bin/travis/resources/tes.conf
 create mode 100644 src/bin/travis/resources/tes_centaur.conf
 create mode 100755 src/bin/travis/testCentaurTes.sh
 create mode 100644 supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesAsyncBackendJobExecutionActor.scala
 create mode 100644 supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesBackendFileHashing.scala
 create mode 100644 supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesBackendInitializationData.scala
 create mode 100644 supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesBackendLifecycleActorFactory.scala
 create mode 100644 supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesConfiguration.scala
 create mode 100644 supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesInitializationActor.scala
 create mode 100644 supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesJobCachingActorHelper.scala
 create mode 100644 supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesJobPaths.scala
 create mode 100644 supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesResponseJsonFormatter.scala
 create mode 100644 supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesRuntimeAttributes.scala
 create mode 100644 supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesTask.scala
 create mode 100644 supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesWorkflowPaths.scala
 create mode 100644 supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesInitializationActorSpec.scala
 create mode 100644 supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesJobPathsSpec.scala
 create mode 100644 supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesRuntimeAttributesSpec.scala
 create mode 100644 supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesWorkflowPathsSpec.scala
 create mode 100644 supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TestWorkflows.scala

diff --git a/.travis.yml b/.travis.yml
index cfd59b202..27f347cf2 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -24,6 +24,7 @@ env:
     - BUILD_TYPE=checkPublish
     - BUILD_TYPE=centaurJes
     - BUILD_TYPE=centaurLocal
+    - BUILD_TYPE=centaurTes
 script:
   - src/bin/travis/test.sh
 after_success:
diff --git a/README.md b/README.md
index 465d72cb8..05be217ee 100644
--- a/README.md
+++ b/README.md
@@ -33,6 +33,19 @@ A [Workflow Management System](https://en.wikipedia.org/wiki/Workflow_management
     * [Shared Local Filesystem](#shared-local-filesystem)
     * [Google Cloud Storage Filesystem](#google-cloud-storage-filesystem)
   * [Local Backend](#local-backend)
+  * [Google JES Backend](#google-jes-backend)
+    * [Configuring Google Project](#configuring-google-project)
+    * [Configuring Authentication](#configuring-authentication)
+      * [Application Default Credentials](#application-default-credentials)
+      * [Service Account](#service-account)
+      * [Refresh Token](#refresh-token)
+    * [Docker](#docker)
+    * [Monitoring](#monitoring)
+  * [GA4GH TES Backend](#ga4gh-tes-backend)
+    * [Configuring](#configuring)
+    * [Supported File Systems](#supported-file-systems)
+    * [Docker](#docker)
+    * [CPU, Memory and Disk](#cpu-memory-and-disk)
   * [Sun GridEngine Backend](#sun-gridengine-backend)
   * [HtCondor Backend](#htcondor-backend)
     * [Caching configuration](#caching-configuration)
@@ -45,14 +58,6 @@ A [Workflow Management System](https://en.wikipedia.org/wiki/Workflow_management
     * [Spark runtime attributes](#spark-runtime-attributes)
     * [Spark Environment](#spark-environment)
     * [Sample Wdl](#sample-wdl)
-  * [Google JES Backend](#google-jes-backend)
-    * [Configuring Google Project](#configuring-google-project)
-    * [Configuring Authentication](#configuring-authentication)
-      * [Application Default Credentials](#application-default-credentials)
-      * [Service Account](#service-account)
-      * [Refresh Token](#refresh-token)
-    * [Docker](#docker)
-    * [Monitoring](#monitoring)
 * [Runtime Attributes](#runtime-attributes)
   * [Specifying Default Values](#specifying-default-values)
   * [continueOnReturnCode](#continueonreturncode)
@@ -75,7 +80,8 @@ A [Workflow Management System](https://en.wikipedia.org/wiki/Workflow_management
   * [Local Filesystem Options](#local-filesystem-options)
 * [Imports](#imports)
 * [Sub Workflows](#sub-workflows)
-* [Meta blocks](#meta-blocks)
+  * [Execution](#execution)
+  * [Metadata](#metadata)
 * [REST API](#rest-api)
   * [REST API Versions](#rest-api-versions)
   * [POST /api/workflows/:version](#post-apiworkflowsversion)
@@ -507,6 +513,7 @@ Cromwell distribution:
 
 * Local / GridEngine / LSF / etc. - Run jobs as subprocesses or via a dispatcher.  Supports launching in Docker containers. Use `bash`, `qsub`, `bsub`, etc. to run scripts.
 * Google JES - Launch jobs on Google Compute Engine through the Job Execution Service (JES).
+* GA4GH TES - Launch jobs on servers that support the GA4GH Task Execution Schema (TES).
 * HtCondor - Allows to execute jobs using HTCondor.
 * Spark - Adds support for execution of spark jobs.
 
@@ -761,6 +768,244 @@ docker run --rm -v <cwd>:<docker_cwd> -i <docker_image> /bin/bash < <script>
 >
 > See https://docs.docker.com/engine/userguide/dockervolumes/ for more information on volume mounting in Docker.
 
+## Google JES Backend
+
+Google JES (Job Execution Service) is a Docker-as-a-service from Google.
+
+### Configuring Google Project
+
+You'll need the following things to get started:
+
+* A Google Project (Manage/create projects [here](https://console.developers.google.com/project))
+* A Google Cloud Storage bucket (View/create buckets in your project [here](https://console.cloud.google.com/storage/browser))
+
+On your Google project, open up the [API Manager](https://console.developers.google.com/apis/library) and enable the following APIs:
+
+* Google Compute Engine
+* Google Cloud Storage
+* Genomics API
+
+If your project is `my-project` your bucket is `gs://my-bucket/`, then update your [Cromwell configuration file](#configuring-cromwell) as follows:
+
+```hocon
+backend {
+  default = "JES"
+  providers {
+    JES {
+      actor-factory = "cromwell.backend.impl.jes.JesBackendLifecycleActorFactory"
+      config {
+        project = "my-project"
+        root = "gs://my-bucket"
+        genomics-api-queries-per-100-seconds = 1000
+        .
+        .
+        .
+      }
+    }
+  ]
+}
+```
+
+If your project has API quotas other than the defaults set the `genomics-api-queries-per-100-seconds` value to be the lesser of the `Queries per 100 seconds per user` and `Queries per 100 seconds` quotas. This value will be used to help tune Cromwell's rate of interaction with JES.
+
+### Configuring Authentication
+
+The `google` stanza in the Cromwell configuration file defines how to authenticate to Google.  There are four different
+authentication schemes that might be used:
+
+* `application_default` - (default, recommended) Use [application default](https://developers.google.com/identity/protocols/application-default-credentials) credentials.
+* `service_account` - Use a specific service account and key file (in PEM format) to authenticate.
+* `user_account` - Authenticate as a user.
+* `refresh_token` - Authenticate using a refresh token supplied in the workflow options.
+
+The `auths` block in the `google` stanza defines the authorization schemes within a Cromwell deployment:
+
+```hocon
+google {
+  application-name = "cromwell"
+  auths = [
+    {
+      name = "application-default"
+      scheme = "application_default"
+    },
+    {
+      name = "user-via-refresh"
+      scheme = "refresh_token"
+      client-id = "secret_id"
+      client-secret = "secret_secret"
+    },
+    {
+      name = "service-account"
+      scheme = "service_account"
+      service-account-id = "my-service-account"
+      pem-file = "/path/to/file.pem"
+    }
+  ]
+}
+```
+
+These authorization schemes can be referenced by name within other portions of the configuration file.  For example, both
+the `genomics` and `filesystems.gcs` sections within a JES configuration block must reference an auth defined in this block.
+The auth for the `genomics` section governs the interactions with JES itself, while `filesystems.gcs` governs the localization
+of data into and out of GCE VMs.
+
+#### Application Default Credentials
+
+By default, application default credentials will be used.  There is no configuration required for application default
+credentials, only `name` and `scheme` are required.
+
+To authenticate, run the following commands from your command line (requires [gcloud](https://cloud.google.com/sdk/gcloud/)):
+
+```
+$ gcloud auth login
+$ gcloud config set project my-project
+```
+
+#### Service Account
+
+First create a new service account through the [API Credentials](https://console.developers.google.com/apis/credentials) page.  Go to **Create credentials -> Service account key**.  Then in the **Service account** dropdown select **New service account**.  Fill in a name (e.g. `my-account`), and select key type of JSON.
+
+Creating the account will cause the JSON file to be downloaded.  The structure of this file is roughly like this (account name is `my-account`):
+
+```
+{
+  "type": "service_account",
+  "project_id": "my-project",
+  "private_key_id": "OMITTED",
+  "private_key": "-----BEGIN PRIVATE KEY-----\nBASE64 ENCODED KEY WITH \n TO REPRESENT NEWLINES\n-----END PRIVATE KEY-----\n",
+  "client_email": "my-account@my-project.iam.gserviceaccount.com",
+  "client_id": "22377410244549202395",
+  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+  "token_uri": "https://accounts.google.com/o/oauth2/token",
+  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/my-account%40my-project.iam.gserviceaccount.com"
+}
+```
+
+Most importantly, the value of the `client_email` field should go into the `service-account-id` field in the configuration (see below).  The
+`private_key` portion needs to be pulled into its own file (e.g. `my-key.pem`).  The `\n`s in the string need to be converted to newline characters.
+
+While technically not part of Service Account authorization mode, one can also override the default service account that the compute VM is started with via the configuration option `JES.config.genomics.compute-service-account` or through the workflow options parameter `google_compute_service_account`.  It's important that this service account, and the service account specified in `JES.config.genomics.auth` can both read/write the location specified by `JES.config.root`
+
+#### Refresh Token
+
+A **refresh_token** field must be specified in the [workflow options](#workflow-options) when submitting the job.  Omitting this field will cause the workflow to fail.
+
+The refresh token is passed to JES along with the `client-id` and `client-secret` pair specified in the corresponding entry in `auths`.
+
+
+### Docker
+
+It is possible to reference private docker images in DockerHub to be run on JES.
+However, in order for the image to be pulled, the docker credentials with access to this image must be provided in the configuration file.
+
+
+```
+backend {
+  default = "JES"
+  providers {
+    JES {
+      actor-factory = "cromwell.backend.impl.local.LocalBackendLifecycleActorFactory"
+      config {
+        dockerhub {
+          account = "mydockeraccount@mail.com"
+          token = "mydockertoken"
+        }
+      }
+    }
+  }
+}
+```
+
+It is now possible to reference an image only this account has access to:
+
+```
+task mytask {
+  command {
+    ...
+  }
+  runtime {
+    docker: "private_repo/image"
+    memory: "8 GB"
+    cpu: "1"
+  }
+  ...
+}
+```
+
+Note that if the docker image to be used is public there is no need to add this configuration.
+
+### Monitoring
+
+In order to monitor metrics (CPU, Memory, Disk usage...) about the VM during Call Runtime, a workflow option can be used to specify the path to a script that will run in the background and write its output to a log file.
+
+```
+{
+  "monitoring_script": "gs://cromwell/monitoring/script.sh"
+}
+```
+
+The output of this script will be written to a `monitoring.log` file that will be available in the call gcs bucket when the call completes.  This feature is meant to run a script in the background during long-running processes.  It's possible that if the task is very short that the log file does not flush before de-localization happens and you will end up with a zero byte file.
+
+## GA4GH TES Backend
+The TES backend submits jobs to a server that complies with the protocol described by the [GA4GH schema](https://github.com/ga4gh/task-execution-schemas).
+
+This backend creates three files in the `<call_dir>`:
+
+* `script` - A shell script of the job to be run.  This contains the user's command from the `command` section of the WDL code.
+* `stdout` - The standard output of the process
+* `stderr` - The standard error of the process
+
+The `script` file contains:
+
+```
+#!/bin/sh
+cd <container_call_root>
+<user_command>
+echo $? > rc
+```
+
+`<container_call_root>` would be equal to the runtime attribute `dockerWorkingDir`  or `/cromwell-executions/<workflow_uuid>/call-<call_name>/execution` if this attribute is not supplied.
+
+### Configuring
+Configuring the TES backend is straightforward; one must only provide the TES API endpoint for the service. 
+
+```hocon
+backend {
+  default = "TES"
+  providers {
+    TES {
+      actor-factory = "cromwell.backend.impl.tes.TesBackendLifecycleActorFactory"
+      config {
+        endpoint = "https://<some-url>/v1/jobs"
+        root = "cromwell-executions"
+        dockerRoot = "/cromwell-executions"
+        concurrent-job-limit = 1000
+      }
+    }
+  }
+}
+```
+
+### Supported File Systems
+Currently this backend only works with files on a Local or Shared File System. 
+
+### Docker
+This backend supports the following optional runtime attributes / workflow options for working with Docker:
+* docker: Docker image to use such as "Ubuntu".
+* dockerWorkingDir: defines the working directory in the container.
+
+Outputs:
+It will use `dockerOutputDir` runtime attribute / workflow option to resolve the folder in which the execution results will placed. If there is no `dockerWorkingDir` defined it will use `/cromwell-executions/<workflow_uuid>/call-<call_name>/execution`.
+
+### CPU, Memory and Disk
+This backend supports CPU, memory and disk size configuration through the use of the following runtime attributes / workflow options:
+* cpu: defines the amount of CPU to use. Default value: 1. Type: Integer. Ex: 4.
+* memory: defines the amount of memory to use. Default value: "2 GB". Type: String. Ex: "4 GB" or "4096 MB"
+* disk: defines the amount of disk to use. Default value: "2 GB". Type: String. Ex: "1 GB" or "1024 MB"
+
+It they are not set, the TES backend will use default values.
+
 ## Sun GridEngine Backend
 
 The GridEngine and similar backends use programs such as `qsub` to launch a job and will poll the filesystem to determine if a job is completed.
@@ -1213,185 +1458,6 @@ and its accompanying json input as:
 }
 ```
 
-## Google JES Backend
-
-Google JES (Job Execution Service) is a Docker-as-a-service from Google.
-
-### Configuring Google Project
-
-You'll need the following things to get started:
-
-* A Google Project (Manage/create projects [here](https://console.developers.google.com/project))
-* A Google Cloud Storage bucket (View/create buckets in your project [here](https://console.cloud.google.com/storage/browser))
-
-On your Google project, open up the [API Manager](https://console.developers.google.com/apis/library) and enable the following APIs:
-
-* Google Compute Engine
-* Google Cloud Storage
-* Genomics API
-
-If your project is `my-project` your bucket is `gs://my-bucket/`, then update your [Cromwell configuration file](#configuring-cromwell) as follows:
-
-```hocon
-backend {
-  default = "JES"
-  providers {
-    JES {
-      actor-factory = "cromwell.backend.impl.jes.JesBackendLifecycleActorFactory"
-      config {
-        project = "my-project"
-        root = "gs://my-bucket"
-        genomics-api-queries-per-100-seconds = 1000
-        .
-        .
-        .
-      }
-    }
-  ]
-}
-```
-
-If your project has API quotas other than the defaults set the `genomics-api-queries-per-100-seconds` value to be the lesser of the `Queries per 100 seconds per user` and `Queries per 100 seconds` quotas. This value will be used to help tune Cromwell's rate of interaction with JES.
-
-### Configuring Authentication
-
-The `google` stanza in the Cromwell configuration file defines how to authenticate to Google.  There are four different
-authentication schemes that might be used:
-
-* `application_default` - (default, recommended) Use [application default](https://developers.google.com/identity/protocols/application-default-credentials) credentials.
-* `service_account` - Use a specific service account and key file (in PEM format) to authenticate.
-* `user_account` - Authenticate as a user.
-* `refresh_token` - Authenticate using a refresh token supplied in the workflow options.
-
-The `auths` block in the `google` stanza defines the authorization schemes within a Cromwell deployment:
-
-```hocon
-google {
-  application-name = "cromwell"
-  auths = [
-    {
-      name = "application-default"
-      scheme = "application_default"
-    },
-    {
-      name = "user-via-refresh"
-      scheme = "refresh_token"
-      client-id = "secret_id"
-      client-secret = "secret_secret"
-    },
-    {
-      name = "service-account"
-      scheme = "service_account"
-      service-account-id = "my-service-account"
-      pem-file = "/path/to/file.pem"
-    }
-  ]
-}
-```
-
-These authorization schemes can be referenced by name within other portions of the configuration file.  For example, both
-the `genomics` and `filesystems.gcs` sections within a JES configuration block must reference an auth defined in this block.
-The auth for the `genomics` section governs the interactions with JES itself, while `filesystems.gcs` governs the localization
-of data into and out of GCE VMs.
-
-#### Application Default Credentials
-
-By default, application default credentials will be used.  There is no configuration required for application default
-credentials, only `name` and `scheme` are required.
-
-To authenticate, run the following commands from your command line (requires [gcloud](https://cloud.google.com/sdk/gcloud/)):
-
-```
-$ gcloud auth login
-$ gcloud config set project my-project
-```
-
-#### Service Account
-
-First create a new service account through the [API Credentials](https://console.developers.google.com/apis/credentials) page.  Go to **Create credentials -> Service account key**.  Then in the **Service account** dropdown select **New service account**.  Fill in a name (e.g. `my-account`), and select key type of JSON.
-
-Creating the account will cause the JSON file to be downloaded.  The structure of this file is roughly like this (account name is `my-account`):
-
-```
-{
-  "type": "service_account",
-  "project_id": "my-project",
-  "private_key_id": "OMITTED",
-  "private_key": "-----BEGIN PRIVATE KEY-----\nBASE64 ENCODED KEY WITH \n TO REPRESENT NEWLINES\n-----END PRIVATE KEY-----\n",
-  "client_email": "my-account@my-project.iam.gserviceaccount.com",
-  "client_id": "22377410244549202395",
-  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
-  "token_uri": "https://accounts.google.com/o/oauth2/token",
-  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
-  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/my-account%40my-project.iam.gserviceaccount.com"
-}
-```
-
-Most importantly, the value of the `client_email` field should go into the `service-account-id` field in the configuration (see below).  The
-`private_key` portion needs to be pulled into its own file (e.g. `my-key.pem`).  The `\n`s in the string need to be converted to newline characters.
-
-While technically not part of Service Account authorization mode, one can also override the default service account that the compute VM is started with via the configuration option `JES.config.genomics.compute-service-account` or through the workflow options parameter `google_compute_service_account`.  It's important that this service account, and the service account specified in `JES.config.genomics.auth` can both read/write the location specified by `JES.config.root`
-
-#### Refresh Token
-
-A **refresh_token** field must be specified in the [workflow options](#workflow-options) when submitting the job.  Omitting this field will cause the workflow to fail.
-
-The refresh token is passed to JES along with the `client-id` and `client-secret` pair specified in the corresponding entry in `auths`.
-
-
-### Docker
-
-It is possible to reference private docker images in DockerHub to be run on JES.
-However, in order for the image to be pulled, the docker credentials with access to this image must be provided in the configuration file.
-
-
-```
-backend {
-  default = "JES"
-  providers {
-    JES {
-      actor-factory = "cromwell.backend.impl.local.LocalBackendLifecycleActorFactory"
-      config {
-        dockerhub {
-          account = "mydockeraccount@mail.com"
-          token = "mydockertoken"
-        }
-      }
-    }
-  }
-}
-```
-
-It is now possible to reference an image only this account has access to:
-
-```
-task mytask {
-  command {
-    ...
-  }
-  runtime {
-    docker: "private_repo/image"
-    memory: "8 GB"
-    cpu: "1"
-  }
-  ...
-}
-```
-
-Note that if the docker image to be used is public there is no need to add this configuration.
-
-### Monitoring
-
-In order to monitor metrics (CPU, Memory, Disk usage...) about the VM during Call Runtime, a workflow option can be used to specify the path to a script that will run in the background and write its output to a log file.
-
-```
-{
-  "monitoring_script": "gs://cromwell/monitoring/script.sh"
-}
-```
-
-The output of this script will be written to a `monitoring.log` file that will be available in the call gcs bucket when the call completes.  This feature is meant to run a script in the background during long-running processes.  It's possible that if the task is very short that the log file does not flush before de-localization happens and you will end up with a zero byte file.
-
 # Runtime Attributes
 
 Runtime attributes are used to customize tasks. Within a task one can specify runtime attributes to customize the environment for the call.
diff --git a/build.sbt b/build.sbt
index bfaae9773..a8f7e37f6 100644
--- a/build.sbt
+++ b/build.sbt
@@ -44,6 +44,12 @@ lazy val sfsBackend = (project in backendRoot / "sfs")
   .dependsOn(gcsFileSystem)
   .dependsOn(backend % "test->test")
 
+lazy val tesBackend = (project in backendRoot / "tes")
+  .settings(tesBackendSettings:_*)
+  .withTestSettings
+  .dependsOn(sfsBackend)
+  .dependsOn(backend % "test->test")
+
 lazy val htCondorBackend = (project in backendRoot / "htcondor")
   .settings(htCondorBackendSettings:_*)
   .withTestSettings
@@ -93,10 +99,12 @@ lazy val root = (project in file("."))
   .aggregate(htCondorBackend)
   .aggregate(sparkBackend)
   .aggregate(jesBackend)
+  .aggregate(tesBackend)
   .aggregate(engine)
   // Next level of projects to include in the fat jar (their dependsOn will be transitively included)
   .dependsOn(engine)
   .dependsOn(jesBackend)
+  .dependsOn(tesBackend)
   .dependsOn(htCondorBackend)
   .dependsOn(sparkBackend)
   // Dependencies for tests
diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf
index 2cf93549a..33f995266 100644
--- a/core/src/main/resources/reference.conf
+++ b/core/src/main/resources/reference.conf
@@ -102,7 +102,7 @@ workflow-options {
   #workflow-failure-mode: "ContinueWhilePossible"
 }
 
-// Optional call-caching configuration.
+# Optional call-caching configuration.
 call-caching {
   # Allows re-use of existing results for jobs you've already run
   # (default: false)
@@ -175,7 +175,7 @@ backend {
         # Root directory where Cromwell writes job results.  This directory must be
         # visible and writeable by the Cromwell process as well as the jobs that Cromwell
         # launches.
-        root: "cromwell-executions"
+        root = "cromwell-executions"
 
         filesystems {
           local {
@@ -204,6 +204,15 @@ backend {
       }
     }
 
+    #TES {
+    #  actor-factory = "cromwell.backend.impl.tes.TesBackendLifecycleActorFactory"
+    #  config {
+    #    root = "cromwell-executions"
+    #    dockerRoot = "/cromwell-executions"
+    #    endpoint = "http://127.0.0.1:9000/v1/jobs"
+    #  }
+    #}
+
     #SGE {
     #  actor-factory = "cromwell.backend.impl.sfs.config.ConfigBackendLifecycleActorFactory"
     #  config {
diff --git a/project/Dependencies.scala b/project/Dependencies.scala
index 53e51ff36..96646d876 100644
--- a/project/Dependencies.scala
+++ b/project/Dependencies.scala
@@ -132,6 +132,10 @@ object Dependencies {
     "org.mongodb" %% "casbah" % "3.0.0"
   )
 
+  val tesBackendDependencies = List(
+    "io.spray" %% "spray-client" % sprayV
+  ) ++ sprayServerDependencies
+
   val sparkBackendDependencies = List(
     "io.spray" %% "spray-client" % sprayV
   ) ++ sprayServerDependencies
diff --git a/project/Settings.scala b/project/Settings.scala
index d2032360f..c86689b84 100644
--- a/project/Settings.scala
+++ b/project/Settings.scala
@@ -142,6 +142,11 @@ object Settings {
     name := "cromwell-sfs-backend"
   ) ++ commonSettings
 
+  val tesBackendSettings = List(
+    name := "cromwell-tes-backend",
+    libraryDependencies ++= tesBackendDependencies
+  ) ++ commonSettings
+
   val htCondorBackendSettings = List(
     name := "cromwell-htcondor-backend",
     libraryDependencies ++= htCondorBackendDependencies
diff --git a/src/bin/travis/resources/tes.conf b/src/bin/travis/resources/tes.conf
new file mode 100644
index 000000000..33974e302
--- /dev/null
+++ b/src/bin/travis/resources/tes.conf
@@ -0,0 +1,13 @@
+HttpPort: 9000
+Storage:
+  - Local:
+      AllowedDirs:
+        - /home/
+        - /cromwell-executions
+        - /tmp/
+DBPath: /tmp/tes_task.db
+Schedulers:
+  Local:
+    NumWorkers: 4
+Worker:
+  Timeout: 1
diff --git a/src/bin/travis/resources/tes_centaur.conf b/src/bin/travis/resources/tes_centaur.conf
new file mode 100644
index 000000000..4254c1cb4
--- /dev/null
+++ b/src/bin/travis/resources/tes_centaur.conf
@@ -0,0 +1,32 @@
+akka {
+  loggers = ["akka.event.slf4j.Slf4jLogger"]
+  logging-filter = "akka.event.slf4j.Slf4jLoggingFilter"
+}
+
+spray.can {
+  server {
+    request-timeout = 40s
+  }
+  client {
+    request-timeout = 40s
+    connecting-timeout = 40s
+  }
+}
+
+call-caching {
+  enabled = true
+}
+
+backend {
+  default = "TES"
+  providers {
+    TES {
+      actor-factory = "cromwell.backend.impl.tes.TesBackendLifecycleActorFactory"
+      config {
+        root = "cromwell-executions"
+        dockerRoot = "/cromwell-executions"
+        endpoint = "http://127.0.0.1:9000/v1/jobs"
+      }
+    }
+  }
+}
diff --git a/src/bin/travis/test.sh b/src/bin/travis/test.sh
index 9fd024c69..0daa3001a 100755
--- a/src/bin/travis/test.sh
+++ b/src/bin/travis/test.sh
@@ -7,6 +7,8 @@ SCRIPT_DIR=src/bin/travis
 # BUILD_TYPE is coming in from the Travis build matrix
 if [ "$BUILD_TYPE" = "centaurJes" ]; then
     "${SCRIPT_DIR}"/testCentaurJes.sh
+elif [ "$BUILD_TYPE" = "centaurTes" ]; then
+    "${SCRIPT_DIR}"/testCentaurTes.sh
 elif [ "$BUILD_TYPE" = "centaurLocal" ]; then
     "${SCRIPT_DIR}"/testCentaurLocal.sh
 elif [ "$BUILD_TYPE" = "sbt" ]; then
diff --git a/src/bin/travis/testCentaurTes.sh b/src/bin/travis/testCentaurTes.sh
new file mode 100755
index 000000000..0414059c1
--- /dev/null
+++ b/src/bin/travis/testCentaurTes.sh
@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+
+printTravisHeartbeat() {
+    # Sleep one minute between printouts, but don't zombie for more than two hours
+    for ((i=0; i < 120; i++)); do
+        sleep 60
+        printf "…"
+    done &
+    TRAVIS_HEARTBEAT_PID=$!
+}
+
+killTravisHeartbeat() {
+    if [ -n "${TRAVIS_HEARTBEAT_PID+set}" ]; then
+        kill ${TRAVIS_HEARTBEAT_PID} || true
+    fi
+}
+
+exitScript() {
+    echo "TES LOG"
+    cat logs/tes.log
+    echo "CROMWELL LOG"
+    cat logs/cromwell.log
+    echo "CENTAUR LOG"
+    cat logs/centaur.log
+    killTravisHeartbeat
+}
+
+trap exitScript EXIT
+printTravisHeartbeat
+
+set -x
+set -e
+
+sbt assembly
+CROMWELL_JAR=$(find "$(pwd)/target/scala-2.11" -name "cromwell-*.jar")
+TES_CENTAUR_CONF="$(pwd)/src/bin/travis/resources/tes_centaur.conf"
+git clone https://github.com/broadinstitute/centaur.git
+cd centaur
+git checkout ${CENTAUR_BRANCH}
+cd ..
+
+TES_CONF="$(pwd)/src/bin/travis/resources/tes.conf"
+git clone https://github.com/broadinstitute/funnel.git
+cd funnel
+make
+cd ..
+mkdir logs
+nohup funnel/bin/tes-server -config ${TES_CONF} > logs/tes.log 2>&1 &
+
+
+# All tests use ubuntu:latest - make sure it's there before starting the tests 
+# because pulling the image during some of the tests would cause them to fail 
+# (specifically output_redirection which expects a specific value in stderr)
+docker pull ubuntu:latest
+centaur/test_cromwell.sh -j"${CROMWELL_JAR}" -c${TES_CENTAUR_CONF} -elocaldockertest
diff --git a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesAsyncBackendJobExecutionActor.scala b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesAsyncBackendJobExecutionActor.scala
new file mode 100644
index 000000000..58b614cf9
--- /dev/null
+++ b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesAsyncBackendJobExecutionActor.scala
@@ -0,0 +1,206 @@
+package cromwell.backend.impl.tes
+
+import java.nio.file.FileAlreadyExistsException
+
+import cromwell.backend.BackendJobLifecycleActor
+import cromwell.backend.async.{ExecutionHandle, FailedNonRetryableExecutionHandle, PendingExecutionHandle}
+import cromwell.backend.impl.tes.TesResponseJsonFormatter._
+import cromwell.backend.standard.{StandardAsyncExecutionActor, StandardAsyncExecutionActorParams, StandardAsyncJob}
+import cromwell.core.path.Obsolete._
+import cromwell.core.path.Path
+import cromwell.core.retry.SimpleExponentialBackoff
+import spray.client.pipelining._
+import spray.http.HttpRequest
+import spray.httpx.SprayJsonSupport._
+import spray.httpx.unmarshalling._
+import wdl4s.expression.NoFunctions
+import wdl4s.values.WdlFile
+
+import scala.concurrent.duration._
+import scala.concurrent.{ExecutionContext, Future}
+import scala.language.postfixOps
+import scala.util.{Failure, Success}
+
+sealed trait TesRunStatus {
+  def isTerminal: Boolean
+}
+
+case object Running extends TesRunStatus {
+  def isTerminal = false
+}
+
+case object Complete extends TesRunStatus {
+  def isTerminal = true
+}
+
+case object FailedOrError extends TesRunStatus {
+  def isTerminal = true
+}
+
+object TesAsyncBackendJobExecutionActor {
+  val JobIdKey = "tes_job_id"
+}
+
+class TesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyncExecutionActorParams)
+  extends BackendJobLifecycleActor with StandardAsyncExecutionActor with TesJobCachingActorHelper {
+
+  override type StandardAsyncRunInfo = Any
+
+  override type StandardAsyncRunStatus = TesRunStatus
+
+  override lazy val pollBackOff = SimpleExponentialBackoff(
+    initialInterval = 1 seconds,
+    maxInterval = 5 minutes,
+    multiplier = 1.1
+  )
+
+  override lazy val executeOrRecoverBackOff = SimpleExponentialBackoff(
+    initialInterval = 3 seconds,
+    maxInterval = 30 seconds,
+    multiplier = 1.1
+  )
+
+  override lazy val retryable: Boolean = false
+
+  private val tesEndpoint = workflowDescriptor.workflowOptions.getOrElse("endpoint", tesConfiguration.endpointURL)
+
+  override lazy val jobTag = jobDescriptor.key.tag
+
+  private def pipeline[T: FromResponseUnmarshaller]: HttpRequest => Future[T] = sendReceive ~> unmarshal[T]
+
+  // Utility for converting a WdlValue so that the path is localized to the
+  // container's filesystem.
+  override def mapCommandLineWdlFile(wdlFile: WdlFile): WdlFile = {
+    val localPath = Paths.get(wdlFile.valueString).toAbsolutePath
+    localPath match {
+      case p if p.startsWith(tesJobPaths.DockerRoot) =>
+        val containerPath = p.pathAsString
+        WdlFile(containerPath)
+      case p if p.startsWith(tesJobPaths.callExecutionRoot) =>
+        val containerPath = tesJobPaths.containerExec(commandDirectory, localPath.getFileName.pathAsString)
+        WdlFile(containerPath)
+      case p =>
+        val containerPath = tesJobPaths.containerInput(p.pathAsString)
+        WdlFile(containerPath)
+    }
+  }
+
+  override lazy val commandDirectory: Path = {
+    runtimeAttributes.dockerWorkingDir match {
+      case Some(path) => Paths.get(path)
+      case None => tesJobPaths.callExecutionDockerRoot
+    }
+  }
+
+  def createTaskMessage(): TesTaskMessage = {
+    val task = TesTask(jobDescriptor, configurationDescriptor, jobLogger, tesJobPaths, runtimeAttributes, commandDirectory, backendEngineFunctions)
+
+    tesJobPaths.script.write(commandScriptContents)
+
+    TesTaskMessage(
+      Option(task.name),
+      Option(task.description),
+      Option(task.project),
+      Option(task.inputs),
+      Option(task.outputs),
+      task.resources,
+      task.dockerExecutor
+    )
+  }
+
+  override def executeAsync()(implicit ec: ExecutionContext): Future[ExecutionHandle] = {
+    // create call exec dir
+    File(tesJobPaths.callExecutionRoot).createPermissionedDirectories()
+    val taskMessage = createTaskMessage()
+
+    val submitTask = pipeline[TesPostResponse]
+      .apply(Post(tesEndpoint, taskMessage))
+
+    submitTask.map {
+      response =>
+        val jobID = response.value
+        PendingExecutionHandle(jobDescriptor, StandardAsyncJob(jobID), None, previousStatus = None)
+    }
+  }
+
+  override def tryAbort(job: StandardAsyncJob): Unit = {
+
+    val returnCodeTmp = jobPaths.returnCode.plusExt("kill")
+    returnCodeTmp.write(s"$SIGTERM\n")
+    try {
+      returnCodeTmp.moveTo(jobPaths.returnCode)
+    } catch {
+      case _: FileAlreadyExistsException =>
+        // If the process has already completed, there will be an existing rc file.
+        returnCodeTmp.delete(true)
+    }
+
+    val abortRequest = pipeline[TesGetResponse]
+      .apply(Delete(s"$tesEndpoint/${job.jobId}"))
+    abortRequest onComplete {
+      case Success(_) => jobLogger.info("{} Aborted {}", tag: Any, job.jobId)
+      case Failure(ex) => jobLogger.warn("{} Failed to abort {}: {}", tag, job.jobId, ex.getMessage)
+    }
+    ()
+  }
+
+  override def requestsAbortAndDiesImmediately: Boolean = true
+
+  override def pollStatusAsync(handle: StandardAsyncPendingExecutionHandle)(implicit ec: ExecutionContext): Future[TesRunStatus] = {
+    val pollTask = pipeline[TesGetResponse].apply(Get(s"$tesEndpoint/${handle.pendingJob.jobId}"))
+
+    pollTask.map {
+      response =>
+        val state = response.state
+        state match {
+          case s if s.contains("Complete") =>
+            jobLogger.info(s"Job ${handle.pendingJob.jobId} is complete")
+            Complete
+
+          case s if s.contains("Cancel") =>
+            jobLogger.info(s"Job ${handle.pendingJob.jobId} was canceled")
+            FailedOrError
+
+          case s if s.contains("Error") =>
+            jobLogger.info(s"TES reported an error for Job ${handle.pendingJob.jobId}")
+            FailedOrError
+
+          case _ => Running
+        }
+    }
+  }
+
+  override def customPollStatusFailure: PartialFunction[(ExecutionHandle, Exception), ExecutionHandle] = {
+    case (oldHandle: StandardAsyncPendingExecutionHandle@unchecked, e: Exception) =>
+      jobLogger.error(s"$tag TES Job ${oldHandle.pendingJob.jobId} has not been found, failing call")
+      FailedNonRetryableExecutionHandle(e)
+  }
+
+  override def isTerminal(runStatus: TesRunStatus): Boolean = {
+    runStatus.isTerminal
+  }
+
+  override def isSuccess(runStatus: TesRunStatus): Boolean = {
+    runStatus match {
+      case Complete => true
+      case _ => false
+    }
+  }
+
+  private def hostAbsoluteFilePath(wdlFile: WdlFile): File = {
+    tesJobPaths.callExecutionRoot.resolve(wdlFile.value)
+  }
+  
+  private val outputWdlFiles: Seq[WdlFile] = jobDescriptor.call.task
+    .findOutputFiles(jobDescriptor.fullyQualifiedInputs, NoFunctions)
+    .filter(o => !Paths.get(o.valueString).isAbsolute)
+
+  override def mapOutputWdlFile(wdlFile: WdlFile): WdlFile = {
+    wdlFile match {
+      case fileNotFound if !hostAbsoluteFilePath(fileNotFound).exists && outputWdlFiles.contains(fileNotFound) =>
+        throw new RuntimeException("Could not process output, file not found: " +
+          s"${hostAbsoluteFilePath(wdlFile).pathAsString}")
+      case _ => WdlFile(hostAbsoluteFilePath(wdlFile).pathAsString)
+    }
+  }
+}
diff --git a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesBackendFileHashing.scala b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesBackendFileHashing.scala
new file mode 100644
index 000000000..d6f8d3454
--- /dev/null
+++ b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesBackendFileHashing.scala
@@ -0,0 +1,21 @@
+package cromwell.backend.impl.tes
+
+import akka.event.LoggingAdapter
+import cromwell.backend.callcaching.FileHashingActor.SingleFileHashRequest
+import cromwell.core.path.DefaultPathBuilder
+import cromwell.util.TryWithResource._
+
+import scala.language.postfixOps
+import scala.util.Try
+
+private[tes] object TesBackendFileHashing {
+  def getMd5Result(request: SingleFileHashRequest, log: LoggingAdapter): Try[String] = {
+    val path = DefaultPathBuilder.build(request.file.valueString) recover {
+      case failure => throw new RuntimeException("Failed to construct path to hash", failure)
+    } get
+
+    tryWithResource(() => path.newInputStream) { inputStream =>
+      org.apache.commons.codec.digest.DigestUtils.md5Hex(inputStream)
+    }
+  }
+}
diff --git a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesBackendInitializationData.scala b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesBackendInitializationData.scala
new file mode 100644
index 000000000..132126f4a
--- /dev/null
+++ b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesBackendInitializationData.scala
@@ -0,0 +1,11 @@
+package cromwell.backend.impl.tes
+
+import cromwell.backend.sfs.SharedFileSystemExpressionFunctions
+import cromwell.backend.standard.{StandardInitializationData, StandardValidatedRuntimeAttributesBuilder}
+
+case class TesBackendInitializationData
+(
+  override val workflowPaths: TesWorkflowPaths,
+  override val runtimeAttributesBuilder: StandardValidatedRuntimeAttributesBuilder,
+  tesConfiguration: TesConfiguration
+) extends StandardInitializationData(workflowPaths, runtimeAttributesBuilder, classOf[SharedFileSystemExpressionFunctions])
diff --git a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesBackendLifecycleActorFactory.scala b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesBackendLifecycleActorFactory.scala
new file mode 100644
index 000000000..41c03f083
--- /dev/null
+++ b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesBackendLifecycleActorFactory.scala
@@ -0,0 +1,34 @@
+package cromwell.backend.impl.tes
+
+import akka.actor.ActorRef
+import cromwell.backend._
+import cromwell.backend.callcaching.FileHashingActor.FileHashingFunction
+import cromwell.backend.standard._
+import cromwell.core.JobExecutionToken.JobExecutionTokenType
+import net.ceedubs.ficus.Ficus._
+import wdl4s.TaskCall
+
+case class TesBackendLifecycleActorFactory(name: String, configurationDescriptor: BackendConfigurationDescriptor)
+  extends StandardLifecycleActorFactory {
+
+  override lazy val initializationActorClass: Class[_ <: StandardInitializationActor] = classOf[TesInitializationActor]
+
+  override lazy val asyncExecutionActorClass: Class[_ <: StandardAsyncExecutionActor] =
+    classOf[TesAsyncBackendJobExecutionActor]
+
+  override def jobIdKey: String = TesAsyncBackendJobExecutionActor.JobIdKey
+
+  val tesConfiguration = new TesConfiguration(configurationDescriptor)
+
+  override val jobExecutionTokenType: JobExecutionTokenType = {
+    val concurrentJobLimit = configurationDescriptor.backendConfig.as[Option[Int]]("concurrent-job-limit")
+    JobExecutionTokenType(name, concurrentJobLimit)
+  }
+
+  override def workflowInitializationActorParams(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[TaskCall],
+                                                 serviceRegistryActor: ActorRef): StandardInitializationActorParams = {
+    TesInitializationActorParams(workflowDescriptor, calls, tesConfiguration, serviceRegistryActor)
+  }
+
+  override lazy val fileHashingFunction: Option[FileHashingFunction] = Option(FileHashingFunction(TesBackendFileHashing.getMd5Result))
+}
diff --git a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesConfiguration.scala b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesConfiguration.scala
new file mode 100644
index 000000000..65f4deec6
--- /dev/null
+++ b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesConfiguration.scala
@@ -0,0 +1,7 @@
+package cromwell.backend.impl.tes
+
+import cromwell.backend.BackendConfigurationDescriptor
+
+class TesConfiguration(val configurationDescriptor: BackendConfigurationDescriptor) {
+  val endpointURL = configurationDescriptor.backendConfig.getString("endpoint")
+}
diff --git a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesInitializationActor.scala b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesInitializationActor.scala
new file mode 100644
index 000000000..67a177b32
--- /dev/null
+++ b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesInitializationActor.scala
@@ -0,0 +1,40 @@
+package cromwell.backend.impl.tes
+
+import akka.actor.ActorRef
+import cromwell.backend.standard._
+import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendWorkflowDescriptor}
+import cromwell.core.path.Obsolete._
+import wdl4s.TaskCall
+
+import scala.concurrent.Future
+import scala.util.Try
+
+case class TesInitializationActorParams
+(
+  workflowDescriptor: BackendWorkflowDescriptor,
+  calls: Set[TaskCall],
+  tesConfiguration: TesConfiguration,
+  serviceRegistryActor: ActorRef
+) extends StandardInitializationActorParams {
+  override val configurationDescriptor: BackendConfigurationDescriptor = tesConfiguration.configurationDescriptor
+}
+
+class TesInitializationActor(params: TesInitializationActorParams)
+  extends StandardInitializationActor(params) {
+
+  private val tesConfiguration = params.tesConfiguration
+
+  override lazy val workflowPaths: TesWorkflowPaths =
+    new TesWorkflowPaths(workflowDescriptor, tesConfiguration.configurationDescriptor.backendConfig, pathBuilders)
+
+  override lazy val runtimeAttributesBuilder: StandardValidatedRuntimeAttributesBuilder =
+    TesRuntimeAttributes.runtimeAttributesBuilder
+
+  override def beforeAll(): Future[Option[BackendInitializationData]] = {
+    Future.fromTry(Try {
+      publishWorkflowRoot(workflowPaths.workflowRoot.toString)
+      File(workflowPaths.workflowRoot).createPermissionedDirectories()
+      Option(TesBackendInitializationData(workflowPaths, runtimeAttributesBuilder, tesConfiguration))
+    })
+  }
+}
diff --git a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesJobCachingActorHelper.scala b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesJobCachingActorHelper.scala
new file mode 100644
index 000000000..0f12d802f
--- /dev/null
+++ b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesJobCachingActorHelper.scala
@@ -0,0 +1,22 @@
+package cromwell.backend.impl.tes
+
+
+import akka.actor.Actor
+import cromwell.backend.standard.StandardCachingActorHelper
+import cromwell.core.logging.JobLogging
+
+trait TesJobCachingActorHelper extends StandardCachingActorHelper {
+  this: Actor with JobLogging =>
+
+  lazy val initializationData: TesBackendInitializationData = {
+    backendInitializationDataAs[TesBackendInitializationData]
+  }
+
+  lazy val tesWorkflowPaths: TesWorkflowPaths = workflowPaths.asInstanceOf[TesWorkflowPaths]
+
+  lazy val tesJobPaths: TesJobPaths = jobPaths.asInstanceOf[TesJobPaths]
+
+  lazy val tesConfiguration: TesConfiguration = initializationData.tesConfiguration
+
+  lazy val runtimeAttributes = TesRuntimeAttributes(validatedRuntimeAttributes)
+}
diff --git a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesJobPaths.scala b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesJobPaths.scala
new file mode 100644
index 000000000..ac7c9aff2
--- /dev/null
+++ b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesJobPaths.scala
@@ -0,0 +1,58 @@
+package cromwell.backend.impl.tes
+
+import com.typesafe.config.Config
+import cromwell.backend.io.{JobPaths, WorkflowPaths}
+import cromwell.backend.{BackendJobDescriptorKey, BackendWorkflowDescriptor}
+import cromwell.core.path.Obsolete._
+import cromwell.core.path.{Path, PathBuilder}
+
+class TesJobPaths(val jobKey: BackendJobDescriptorKey,
+                  workflowDescriptor: BackendWorkflowDescriptor,
+                  config: Config,
+                  pathBuilders: List[PathBuilder] = WorkflowPaths.DefaultPathBuilders) extends TesWorkflowPaths(
+  workflowDescriptor, config, pathBuilders) with JobPaths {
+
+  import JobPaths._
+
+  override lazy val callExecutionRoot = {
+    callRoot.resolve("execution")
+  }
+  val callDockerRoot = callPathBuilder(dockerWorkflowRoot, jobKey)
+  val callExecutionDockerRoot = callDockerRoot.resolve("execution")
+  val callInputsDockerRoot = callDockerRoot.resolve("inputs")
+  val callInputsRoot = callRoot.resolve("inputs")
+
+  //TODO move to TesConfiguration
+  private def prefixScheme(path: String): String = "file://" + path
+
+  def storageInput(path: String): String = prefixScheme(path)
+
+  // Given an output path, return a path localized to the storage file system
+  def storageOutput(path: String): String = {
+    prefixScheme(callExecutionRoot.resolve(path).toString)
+  }
+
+  def containerInput(path: String): String = {
+    cleanContainerInputPath(callInputsDockerRoot, Paths.get(path))
+  }
+
+  // Given an output path, return a path localized to the container file system
+  def containerOutput(cwd: Path, path: String): String = containerExec(cwd, path)
+
+  // TODO this could be used to create a separate directory for outputs e.g.
+  // callDockerRoot.resolve("outputs").resolve(name).toString
+
+  // Given an file name, return a path localized to the container's execution directory
+  def containerExec(cwd: Path, path: String): String = {
+    cwd.resolve(path).toString
+  }
+
+  private def cleanContainerInputPath(inputDir: Path, path: Path): String = {
+    path.toAbsolutePath match {
+      case p if p.startsWith(callExecutionRoot) =>
+        callExecutionDockerRoot.resolve(p.getFileName.toString).toString
+      case p =>
+        inputDir + p.toString
+    }
+  }
+}
diff --git a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesResponseJsonFormatter.scala b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesResponseJsonFormatter.scala
new file mode 100644
index 000000000..111f7e198
--- /dev/null
+++ b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesResponseJsonFormatter.scala
@@ -0,0 +1,25 @@
+package cromwell.backend.impl.tes
+
+import spray.json._
+
+final case class TesPostResponse(value: String)
+
+final case class TesGetResponse(
+                                 jobID: String,
+                                 task: TesTaskMessage,
+                                 state: String,
+                                 logs: Option[Seq[JobLogs]]
+                               )
+
+
+object TesResponseJsonFormatter extends DefaultJsonProtocol {
+  implicit val volumeFormat = jsonFormat5(Volume)
+  implicit val resourcesFormat = jsonFormat5(Resources)
+  implicit val taskParameterFormat = jsonFormat6(TaskParameter)
+  implicit val portsFormat = jsonFormat2(Ports)
+  implicit val dockerExecutorFormat = jsonFormat7(DockerExecutor)
+  implicit val tesTaskMessageFormat = jsonFormat7(TesTaskMessage)
+  implicit val jobLogsFormat = jsonFormat8(JobLogs)
+  implicit val tesPostResponseFormat = jsonFormat1(TesPostResponse)
+  implicit val tesGetResponseFormat = jsonFormat4(TesGetResponse)
+}
diff --git a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesRuntimeAttributes.scala b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesRuntimeAttributes.scala
new file mode 100644
index 000000000..06069c12b
--- /dev/null
+++ b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesRuntimeAttributes.scala
@@ -0,0 +1,133 @@
+package cromwell.backend.impl.tes
+
+import cats.syntax.validated._
+import cromwell.backend.MemorySize
+import cromwell.backend.standard.StandardValidatedRuntimeAttributesBuilder
+import cromwell.backend.validation._
+import lenthall.validation.ErrorOr.ErrorOr
+import wdl4s.parser.MemoryUnit
+import wdl4s.values.{WdlInteger, WdlString, WdlValue}
+
+case class TesRuntimeAttributes(continueOnReturnCode: ContinueOnReturnCode,
+                                dockerImage: String,
+                                dockerWorkingDir: Option[String],
+                                failOnStderr: Boolean,
+                                cpu: Int,
+                                memory: MemorySize,
+                                disk: MemorySize)
+
+object TesRuntimeAttributes {
+
+  val DockerWorkingDirKey = "dockerWorkingDir"
+
+  private val MemoryDefaultValue = "2 GB"
+
+  val DiskSizeKey = "disk"
+  private val DiskSizeDefaultValue = "2 GB"
+
+  private val cpuValidation: RuntimeAttributesValidation[Int] = CpuValidation.default
+
+  private val diskSizeValidation: RuntimeAttributesValidation[MemorySize] =
+    DiskSizeValidation.withDefaultDiskSize(MemorySize.parse(DiskSizeDefaultValue).get)
+
+  private val dockerValidation: RuntimeAttributesValidation[String] = DockerValidation.instance
+
+  private val dockerWorkingDirValidation: OptionalRuntimeAttributesValidation[String] = DockerWorkingDirValidation.optional
+
+  private val memoryValidation: RuntimeAttributesValidation[MemorySize] =
+    MemoryValidation.withDefaultMemory(MemorySize.parse(MemoryDefaultValue).get)
+
+  def runtimeAttributesBuilder: StandardValidatedRuntimeAttributesBuilder =
+    StandardValidatedRuntimeAttributesBuilder.default.withValidation(
+      cpuValidation,
+      memoryValidation,
+      diskSizeValidation,
+      dockerValidation,
+      dockerWorkingDirValidation
+    )
+
+  def apply(validatedRuntimeAttributes: ValidatedRuntimeAttributes): TesRuntimeAttributes = {
+    val docker: String = RuntimeAttributesValidation.extract(dockerValidation, validatedRuntimeAttributes)
+    val dockerWorkingDir: Option[String] = RuntimeAttributesValidation.extractOption(dockerWorkingDirValidation.key, validatedRuntimeAttributes)
+    val cpu: Int = RuntimeAttributesValidation.extract(cpuValidation, validatedRuntimeAttributes)
+    val memory: MemorySize = RuntimeAttributesValidation.extract(memoryValidation, validatedRuntimeAttributes)
+    val disk: MemorySize = RuntimeAttributesValidation.extract(diskSizeValidation, validatedRuntimeAttributes)
+    val failOnStderr: Boolean =
+      RuntimeAttributesValidation.extract(FailOnStderrValidation.default, validatedRuntimeAttributes)
+    val continueOnReturnCode: ContinueOnReturnCode =
+      RuntimeAttributesValidation.extract(ContinueOnReturnCodeValidation.default, validatedRuntimeAttributes)
+
+    new TesRuntimeAttributes(
+      continueOnReturnCode,
+      docker,
+      dockerWorkingDir,
+      failOnStderr,
+      cpu,
+      memory,
+      disk
+    )
+  }
+}
+
+object DockerWorkingDirValidation {
+  lazy val instance: RuntimeAttributesValidation[String] = new DockerWorkingDirValidation
+  lazy val optional: OptionalRuntimeAttributesValidation[String] = instance.optional
+}
+
+class DockerWorkingDirValidation extends StringRuntimeAttributesValidation(TesRuntimeAttributes.DockerWorkingDirKey) {
+  // NOTE: Docker's current test specs don't like WdlInteger, etc. auto converted to WdlString.
+  override protected def validateValue: PartialFunction[WdlValue, ErrorOr[String]] = {
+    case WdlString(value) => value.validNel
+  }
+}
+
+object DiskSizeValidation {
+  lazy val instance: RuntimeAttributesValidation[MemorySize] = new DiskSizeValidation
+  lazy val optional: OptionalRuntimeAttributesValidation[MemorySize] = instance.optional
+
+  def withDefaultDiskSize(memorySize: MemorySize): RuntimeAttributesValidation[MemorySize] =
+    instance.withDefault(WdlInteger(memorySize.bytes.toInt))
+
+  private val wrongAmountFormat =
+    s"Expecting ${TesRuntimeAttributes.DiskSizeKey} runtime attribute value greater than 0 but got %s"
+  private val wrongTypeFormat =
+    s"Expecting ${TesRuntimeAttributes.DiskSizeKey} runtime attribute to be an Integer or String with format '8 GB'." +
+      s" Exception: %s"
+
+  def validateDiskSizeString(wdlString: WdlString): ErrorOr[MemorySize] =
+    validateDiskSizeString(wdlString.value)
+
+  def validateDiskSizeString(value: String): ErrorOr[MemorySize] = {
+    MemorySize.parse(value) match {
+      case scala.util.Success(memorySize: MemorySize) if memorySize.amount > 0 =>
+        memorySize.to(MemoryUnit.GB).validNel
+      case scala.util.Success(memorySize: MemorySize) =>
+        wrongAmountFormat.format(memorySize.amount).invalidNel
+      case scala.util.Failure(throwable) =>
+        wrongTypeFormat.format(throwable.getMessage).invalidNel
+    }
+  }
+
+  def validateDiskSizeInteger(wdlInteger: WdlInteger): ErrorOr[MemorySize] =
+    validateDiskSizeInteger(wdlInteger.value)
+
+  def validateDiskSizeInteger(value: Int): ErrorOr[MemorySize] = {
+    if (value <= 0)
+      wrongAmountFormat.format(value).invalidNel
+    else
+      MemorySize(value.toDouble, MemoryUnit.Bytes).to(MemoryUnit.GB).validNel
+  }
+}
+
+class DiskSizeValidation extends MemoryValidation {
+  override def key = TesRuntimeAttributes.DiskSizeKey
+
+  override protected def validateValue: PartialFunction[WdlValue, ErrorOr[MemorySize]] = {
+    case WdlInteger(value) => DiskSizeValidation.validateDiskSizeInteger(value)
+    case WdlString(value) => DiskSizeValidation.validateDiskSizeString(value)
+  }
+
+  override def missingValueMessage: String = DiskSizeValidation.wrongTypeFormat.format("Not supported WDL type value")
+}
+
+
diff --git a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesTask.scala b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesTask.scala
new file mode 100644
index 000000000..c5f540ec1
--- /dev/null
+++ b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesTask.scala
@@ -0,0 +1,235 @@
+package cromwell.backend.impl.tes
+
+import cromwell.backend.standard.StandardExpressionFunctions
+import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptor}
+import cromwell.core.logging.JobLogger
+import cromwell.core.path.Obsolete.Paths
+import cromwell.core.path.Path
+import wdl4s.FullyQualifiedName
+import wdl4s.expression.NoFunctions
+import wdl4s.parser.MemoryUnit
+import wdl4s.values.{WdlFile, WdlGlobFile, WdlSingleFile}
+
+final case class TesTask(jobDescriptor: BackendJobDescriptor,
+                         configurationDescriptor: BackendConfigurationDescriptor,
+                         jobLogger: JobLogger,
+                         tesPaths: TesJobPaths,
+                         runtimeAttributes: TesRuntimeAttributes,
+                         containerWorkDir: Path,
+                         backendEngineFunctions: StandardExpressionFunctions) {
+
+  private val workflowDescriptor = jobDescriptor.workflowDescriptor
+  private val workflowName = workflowDescriptor.workflow.unqualifiedName
+  private val fullyQualifiedTaskName = jobDescriptor.call.fullyQualifiedName
+  val name = fullyQualifiedTaskName
+  val description = jobDescriptor.toString
+
+  // TODO validate "project" field of workflowOptions
+  val project = {
+    workflowDescriptor.workflowOptions.getOrElse("project", workflowName)
+  }
+
+  // contains the script to be executed
+  private val commandScript = TaskParameter(
+    Option("commandScript"),
+    Option(fullyQualifiedTaskName + ".commandScript"),
+    tesPaths.storageInput(tesPaths.script.toString),
+    tesPaths.callExecutionDockerRoot.resolve("script").toString,
+    "File",
+    Option(false)
+  )
+
+  private val writeFunctionFiles: Map[FullyQualifiedName, Seq[WdlFile]] = jobDescriptor
+    .call
+    .task
+    .evaluateFilesFromCommand(jobDescriptor.fullyQualifiedInputs, backendEngineFunctions)
+    .map {
+    case (expression, file) => expression.toWdlString -> Seq(file)
+  }
+
+  private val callInputFiles: Map[FullyQualifiedName, Seq[WdlFile]] = jobDescriptor
+    .fullyQualifiedInputs
+    .mapValues {
+      _.collectAsSeq { case w: WdlFile => w }
+    }
+
+  val inputs: Seq[TaskParameter] = (callInputFiles ++ writeFunctionFiles)
+    .flatMap {
+      case (name, files) => files.zipWithIndex.map {
+        case (f, index) => TaskParameter(
+          Option(name + "." + index),
+          Option(workflowName + "." + name + "." + index),
+          tesPaths.storageInput(f.value),
+          tesPaths.containerInput(f.value),
+          "File",
+          Option(false)
+        )
+      }
+    }.toList ++ Seq(commandScript)
+
+  // TODO add TES logs to standard outputs
+  private val standardOutputs = Seq("rc", "stdout", "stderr").map {
+    f =>
+      TaskParameter(
+        Option(f),
+        Option(fullyQualifiedTaskName + "." + f),
+        tesPaths.storageOutput(f),
+        tesPaths.containerOutput(containerWorkDir, f),
+        "File",
+        Option(false)
+      )
+  }
+
+  // TODO extract output file variable names and match with Files below
+  // The problem is that we only care about the files CREATED, so stdout and input redirects are ignored and
+  // thus we can't directly match the names returned here to the files returned below. Also we have to consider Arrays
+  //
+  //  private val outputFileNames = jobDescriptor.call.task.outputs
+  //    .filter(o => o.wdlType.toWdlString == "Array[File]" || o.wdlType.toWdlString == "File")
+  //    .map(_.unqualifiedName)
+
+  // extract output files
+  // if output paths are absolute we will ignore them here and assume they are redirects
+  private val outputWdlFiles: Seq[WdlFile] = jobDescriptor.call.task
+    .findOutputFiles(jobDescriptor.fullyQualifiedInputs, NoFunctions)
+    .filter(o => !Paths.get(o.valueString).isAbsolute)
+
+  private val wdlOutputs = outputWdlFiles
+    .zipWithIndex
+    .flatMap {
+      case (f: WdlSingleFile, index) =>
+        val outputFile = f.value
+        Seq(
+          TaskParameter(
+            Option(fullyQualifiedTaskName + ".output." + index),
+            Option(fullyQualifiedTaskName + ".output." + index),
+            tesPaths.storageOutput(outputFile),
+            tesPaths.containerOutput(containerWorkDir, outputFile),
+            "File",
+            Option(false)
+          )
+        )
+      case (g: WdlGlobFile, index) =>
+        val globName = backendEngineFunctions.globName(g.value)
+        val globDirName = "globDir." + index
+        val globDirectory = globName + "/"
+        val globListName =  "globList." + index
+        val globListFile = globName + ".list"
+        Seq(
+          TaskParameter(
+            Option(globDirName),
+            Option(fullyQualifiedTaskName + "." + globDirName),
+            tesPaths.storageOutput(globDirectory),
+            tesPaths.containerOutput(containerWorkDir, globDirectory),
+            "Directory",
+            Option(false)
+          ),
+          TaskParameter(
+            Option(globListName),
+            Option(fullyQualifiedTaskName + "." + globListName),
+            tesPaths.storageOutput(globListFile),
+            tesPaths.containerOutput(containerWorkDir, globListFile),
+            "File",
+            Option(false)
+          )
+        )
+    }
+
+  val outputs: Seq[TaskParameter] = wdlOutputs ++ standardOutputs
+
+  // TODO all volumes currently get the same disk requirements
+  private val workingDirVolume = runtimeAttributes
+    .dockerWorkingDir
+    .map(path => Volume(
+      Option(path),
+      runtimeAttributes.disk.to(MemoryUnit.GB).amount.toInt,
+      None,
+      path,
+      Option(false)
+    ))
+
+  val volumes = Seq(
+    Volume(
+      Option(tesPaths.callInputsDockerRoot.toString),
+      runtimeAttributes.disk.to(MemoryUnit.GB).amount.toInt,
+      None,
+      tesPaths.callInputsDockerRoot.toString,
+      // inputs in read-only volume
+      Option(true)
+    ),
+    Volume(
+      Option(tesPaths.callExecutionDockerRoot.toString),
+      runtimeAttributes.disk.to(MemoryUnit.GB).amount.toInt,
+      None,
+      tesPaths.callExecutionDockerRoot.toString,
+      Option(false)
+    )
+  ) ++ workingDirVolume
+
+  val resources = Resources(
+    runtimeAttributes.cpu,
+    runtimeAttributes.memory.to(MemoryUnit.GB).amount.toInt,
+    Option(false),
+    volumes,
+    None
+  )
+
+  val dockerExecutor = Seq(DockerExecutor(
+    runtimeAttributes.dockerImage,
+    Seq("/bin/bash", commandScript.path),
+    runtimeAttributes.dockerWorkingDir,
+    Option(tesPaths.containerOutput(containerWorkDir, "stdout")),
+    Option(tesPaths.containerOutput(containerWorkDir, "stderr")),
+    None,
+    None
+  ))
+}
+
+// Field requirements in classes below based off GA4GH schema
+
+final case class TesTaskMessage(name: Option[String],
+                                description: Option[String],
+                                projectId: Option[String],
+                                inputs: Option[Seq[TaskParameter]],
+                                outputs: Option[Seq[TaskParameter]],
+                                resources: Resources,
+                                docker: Seq[DockerExecutor])
+
+final case class DockerExecutor(imageName: String,
+                                cmd: Seq[String],
+                                workdir: Option[String],
+                                stdout: Option[String],
+                                stderr: Option[String],
+                                stdin: Option[String],
+                                ports: Option[Seq[Ports]])
+
+final case class TaskParameter(name: Option[String],
+                               description: Option[String],
+                               location: String,
+                               path: String,
+                               `class`: String,
+                               create: Option[Boolean])
+
+final case class Resources(minimumCpuCores: Int,
+                           minimumRamGb: Int,
+                           preemptible: Option[Boolean],
+                           volumes: Seq[Volume],
+                           zones: Option[Seq[String]])
+
+final case class Volume(name: Option[String],
+                        sizeGb: Int,
+                        source: Option[String],
+                        mountPoint: String,
+                        readonly: Option[Boolean])
+
+final case class JobLogs(cmd: Option[Seq[String]],
+                         startTime: Option[String],
+                         endTime: Option[String],
+                         stdout: Option[String],
+                         stderr: Option[String],
+                         exitCode: Option[Int],
+                         hostIP: Option[String],
+                         ports: Option[Seq[Ports]])
+
+final case class Ports(host: Option[String],
+                       container: String)
\ No newline at end of file
diff --git a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesWorkflowPaths.scala b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesWorkflowPaths.scala
new file mode 100644
index 000000000..0fcd9c4f9
--- /dev/null
+++ b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesWorkflowPaths.scala
@@ -0,0 +1,24 @@
+package cromwell.backend.impl.tes
+
+import com.typesafe.config.Config
+import cromwell.backend.io.WorkflowPaths
+import cromwell.backend.{BackendJobDescriptorKey, BackendWorkflowDescriptor}
+import cromwell.core.path.{PathBuilder, PathFactory}
+import net.ceedubs.ficus.Ficus._
+
+class TesWorkflowPaths(override val workflowDescriptor: BackendWorkflowDescriptor,
+                       override val config: Config,
+                       override val pathBuilders: List[PathBuilder] = WorkflowPaths.DefaultPathBuilders) extends WorkflowPaths {
+
+  val DockerRootString = config.as[Option[String]]("dockerRoot").getOrElse("/cromwell-executions")
+  var DockerRoot = PathFactory.buildPath(DockerRootString, pathBuilders)
+  if (!DockerRoot.isAbsolute) {
+    DockerRoot = PathFactory.buildPath("/".concat(DockerRootString), pathBuilders)
+  }
+  val dockerWorkflowRoot = workflowPathBuilder(DockerRoot)
+
+  override def toJobPaths(jobKey: BackendJobDescriptorKey,
+                          jobWorkflowDescriptor: BackendWorkflowDescriptor): TesJobPaths = {
+    new TesJobPaths(jobKey, jobWorkflowDescriptor, config, pathBuilders)
+  }
+}
diff --git a/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesInitializationActorSpec.scala b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesInitializationActorSpec.scala
new file mode 100644
index 000000000..0f033bbb1
--- /dev/null
+++ b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesInitializationActorSpec.scala
@@ -0,0 +1,101 @@
+package cromwell.backend.impl.tes
+
+import java.util.UUID
+
+import akka.actor.Props
+import akka.testkit.{EventFilter, ImplicitSender, TestDuration}
+import com.typesafe.config.{Config, ConfigFactory}
+import cromwell.backend.BackendSpec._
+import cromwell.backend.BackendWorkflowInitializationActor.{InitializationFailed, InitializationSuccess, Initialize}
+import cromwell.backend.async.RuntimeAttributeValidationFailures
+import cromwell.backend.{BackendConfigurationDescriptor, BackendWorkflowDescriptor}
+import cromwell.core.TestKitSuite
+import cromwell.core.logging.LoggingTest._
+import org.scalatest.{Matchers, WordSpecLike}
+import wdl4s.TaskCall
+
+import scala.concurrent.duration._
+
+class TesInitializationActorSpec extends TestKitSuite("TesInitializationActorSpec")
+  with WordSpecLike with Matchers with ImplicitSender {
+  val Timeout = 10.second.dilated
+
+  val HelloWorld =
+    s"""
+       |task hello {
+       |  String addressee = "you"
+       |  command {
+       |    echo "Hello $${addressee}!"
+       |  }
+       |  output {
+       |    String salutation = read_string(stdout())
+       |  }
+       |
+       |  RUNTIME
+       |}
+       |
+       |workflow wf_hello {
+       |  call hello
+       |}
+    """.stripMargin
+
+  val globalConfig: Config = ConfigFactory.parseString("")
+
+  val backendConfigTemplate: String =
+    """
+      |// Base bucket for workflow executions
+      |root = "cromwell-executions"
+      |endpoint = "0.0.0.0"
+      |
+      |// Polling for completion backs-off gradually for slower-running jobs.
+      |// This is the maximum polling interval (in seconds):
+      |maximum-polling-interval = 600
+      |""".stripMargin
+
+
+  private def getActorRef(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[TaskCall],
+                          conf: BackendConfigurationDescriptor) = {
+    val params = TesInitializationActorParams(workflowDescriptor, calls, new TesConfiguration(conf), emptyActor)
+    val props = Props(new TesInitializationActor(params))
+    system.actorOf(props, "TesInitializationActor" + UUID.randomUUID)
+  }
+
+  val backendConfig: Config = ConfigFactory.parseString(backendConfigTemplate)
+  val conf = BackendConfigurationDescriptor(backendConfig, globalConfig)
+
+  "TesInitializationActor" should {
+    "log a warning message when there are unsupported runtime attributes" in {
+      within(Timeout) {
+        val workflowDescriptor = buildWorkflowDescriptor(HelloWorld,
+          runtime = """runtime { docker: "ubuntu/latest" test: true }""")
+        val backend = getActorRef(workflowDescriptor, workflowDescriptor.workflow.taskCalls, conf)
+        val eventPattern =
+          "Key/s [test] is/are not supported by backend. Unsupported attributes will not be part of job executions."
+        EventFilter.warning(pattern = escapePattern(eventPattern), occurrences = 1) intercept {
+          backend ! Initialize
+        }
+        expectMsgPF() {
+          case InitializationSuccess(_) => // Docker entry is present.
+          case InitializationFailed(failure) => fail(s"InitializationSuccess was expected but got $failure")
+        }
+      }
+    }
+
+    "return InitializationFailed when docker runtime attribute key is not present" in {
+      within(Timeout) {
+        val workflowDescriptor = buildWorkflowDescriptor(HelloWorld, runtime = """runtime { }""")
+        val backend = getActorRef(workflowDescriptor, workflowDescriptor.workflow.taskCalls, conf)
+        backend ! Initialize
+        expectMsgPF() {
+          case InitializationFailed(failure) =>
+            failure match {
+              case exception: RuntimeAttributeValidationFailures =>
+                if (!exception.getMessage.equals("Runtime validation failed:\nTask hello has an invalid runtime attribute docker = !! NOT FOUND !!"))
+                  fail("Exception message is not equal to 'Runtime validation failed:\nTask hello has an invalid runtime attribute docker = !! NOT FOUND !!'.")
+            }
+        }
+      }
+    }
+  }
+}
+
diff --git a/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesJobPathsSpec.scala b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesJobPathsSpec.scala
new file mode 100644
index 000000000..a5c1e6eeb
--- /dev/null
+++ b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesJobPathsSpec.scala
@@ -0,0 +1,73 @@
+package cromwell.backend.impl.tes
+
+import better.files._
+import com.typesafe.config.ConfigFactory
+import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptorKey, BackendSpec}
+import org.scalatest.{FlatSpec, Matchers}
+import wdl4s.TaskCall
+
+class TesJobPathsSpec extends FlatSpec with Matchers with BackendSpec {
+
+  val configString =
+    """
+      |root = "local-cromwell-executions"
+      |dockerRoot = "/cromwell-executions"
+      |
+      |filesystems {
+      |  local {
+      |    localization: [
+      |      "hard-link", "soft-link", "copy"
+      |    ]
+      |  }
+      |  gcs {
+      |    auth = "application-default"
+      |  }
+      |}
+      |""".stripMargin
+
+  val globalConfig = ConfigFactory.load()
+  val backendConfig =  ConfigFactory.parseString(configString)
+  val defaultBackendConfigDescriptor = BackendConfigurationDescriptor(backendConfig, globalConfig)
+
+  "JobPaths" should "provide correct paths for a job" in {
+
+    val wd = buildWorkflowDescriptor(TestWorkflows.HelloWorld)
+    val call: TaskCall = wd.workflow.taskCalls.head
+    val jobKey = BackendJobDescriptorKey(call, None, 1)
+    val jobPaths = new TesJobPaths(jobKey, wd, backendConfig)
+    val id = wd.id
+    jobPaths.callRoot.toString shouldBe
+      File(s"local-cromwell-executions/wf_hello/$id/call-hello").pathAsString
+    jobPaths.callExecutionRoot.toString shouldBe
+      File(s"local-cromwell-executions/wf_hello/$id/call-hello/execution").pathAsString
+    jobPaths.returnCode.toString shouldBe
+      File(s"local-cromwell-executions/wf_hello/$id/call-hello/execution/rc").pathAsString
+    jobPaths.script.toString shouldBe
+      File(s"local-cromwell-executions/wf_hello/$id/call-hello/execution/script").pathAsString
+    jobPaths.stderr.toString shouldBe
+      File(s"local-cromwell-executions/wf_hello/$id/call-hello/execution/stderr").pathAsString
+    jobPaths.stdout.toString shouldBe
+      File(s"local-cromwell-executions/wf_hello/$id/call-hello/execution/stdout").pathAsString
+    jobPaths.callExecutionRoot.toString shouldBe
+      File(s"local-cromwell-executions/wf_hello/$id/call-hello/execution").pathAsString
+    jobPaths.callDockerRoot.toString shouldBe
+      File(s"/cromwell-executions/wf_hello/$id/call-hello").pathAsString
+    jobPaths.callExecutionDockerRoot.toString shouldBe
+      File(s"/cromwell-executions/wf_hello/$id/call-hello/execution").pathAsString
+
+    val jobKeySharded = BackendJobDescriptorKey(call, Option(0), 1)
+    val jobPathsSharded = new TesJobPaths(jobKeySharded, wd, backendConfig)
+    jobPathsSharded.callExecutionRoot.toString shouldBe
+      File(s"local-cromwell-executions/wf_hello/$id/call-hello/shard-0/execution").pathAsString
+
+    val jobKeyAttempt = BackendJobDescriptorKey(call, None, 2)
+    val jobPathsAttempt = new TesJobPaths(jobKeyAttempt, wd, backendConfig)
+    jobPathsAttempt.callExecutionRoot.toString shouldBe
+      File(s"local-cromwell-executions/wf_hello/$id/call-hello/attempt-2/execution").pathAsString
+
+    val jobKeyShardedAttempt = BackendJobDescriptorKey(call, Option(0), 2)
+    val jobPathsShardedAttempt = new TesJobPaths(jobKeyShardedAttempt, wd, backendConfig)
+    jobPathsShardedAttempt.callExecutionRoot.toString shouldBe
+      File(s"local-cromwell-executions/wf_hello/$id/call-hello/shard-0/attempt-2/execution").pathAsString
+  }
+}
diff --git a/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesRuntimeAttributesSpec.scala b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesRuntimeAttributesSpec.scala
new file mode 100644
index 000000000..5fb711925
--- /dev/null
+++ b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesRuntimeAttributesSpec.scala
@@ -0,0 +1,178 @@
+package cromwell.backend.impl.tes
+
+import cromwell.backend.validation.ContinueOnReturnCodeSet
+import cromwell.backend.{MemorySize, RuntimeAttributeDefinition}
+import cromwell.core.WorkflowOptions
+import org.scalatest.{Matchers, WordSpecLike}
+import org.slf4j.helpers.NOPLogger
+import spray.json._
+import wdl4s.parser.MemoryUnit
+import wdl4s.types.{WdlArrayType, WdlIntegerType, WdlStringType}
+import wdl4s.values.{WdlArray, WdlBoolean, WdlInteger, WdlString, WdlValue}
+
+class TesRuntimeAttributesSpec extends WordSpecLike with Matchers {
+
+  val expectedDefaults = new TesRuntimeAttributes(
+    ContinueOnReturnCodeSet(Set(0)),
+    "ubuntu:latest",
+    None,
+    false,
+    1,
+    MemorySize(2, MemoryUnit.GB),
+    MemorySize(2, MemoryUnit.GB)
+  )
+
+  val expectedDefaultsPlusUbuntuDocker = expectedDefaults.copy(dockerImage = "ubuntu:latest")
+
+  def workflowOptionsWithDefaultRA(defaults: Map[String, JsValue]) = {
+    WorkflowOptions(JsObject(Map(
+      "default_runtime_attributes" -> JsObject(defaults)
+    )))
+  }
+
+  "TesRuntimeAttributes" should {
+
+    "throw an exception when there are no runtime attributes defined." in {
+      val runtimeAttributes = Map.empty[String, WdlValue]
+      assertFailure(runtimeAttributes, "Can't find an attribute value for key docker")
+    }
+
+    "validate a valid Docker entry" in {
+      val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest"))
+      val expectedRuntimeAttributes = expectedDefaults.copy(dockerImage = "ubuntu:latest")
+      assertSuccess(runtimeAttributes, expectedRuntimeAttributes)
+    }
+
+    "fail to validate an invalid Docker entry" in {
+      val runtimeAttributes = Map("docker" -> WdlInteger(1))
+      assertFailure(runtimeAttributes, "Expecting docker runtime attribute to be a String")
+    }
+
+    "validate a valid failOnStderr entry" in {
+      val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest"), "failOnStderr" -> WdlBoolean(true))
+      val expectedRuntimeAttributes = expectedDefaultsPlusUbuntuDocker.copy(failOnStderr = true)
+      assertSuccess(runtimeAttributes, expectedRuntimeAttributes)
+    }
+
+    "fail to validate an invalid failOnStderr entry" in {
+      val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest"), "failOnStderr" -> WdlString("yes"))
+      assertFailure(runtimeAttributes, "Expecting failOnStderr runtime attribute to be a Boolean or a String with values of 'true' or 'false'")
+    }
+
+    "validate a valid continueOnReturnCode entry" in {
+      val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest"), "continueOnReturnCode" -> WdlInteger(1))
+      val expectedRuntimeAttributes = expectedDefaultsPlusUbuntuDocker.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1)))
+      assertSuccess(runtimeAttributes, expectedRuntimeAttributes)
+    }
+
+    "validate a valid continueOnReturnCode array entry" in {
+      val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest"), "continueOnReturnCode" -> WdlArray(WdlArrayType(WdlIntegerType), Array(WdlInteger(1), WdlInteger(2))))
+      val expectedRuntimeAttributes = expectedDefaultsPlusUbuntuDocker.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1, 2)))
+      assertSuccess(runtimeAttributes, expectedRuntimeAttributes)
+    }
+
+    "coerce then validate a valid continueOnReturnCode array entry" in {
+      val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest"), "continueOnReturnCode" -> WdlArray(WdlArrayType(WdlStringType), Array(WdlString("1"), WdlString("2"))))
+      val expectedRuntimeAttributes = expectedDefaultsPlusUbuntuDocker.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1, 2)))
+      assertSuccess(runtimeAttributes, expectedRuntimeAttributes)
+    }
+
+    "fail to validate an invalid continueOnReturnCode entry" in {
+      val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest"), "continueOnReturnCode" -> WdlString("value"))
+      assertFailure(runtimeAttributes, "Expecting continueOnReturnCode runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]")
+    }
+
+    "validate a valid cpu entry" in assertSuccess(
+      Map("docker" -> WdlString("ubuntu:latest"), "cpu" -> WdlInteger(2)),
+      expectedDefaultsPlusUbuntuDocker.copy(cpu = 2)
+    )
+
+    "validate a valid cpu string entry" in {
+      val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest"), "cpu" -> WdlString("2"))
+      val expectedRuntimeAttributes = expectedDefaultsPlusUbuntuDocker.copy(cpu = 2)
+      assertSuccess(runtimeAttributes, expectedRuntimeAttributes)
+    }
+
+    "fail to validate an invalid cpu entry" in {
+      val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest"), "cpu" -> WdlString("value"))
+      assertFailure(runtimeAttributes, "Expecting cpu runtime attribute to be an Integer")
+    }
+
+    "validate a valid memory entry" in {
+      val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest"), "memory" -> WdlString("1 GB"))
+      val expectedRuntimeAttributes = expectedDefaults.copy(memory = MemorySize.parse("1 GB").get)
+      assertSuccess(runtimeAttributes, expectedRuntimeAttributes)
+    }
+
+    "fail to validate an invalid memory entry" in {
+      val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest"), "memory" -> WdlString("blah"))
+      assertFailure(runtimeAttributes, "Expecting memory runtime attribute to be an Integer or String with format '8 GB'")
+    }
+
+    "validate a valid disks entry" in {
+      val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest"), "disk" -> WdlString("1 GB"))
+      val expectedRuntimeAttributes = expectedDefaults.copy(disk = MemorySize.parse("1 GB").get)
+      assertSuccess(runtimeAttributes, expectedRuntimeAttributes)
+    }
+
+    "fail to validate an invalid disks entry" in {
+      val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest"), "disk" -> WdlString("blah"))
+      assertFailure(runtimeAttributes, "Expecting disk runtime attribute to be an Integer or String with format '8 GB'")
+    }
+
+    "validate a valid dockerWorkingDir entry" in {
+      val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest"), "dockerWorkingDir" -> WdlString("/tmp"))
+      val expectedRuntimeAttributes = expectedDefaults.copy(dockerWorkingDir = Option("/tmp"))
+      assertSuccess(runtimeAttributes, expectedRuntimeAttributes)
+    }
+
+    "fail to validate an invalid dockerWorkingDir entry" in {
+      val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest"), "dockerWorkingDir" -> WdlInteger(1))
+      assertFailure(runtimeAttributes, "Expecting dockerWorkingDir runtime attribute to be a String")
+    }
+
+    "use reasonable default values" in assertSuccess(
+      Map("docker" -> WdlString("ubuntu:latest")),
+      expectedDefaultsPlusUbuntuDocker
+    )
+  }
+
+  private def assertSuccess(runtimeAttributes: Map[String, WdlValue],
+                            expectedRuntimeAttributes: TesRuntimeAttributes,
+                            workflowOptions: WorkflowOptions = emptyWorkflowOptions): Unit = {
+
+    try {
+      val actualRuntimeAttributes = toTesRuntimeAttributes(runtimeAttributes, workflowOptions)
+      assert(actualRuntimeAttributes == expectedRuntimeAttributes)
+    } catch {
+      case ex: RuntimeException => fail(s"Exception was not expected but received: ${ex.getMessage}")
+    }
+    ()
+  }
+
+  private def assertFailure(runtimeAttributes: Map[String, WdlValue],
+                            exMsg: String,
+                            workflowOptions: WorkflowOptions = emptyWorkflowOptions): Unit = {
+    try {
+      toTesRuntimeAttributes(runtimeAttributes, workflowOptions)
+      fail("A RuntimeException was expected.")
+    } catch {
+      case ex: RuntimeException => assert(ex.getMessage.contains(exMsg))
+    }
+    ()
+  }
+
+  private val emptyWorkflowOptions = WorkflowOptions.fromMap(Map.empty).get
+  private val staticRuntimeAttributeDefinitions: Set[RuntimeAttributeDefinition] =
+    TesRuntimeAttributes.runtimeAttributesBuilder.definitions.toSet
+
+
+  private def toTesRuntimeAttributes(runtimeAttributes: Map[String, WdlValue],
+                                     workflowOptions: WorkflowOptions): TesRuntimeAttributes = {
+    val runtimeAttributesBuilder = TesRuntimeAttributes.runtimeAttributesBuilder
+    val defaultedAttributes = RuntimeAttributeDefinition.addDefaultsToAttributes(
+      staticRuntimeAttributeDefinitions, workflowOptions)(runtimeAttributes)
+    val validatedRuntimeAttributes = runtimeAttributesBuilder.build(defaultedAttributes, NOPLogger.NOP_LOGGER)
+    TesRuntimeAttributes(validatedRuntimeAttributes)
+  }
+}
\ No newline at end of file
diff --git a/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesWorkflowPathsSpec.scala b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesWorkflowPathsSpec.scala
new file mode 100644
index 000000000..591b65193
--- /dev/null
+++ b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesWorkflowPathsSpec.scala
@@ -0,0 +1,75 @@
+package cromwell.backend.impl.tes
+
+import better.files._
+import com.typesafe.config.ConfigFactory
+import cromwell.backend.{BackendJobBreadCrumb, BackendSpec, BackendWorkflowDescriptor}
+import cromwell.core.{JobKey, WorkflowId}
+import org.scalatest.{FlatSpec, Matchers}
+import wdl4s.{Call, Workflow}
+
+class TesWorkflowPathsSpec extends FlatSpec with Matchers with BackendSpec {
+  val configString =
+    """
+      |root = "local-cromwell-executions"
+      |dockerRoot = "cromwell-executions"
+      |
+      |filesystems {
+      |  local {
+      |    localization: [
+      |      "hard-link", "soft-link", "copy"
+      |    ]
+      |  }
+      |  gcs {
+      |    auth = "application-default"
+      |  }
+      |}
+      |""".stripMargin
+
+  val globalConfig = ConfigFactory.load()
+  val backendConfig =  ConfigFactory.parseString(configString)
+
+  "WorkflowPaths" should "provide correct paths for a workflow" in {
+    val wd = buildWorkflowDescriptor(TestWorkflows.HelloWorld)
+    val workflowPaths = new TesWorkflowPaths(wd, backendConfig)
+    val id = wd.id
+    workflowPaths.workflowRoot.toString shouldBe
+      File(s"local-cromwell-executions/wf_hello/$id").pathAsString
+    workflowPaths.dockerWorkflowRoot.toString shouldBe
+      s"/cromwell-executions/wf_hello/$id"
+  }
+
+  "WorkflowPaths" should "provide correct paths for a sub workflow" in {
+    val rootWd = mock[BackendWorkflowDescriptor]
+    val rootWorkflow = mock[Workflow]
+    val rootWorkflowId = WorkflowId.randomId()
+    rootWorkflow.unqualifiedName returns "rootWorkflow"
+    rootWd.workflow returns rootWorkflow
+    rootWd.id returns rootWorkflowId
+
+    val subWd = mock[BackendWorkflowDescriptor]
+    val subWorkflow = mock[Workflow]
+    val subWorkflowId = WorkflowId.randomId()
+    subWorkflow.unqualifiedName returns "subWorkflow"
+    subWd.workflow returns subWorkflow
+    subWd.id returns subWorkflowId
+    
+    val call1 = mock[Call]
+    call1.unqualifiedName returns "call1"
+    val call2 = mock[Call]
+    call2.unqualifiedName returns "call2"
+    
+    val jobKey = new JobKey {
+      override def scope = call1
+      override def tag: String = "tag1"
+      override def index: Option[Int] = Option(1)
+      override def attempt: Int = 2
+    }
+
+    subWd.breadCrumbs returns List(BackendJobBreadCrumb(rootWorkflow, rootWorkflowId, jobKey))
+    subWd.id returns subWorkflowId
+    
+    val workflowPaths = new TesWorkflowPaths(subWd, backendConfig)
+    workflowPaths.workflowRoot.toString shouldBe File(s"local-cromwell-executions/rootWorkflow/$rootWorkflowId/call-call1/shard-1/attempt-2/subWorkflow/$subWorkflowId").pathAsString
+    workflowPaths.dockerWorkflowRoot.toString shouldBe s"/cromwell-executions/rootWorkflow/$rootWorkflowId/call-call1/shard-1/attempt-2/subWorkflow/$subWorkflowId"
+  }
+}
diff --git a/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TestWorkflows.scala b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TestWorkflows.scala
new file mode 100644
index 000000000..e98ca5210
--- /dev/null
+++ b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TestWorkflows.scala
@@ -0,0 +1,141 @@
+package cromwell.backend.impl.tes
+
+import cromwell.backend.BackendJobExecutionActor.BackendJobExecutionResponse
+import cromwell.backend.{BackendConfigurationDescriptor, BackendWorkflowDescriptor}
+
+object TestWorkflows {
+
+  case class TestWorkflow(workflowDescriptor: BackendWorkflowDescriptor,
+                          config: BackendConfigurationDescriptor,
+                          expectedResponse: BackendJobExecutionResponse)
+
+  val HelloWorld =
+    s"""
+      |task hello {
+      |  String addressee = "you "
+      |  command {
+      |    echo "Hello $${addressee}!"
+      |  }
+      |  output {
+      |    String salutation = read_string(stdout())
+      |  }
+      |
+      |  RUNTIME
+      |}
+      |
+      |workflow wf_hello {
+      |  call hello
+      |}
+      |""".stripMargin
+
+  val GoodbyeWorld =
+    """
+      |task goodbye {
+      |  command {
+      |    sh -c "exit 1"
+      |  }
+      |  output {
+      |    String out = read_string(stdout())
+      |  }
+      |}
+      |
+      |workflow wf_goodbye {
+      |  call goodbye
+      |}
+      |""".stripMargin
+
+  val InputFiles =
+    s"""
+      |task localize {
+      |  File inputFileFromJson
+      |  File inputFileFromCallInputs
+      |  command {
+      |    cat $${inputFileFromJson}
+      |    echo ""
+      |    cat $${inputFileFromCallInputs}
+      |  }
+      |  output {
+      |    Array[String] out = read_lines(stdout())
+      |  }
+      |
+      |  RUNTIME
+      |}
+      |
+      |workflow wf_localize {
+      |  File workflowFile
+      |  call localize { input: inputFileFromCallInputs = workflowFile }
+      |}
+      |""".stripMargin
+
+  val Sleep10 =
+    """
+      |task abort {
+      |  command {
+      |    sleep 10
+      |    echo "something after sleep"
+      |  }
+      |}
+      |
+      |workflow wf_abort {
+      |  call abort
+      |}
+      |""".stripMargin
+
+  val Scatter =
+    s"""
+      |task scattering {
+      |  Int intNumber
+      |  command {
+      |    echo $${intNumber}
+      |  }
+      |  output {
+      |    Int out = read_string(stdout())
+      |  }
+      |}
+      |
+      |workflow wf_scattering {
+      |  Array[Int] numbers = [1, 2, 3]
+      |  scatter (i in numbers) {
+      |     call scattering { input: intNumber = i }
+      |  }
+      |}
+      |""".stripMargin
+
+  val OutputProcess = {
+    """
+      |task localize {
+      |  File inputFile
+      |  command {
+      |    echo "Hello" > a
+      |    mkdir dir
+      |    echo "world" > dir/b
+      |  }
+      |  output {
+      |    File o1 = "a"
+      |    Array[File] o2 = ["a", "dir/b"]
+      |    File o3 = inputFile
+      |  }
+      |}
+      |
+      |workflow wf_localize {
+      |  call localize
+      |}
+      |""".stripMargin
+  }
+
+  val MissingOutputProcess = {
+    """
+      |task localize {
+      |  command {
+      |  }
+      |  output {
+      |    File o1 = "c"
+      |  }
+      |}
+      |
+      |workflow wf_localize {
+      |  call localize
+      |}
+      |""".stripMargin
+  }
+}

From 92d62fe1a1170c2e0da2a6e3aabbcda220ca2381 Mon Sep 17 00:00:00 2001
From: geoffjentry <geoffjentry@gmail.com>
Date: Fri, 17 Feb 2017 17:18:48 -0500
Subject: [PATCH 62/84] hmm

---
 .travis.yml                      | 2 ++
 src/bin/travis/testCentaurJes.sh | 1 -
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index cfd59b202..5f095217f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -15,6 +15,8 @@ before_cache:
   # Tricks to avoid unnecessary cache updates
   - find $HOME/.ivy2 -name "ivydata-*.properties" -delete
   - find $HOME/.sbt -name "*.lock" -delete
+before_install:
+  - openssl aes-256-cbc -K "$encrypted_e1386386891c_key" -iv "$encrypted_e1386386891c_iv" -in src/bin/travis/resources/jesConf.tar.enc -out jesConf.tar -d
 env:
   global:
     - CENTAUR_BRANCH=develop
diff --git a/src/bin/travis/testCentaurJes.sh b/src/bin/travis/testCentaurJes.sh
index 059de28b2..3cbd3c3f5 100755
--- a/src/bin/travis/testCentaurJes.sh
+++ b/src/bin/travis/testCentaurJes.sh
@@ -32,7 +32,6 @@ printTravisHeartbeat
 set -x
 
 # Unpack our credentials and such
-openssl aes-256-cbc -K "$encrypted_e1386386891c_key" -iv "$encrypted_e1386386891c_iv" -in src/bin/travis/resources/jesConf.tar.enc -out jesConf.tar -d
 tar xvf jesConf.tar
 
 # Do a bunch of crap to enable gsutil. It's possible this is overkill but it doesn't take long anyways

From fe4c39722f611385dd5d8d1a5dba5980bd100262 Mon Sep 17 00:00:00 2001
From: Thib <Horneth@users.noreply.github.com>
Date: Sat, 18 Feb 2017 15:48:19 -0500
Subject: [PATCH 63/84] fail file hashing if the file does not exist (#2006)

---
 .../backend/impl/sfs/config/ConfigHashingStrategy.scala | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategy.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategy.scala
index 2d89d1267..47136bae0 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategy.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategy.scala
@@ -1,5 +1,7 @@
 package cromwell.backend.impl.sfs.config
 
+import java.io.FileNotFoundException
+
 import akka.event.LoggingAdapter
 import com.typesafe.config.Config
 import cromwell.backend.callcaching.FileHashingActor.SingleFileHashRequest
@@ -41,13 +43,14 @@ abstract class ConfigHashingStrategy {
     def usingStandardInitData(initData: StandardInitializationData) = {
       val pathBuilders = initData.workflowPaths.pathBuilders
       val file = PathFactory.buildPath(request.file.valueString, pathBuilders).followSymbolicLinks
-
-      if (checkSiblingMd5) {
-        precomputedMd5(file) match {
-          case Some(md5) => Try(md5.contentAsString)
-          case None => hash(file)
-        }
-      } else hash(file)
+      if (!file.exists) Failure(new FileNotFoundException(s"Cannot hash file $file because it can't be found")) else {
+        if (checkSiblingMd5) {
+          precomputedMd5(file) match {
+            case Some(md5) => Try(md5.contentAsString)
+            case None => hash(file)
+          }
+        } else hash(file)
+      }
     }
 
     request.initializationData match {

From aad4a2f7869376655e4331fb2291870f29e94917 Mon Sep 17 00:00:00 2001
From: Jeff Gentry <geoffjentry@gmail.com>
Date: Sun, 19 Feb 2017 14:14:41 -0500
Subject: [PATCH 64/84] Rm new (#2008)

Updating travis
---
 .travis.yml                              |   2 +-
 src/bin/travis/resources/centaur.wdl     |   3 ++-
 src/bin/travis/resources/jesConf.tar.enc | Bin 16400 -> 18448 bytes
 src/bin/travis/testCentaurJes.sh         |   2 +-
 4 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 0fe098e76..96e5ea4eb 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -16,7 +16,7 @@ before_cache:
   - find $HOME/.ivy2 -name "ivydata-*.properties" -delete
   - find $HOME/.sbt -name "*.lock" -delete
 before_install:
-  - openssl aes-256-cbc -K "$encrypted_e1386386891c_key" -iv "$encrypted_e1386386891c_iv" -in src/bin/travis/resources/jesConf.tar.enc -out jesConf.tar -d
+  - openssl aes-256-cbc -K "$encrypted_5ebd3ff04788_key" -iv "$encrypted_5ebd3ff04788_iv" -in src/bin/travis/resources/jesConf.tar.enc -out jesConf.tar -d
 env:
   global:
     - CENTAUR_BRANCH=develop
diff --git a/src/bin/travis/resources/centaur.wdl b/src/bin/travis/resources/centaur.wdl
index ddead03bb..8795c756b 100644
--- a/src/bin/travis/resources/centaur.wdl
+++ b/src/bin/travis/resources/centaur.wdl
@@ -13,7 +13,7 @@ task centaur {
         cd centaur
         git checkout ${centaur_branch}
         cd ..
-        centaur/test_cromwell.sh -j${cromwell_jar} -c${conf} -r/cromwell_root -t ${secret} -elocaldockertest -p100
+        centaur/test_cromwell.sh -j${cromwell_jar} -c${conf} -r/cromwell_root -t${secret} -elocaldockertest -p100
     >>>
 
     output {
@@ -31,3 +31,4 @@ task centaur {
 workflow centaur_workflow {
     call centaur
 }
+
diff --git a/src/bin/travis/resources/jesConf.tar.enc b/src/bin/travis/resources/jesConf.tar.enc
index a84a42c63fd0d3e15218cf77c2f27cbce9b621d8..85ec60f88a34e21463f498f3f7054f885503ea02 100644
GIT binary patch
literal 18448
zcmV(pK=8l2dP$Hv@O0(GfwJ*uJ+<t%qmpqMdciUGfX@JyE-tLY3MDn8HjC~RcXU4S
z3CWsh?;*lJW-5rkq6Am|zu3rOl4}DDDW9_e6-Db==#<ZvZ&j~3hJ@!|CKsPQQVMZ_
zoDC_jZWx2e&xI=%L!Ql)s31e+pp&_H=mJ5k=PU2xj22aud<=NIdYPb8)qte3u*W0g
zC9&B4k6cmh+jFW?1Q*@%IRbkx`Jq6EGQFR$<J$qp<R<ox_U?1dazTR3{v5R`?mRl2
z<gjuoiPC9<2rP<&7p{*~0Z$PX{sWrUoGIbbg<BIW3RxFE9KITQ2WcR|yYI#=a)Wi0
z+!h0Jgli7p4EBkki~6jY-JB2u&t-2RT+$5l){mh8sMdmbgnb^9mgzhGZ0gp1Gc4`}
z;bxb-*|~TSzu<}6WcW&Ici_><WLwNoCsSH7lKMr<^tqboZ;gieRaFgy__(tJB@J~b
z)~C*|TS^dxvd4JXCRcS|8?5kIny2^t0BGQpih0X;<?9TurO=n=^Ni8>)8`%_&R@-{
z=T!q(+Y8RGEhcvNr6A*FRL=VsCkqBb21TXO;r3xl|3vXuuh*BMy<Is$nP!ZXblt2L
zpTr&-w}TzbQ}`|_SIc>xZw>7AAciR!#sGv5eRv0FUw%V|IjaEqnIPbm|HgOP7463n
zBG-?(t$;&sMYXVh|DJ8aiX=d6J&ChaA{?niwhsK=V`L1i5p3PzKwr`tuJd1A5b%><
zayL(enT4FnV?YM>>G5q^pdMAT15#ZWjRi(zrB9PRH?d(Hyvzf9z!R!RT$ZcZV+I$x
z3QQTwKhSRk^x=j<d}sE`3aT<)l^z($KQD)0*f`wP7P_jo1)X^bP=5Bo&U1o4$w9X$
zKd@-hg0{AwQqOUU^_q}#0V9=CSIg4xxN@3j@I^q;%fs$h%7GV`1Ja;hiV9WK(j_K8
zLIF`a5M^V;V+h{r4bRBOqz?xR1Pdy;Y86&ISr*|{$y~Kk5XcZGd)-IYu8pzBsat>m
z%2~a0t1Ajwi9PoDx0p|yPmkLlxs<MG4G0e45mnVTI@(-RQt(~Gb39uWIJAi6`-;C=
z!(u39%d6H}6`5zoCJC(T;YQUM%JV%?vA&LZmR$)iy^jK<rUPQv*%54!S&|g9?>hj~
z<z}o4pHo6J&U*h1=Dj8~JLrcf9DpV-1;zI>#T0j0`tKy8oS8HpH%_6G$K>RqAHgbj
zf6(L~^vd&vD>ZE5>foJ-2eb;NMA~JWJq$_XG(dnS_m;hd@9$6PU-S3!c-BLoR~Z1D
zl#6%d9<c`3r0#K-B(^=3ri3}1>St}Lz@i_*jgU9*IiJ-p|D@ZO7g7{D;BgUqxiuuI
zMFs069e34ogyMM}HdxN<o;MPfW0>WH<;BW*<Ru#lmMtkeP}grR>|=yl5QbE0H4KuQ
zIBUJ~_n-8LI(>T?m}S6Z`<YVV;X~iwKx_uD{SAvyB5o`nujNDhk`Pn6__Gx;r}So&
z>y609V=>UwgCRYva5jRAWSyE8ju*1!AORguKvDGViF5m@|NM_0JK>1$EGU@nSKviD
z>*u&yGt(oK538Mp^7O<9NVHU(&OxJ;uE}h|zhr&`BW-K?rwlkN-5GU}YegMi^lvb}
z-{Hf5jTe9L5h)B)I*)1aAL%=>cWWs`{ajuNdtn62_9XS4iqXgwjK2GC*RupG{Wt<D
zHvLr>gZrT7ffB<}ryo1o6<?!c;q&PyVlwd|n@@E{F&h&-m5jAZZjHKQ7BGxL4KGSB
z$CXHG<_H!TKwlV^)+VyrrIASCkYTO#6Ue$BmLih&nl%-cc$Ael0_$A&P__Eu%0RX=
zdNR{n9u&eZ36M~c@)6@?={t$w$tHs1K7@bqPn#a*dSQFg0K#D-)4%p=Nb}N$2q@KU
zM;!|f1X9bSiaL5Gg92NFiZC>bqvM5)0hz;QTxBSwj=()6XzDQ&eYgvz+ouJy(H3XK
zWxrlMRmXXy>axHKKSl*wc)tg<G2mq}G!}np^iijwX})D#taH1WTMm=&JYS|RN;%~G
zM^WUlu$5{OFGlDxB?>iwdq^!iPt?-F_f#y>{Z(EtIh6faYHp_wZdgzxPQtYTabE29
zu?54;UR)+z(%28bjJaq)E58Z$6g>#2Q7yEJ?%dbcAbM&K>1o#8z$c|4(24Df8=7Vt
zI82IspwA(19HFkh>$^>p8?NQTU{;O)^Qh8IrYWi{JR6Of539ng@5VBBx(aQp%`^;|
zCK(ACGYl2utKKt<^Xl*j>(t0E2(i^wv+_J_Cavfe(S8IUEk>ULpI@){+tqhTQ*4sm
zjAZ7`dlCW1ROf;;fZ6v2fgP-;;zbm^6>;^^lVLE#Fr<S_*QA>~6zOu^W6X)VjAoQ6
zJD`flSiWh9bwOv82#pm{twF}IjjQ7s=5?<u!t`AX{?_ZJUuxv>TWw0lf3mzlmL~F=
z0QKWwur51W<7CFC>@d)@`%0D!9J;my<qkwY56#_P=O|=<2cl!z|7TS(aF;Im4tC)}
zk&%Z%klkDZ32|0|-Cuj*$D@tF#K9i^vUr>G3HLNsjFXP0)FbYwZAf4CGXVDikz!!Y
z`K6%MgPkXI(7erzX?he9n#ihj8y-+l@+CZ91^hFuNmj9!9+Cv^9$ecPmBVQZv$RlY
za=j%m0Xurbmm(@_TO7<$ZSe29qa4$2!|_~+qbW4Y2>m@m-2#CT59|2*JUHczKp$D7
zfF{`rkZNePB7Ye(8nzK5^Q5qt=B>eGnMfd{Z7OZU6FJFYK$aL<{7dko;?%q9X#Vm(
z#GX?wQ&yF)Q(#)WP!*Hno+8)cLZ;7!3mJ0>Wh_m-A7R;s&9FQUcSzL*r$cy;il??t
z&^>=_2KoVJ$VW6WmlEp>z-7>*_xc6y`eN-^CQBltLWGqd{4=Pk`44gG8ssek5J%Yc
z8X%KaTwHM#mAmpLTnznS1fRsH?453<#uXO-vimhOwgqNl3F3M-Cx|<O-{pPL&h0rY
z`IcE?)xjrG`K$iF^dh!{fM<F`!JcY36y&-6Qx0jN%}Y37y%irM&**~99>*&M;wC4M
zMU4VLu2EwZ!;-1sww(n8+K^6qKj-(%&TO{dE<JEnA!{Kscrs+A)9JP9L@07#oNH&0
z-T{Z?_1ewSy&Z&sR1d|pScN790{JnqYV$f>LdP>--!Hpc*U(y%N-XxG4(Reu=mSBw
zJg=4x60JBx>>({C{h4LJ`f&@OyxB{(Cs`4xwwpH;TiG$Np}wg5^`J!@z#pVX)FiSN
zBQcXkX469Muw{%h50aL~J&6U^bf6STxFQ`|x}m`uP}z}&U&Zaq<#i?F_J1N-pCm^{
z09klRuAsuBX}-qh7{>3CW-M5-(SGc#$`7VVX0()kq>T*(+sjL6B-<{LZmVaOUZLJ0
zc9J2f_EmK*wh-svzKhs?evrgqKPIxt;7(!$B2A<5Lgs+sciFB9@{)7<f67v#eAA!J
z^PEn!lZ6(%G(|~%`&pBksfdIBb;M)4nPu+LkHBbl?v!nyD4griPeX*}sT#V#c(BlU
zY0F))$<$ryBWnVJ$I7Fy2Myc++Rn1W76xV#Kk>M~lV2!b9t!;`(cg<4$5L9p)JQYZ
z$?R@u#Z~{z6+*O{&gvKsv!N?;5=lZ$N0Gp9%WLv58L+?ED}eIrpTx@8({<v=at1C1
zfZNh6LG=s^L)!CNmr3I;CBdKBy|koo<9#Uow?ewFn5(1~^-J6*IS2wwx<yue>*gL#
z&XJ1`3u9V5`Qq8QB;x-t{m+>0<$GV0q5cS#y!G{)<!S`Y(1vFn&O=++m(H}#uh?ua
zaHv{_AHLFKD_5B;ZjN!Bd3ayklkr}#Z%Yynw}PQf<wsH_x(-tfNTF5#9iFCQwis=S
zsCmHx6^rtl(&Gi;wFf<lyC)6Fk1zyU7Z$JPiwm_}2|>1-)2myk5QZf(UKKNI2=l<?
zCc{(%A!Ue?haaUY<B|o1mV*DXP%8Y^L`aShNCE?)rP>f0<wl@n)>GoFfI*+Ts21y>
zCWcH~`#g%!(S)WS(b(ubnNh5w53zi5K!^A4m~TOR;Rel+Fp%3X>)YieUTQwA919ls
z)0#4d`R)xHQ4cj~!6Kf6dh!%biU;fBsRpVpU6gB5IV<)%pM54)ak+}34&hUwi$QkG
z-BPdvj)5P!(S2<jvljluoQw(_j%HnmTUj1KV@fhgvfjc1n9H;NrK;D7*=ksKEThMQ
zvl}yrS$IWwVr-IMZ5eHWO`2Oly{65O2-;>x12vUaCUWY{%-pI^#SB#h9FF$u59gis
z?I2J-O{3I5THPQw=~uNk(gYd-ppHzl(yl?*<y)bN+2D&~8S>r?)wHA@L~k_T-gkXL
zw9qZ#ox0_FJTA#U#s1e)tAMi#1{$RQCPt4ujz<!OoqwK?Fpo%`JxgF9`k={D=1(3l
zw!BVO8zn=j!1lX!;}fDdC=KuizFZ24e;HHau!nV$STn}YQ$A8H3l`%1TT@4u1nGi5
z=VuXMJg^_us|l>6Bl|pCQof=08^LN5OVp`PtZTp#eDFPUwi=KisM|okIp-}=*o)%+
z&Um1<k>&}y45@p$!jv)3kCg&XY5XeE)>+nz4-0Sn7aFbw7!~bxlV1ai#8ELA6dC|3
zGY3X$%c<BI<Dtw~`?oclP#g+`!mo=2TirUaB#<oL2?nu=+;FYF9%Yy76sQd@R$q!R
z!Q5ErXa><tmSI%OFL%PoI{e0lGAPjGv5N$1Z)S^u&_#~K`&AHZ6;DFCDwF+1Nzg21
zY-t)0g7JGSp0~yv?59|QtWwwBVy2$LA)Nl)AL+#0<P=*9+|M>D6_K-BDuIOq7az4>
zN32)BuC1XyWG2t_*(4b!d6vU8g7-AWgGfTdO9=tEDCmj#Zi!lAsdB0{TpT%bh*Di)
z^`+%L2w+p(MNRY`UVK|McJiY=aOD1GN}-W+1o;oEh@r9B(<yvZz8Mz{M*4IgVlA*S
zcSFlDXuu?e<aj2iE${ush&5{PAi?y$A_0u>jI}ctJ4!vk2imEZWz)~CgO}s@G)Z!z
zab#`=)1L$TQx^fY@vseuL2;oO{|~PfUzxmodB|YxHt2Bw=@iGwd`9=#Nko`UrX|40
zbH1)p!hyswq&tayL=krlfZuK)g=!+ne@AQ^>s`+C8^Mr?FWNe)FC6>kwLZR)v6h<S
z^}}Qj4q%^1#MiysPS3dQ>fp0<4=FKl_bF>W{y*-)QVpC5k=tU_x_7<}16NS&+mv8M
zGaJ-=3dt6*f}^hKy(1E>!nN(qn8XmK5z%D{a_Fath%T^o0jBQFalT3*QHEYuiR4uB
zPRf%%0Vn&0U@%!Or@aJ&thQ;i04Uc;Qg=PhO@)YKCXg|8#moi(#K<^Ow5RCj)4;dE
zKz|@-lrN)*`++S_!;KP+<CWTp=AX1&+V6adO(ZHmAJPuH&H`rCbq>v-DOKv&%+v2j
zv2Rb@8RUqE4Zm9zoL<k0o%>I|>px&~3{W<fa3F`-dS#g?PK}hqLvv4g@M}m;zJ9wI
z6H`D#WboycF*3OBxcMv&s#=7HD4s-}|6_O2z&xy8*T|vS*o*8^j0|li_9(Yq^)&xz
zz(XW|p-h8umDgiX((%6YOgEYauj8|xY})z#P(t`44t>hZU|;cgs}k&}XYc@QyZ|cj
z<sm<kwHM9ur~l*a7(KR_IjgnX&$Qt19yJ@m?2EF;aC6%=Arq#_`n06pVOZ(i-O<3U
znm&22-%Q;}#}JrKVf9+c^_?Gj$P8L<*6Isc2H;(_m(<>c(4f~u8W>Vcu?BI3xA!T8
zLpDkdtQwSs08KgVw7<K7T^swLi-E>ydLP&&P)sabqF2<Pyl|?n;y?{pGolWm*++0{
zXJJ^XY^Iy&sswK<L=hhdwANX!UuVYROFqX~>S|DCCQnusyVO(iTNrisjMr<QJ+&d`
z5aQdKwlbr^Ucxd-VqvE&mMInbz@K>fHlCfh!mI$1lhV>7wGplT1<@gmPI>HRTU&?T
zy8u%b;gDsk{ko&2AAnQb+LveH1Wb)N$|H0+;=nY=&3G;Hjp<=>sKSZ3fhPuH8bm2C
z*$VPTR$@;0lP2W&Ju;I#q1YQEP}qb3Vpd$2=|>9m8#kgXn^&$rFL)$4LT$s9oh}LI
z@ijgpcaLhcBGo;N{;aL}n&J12(RNd1;lFgpAIE{y`^ruO&<jrJ+gH)ICB`K9E&2mM
z<>M#b_^V&;6H0N9{mA=Qv6r&mXfg5{S~VDeU`nD{6wsq-)qjPvo2a`HjUW69b<g%@
zDHAlo5#e##=WACf^uxW5bgb9IhFA)r2;Zdy7_0+?0;EL_3v#c`+9}y91s8{y_GJ34
zTmvc)GW|d4cDi|<%tfy1tUO|n=WTb^95-D+tgBr^A*=}7`?jk)b93DihDe8abvQYk
z<>0ThGTn#{9~wrA5FrGWo1g&Vhq_}%Yc1-HG)ziB6KDbLL$rJjzf{q7U5K<kSRE=t
z5wXubWYo|4nB~^>Pk-^&EJSdZ`ve|=@Vkg8?UAJeU?Y%oG4Z!ON;^cVHey{fx6Qa_
zqk+cSlQ&-nd!>qBM7@4Hs<@$0B^SjqnFX@`0Awjd)~QP$M%)*42v=mO2b7I;=4AGY
zJHgE5oR3m5<0zpx$7K7kH<i&-OgzZn>&aL~S-9MG3`D`6_)_)uKmIxxp;=Scbi_QQ
zKRnDC0On@g-}>>%SypS5cI?e`O0_}js$i#y^Y!>@a^g238v_0YaX1>!t}2LiY?|q}
z|6?8=+g~@(C$;AsNIFv%TNMK@U;lM3=6Wl@g^)<Cug@4-ISCVgA)J@R2mU9KXmCKU
zQuIMirT5=5%C!Jh3uJM0bLG-O21TuQCH7veQCQ!*SivFs#grJLZ)$e*&jdLbt&Zze
z^~5=uu7Z(!A-=(S!E9Rk{KXyEbrBh2WXQzJ{5s9okRN1R1o^@!=e^^f(keLy78)8N
z-$)XK`?A8QQbpJw^?KuJ6Ej+7>sfojLw~7;Gss5coh7C|&IjwJ(@M3iGOmrB3Nux&
z8bn_@m@ai9YDmnUx<k41YSc%AtKz$G>ce$%^WZ6kM87rXvpV&8n&YqgDp%W=hj{nt
zlfcP;dfTw7Kny-L16a)l(x~IiMx*rFh^z1djgx#y1upedpyoL>!)}d>MYw^y(_;AJ
zMbhA{!xXcOgu8$%o##U%>|D0;*;EItjU`8H5xS0y3(43VElkjw!kBgnzk>~w-v`s{
zJ$eL3>Z}T-?{_xz$<6|7DK`w~DJs$`9rPEa%cU2tXT*`XVA`U2;5ZH(1X5@P1~!cp
z6qF)$NwEfS=!6-)l8RFyLgWms7lG2+BlA2`Z4vSLL09VpN)F=80NWrv!7LUOy=gYI
zGnhMZjtkKv-kA`=l?57)QcZlH@z{Er15hRe5RUH|wx*?9ohfAaJ#WeqHCUfhP$?D3
zKT4LnAuQ%uZ!yAX@$|PxmpT_wIWFmV=7CQ8_hzbgC~CA4H?R;VvVY;G-B|Jm%5wvR
z3h-uZPqqIz7m9)$xx1zfqH&k<kGV&G)WaFXdA3EmAe<0ug!yJqK>BvCqk4q;U<s?p
zTUE2)ER+7<zD_GhdgwL>-n_b9TF@#r@diJIMbU6kWATgNh^B(|Knc-Oa7WG4j2sID
z+t%g+5pd(K?U2inqz6b?Ed3&V+0S<NL8MITb2NV%J^I7B%%-?nRiYuXp5}{EV8G}4
zizpPwzhugU!PRWbXIRoo8cef$2gFzQ<RL3!)ILqw2;%Wb#GAj4)|}%`YL{awu_*%G
z>-PjlO83_XNhfHm_l=Gl3XBvdcC5QUsVj}=*KA`34s#(UpXo<6qMnXLC{)^=J686<
z9+Yd?)yAHu`%UUa)JQ_A0h`VmS8PJ&FVw4}bZdPfKaXnQRNg84`!>=T*<9Z0gC0(1
zfb{`WM<iy`YP9moTBGEufj(kL-%d=2rA;3vrpk@1c`ZQzHffgdbHUrP$GI|}!dn;W
zLMfIV?Ji+kMcGS->z@lhnvuo}v}yc}0Au=Smi<A8?m|!y@6NRJp3Y(Qpw!LC<)Pc4
zSg6-xJMK<%`c1LmWPDv<VC?U9i!hk7!?~9%Q#f0gX4HtRI&Fz4SQQIy{Df^H7%(0x
z%9I2ekVI1Qil83aleqiMyNu~4Met?GDZj3d*8F4wJ4>&FiRH?Ai)`EFdZS<jAy)G%
zlmKn$^wYpb5>6oLbYx)qAlE7EnYZ2Z!4L%G6CfJhY1FADD60<aoRh?~z>oiVX3_q)
zk^)A*rjl0uW0BRji7+F*=*iQt53HancNm1`Eg$fJ+{;I<dZGeFSy7qobZ~j4IQQiT
z7riA!c)iXlgJPS(@nZ)|cP&$+2okm!IFxZGE2h<VeXS=Yl(z2?lIs&Ev1C2Rg8hW8
z3vIbHD;37@VhnL96xl9;EOlkx+n4|mrp=rbf3}6pTVXhx!Uc>lB`9-$$*n%g?7RNM
z%l-OBh(m!&;RT1kv)t!{2kmP@No0xFFQ+-ji6C5}s1)3wHVg$YPfFlRZ$QcC+fE9Q
zM5mw&Ep*(jO&hg?)qzR7^G~`L+GYBt(?jCj=si#!5p8AKN+(%ut>{@u%N12jfNx-(
zdPl#sTncO!FGXi-jePYnm7Paa5E)MocRQZ&D<r+nFkk2v-rg)&K*?Y}o>6hfwh+m{
z)LXsVUcT8p{iBAHDLX^9tm9l`BV$=+DGNN*hy6fNfE)>Qhe6evpC{)sSwlyVM1Csk
z6`Ok@OcY?-OhV3M)X6l;JYjh)v=Wqq>@I$&ZWoS0QV(aN`}X6<ph;>@Rc1>c6$D$a
z3@liZi%xBrkvdsC{ROFYt^qyIGmxa!V+mu-q{T53m;>wr0z2f%guMmahIdHD)Xbv`
zS;17BNFRCnXPoNHV4j@Uk?sFV-Yt256F~=dXaGSzL&87GZ%=+hylR#lG(vzzz^4CU
zc=ovW{hWDOoT@ZLIS*C9i|YV|K~a<$PX`OIw=u<dgToA1A-c6Kp2-%e|Lw1;g~hLP
zyaZ#n!`{_X7KiA#3ipMcKf>F1Z%7|Z*?N^r>LIICO2l*C=9w7SEJU$4Lnq!%Ree6l
z`hE?)E;KlNyZ{p0pu4w!*;CAW-Tt_H`NZ9JynR3Jn~R|bA1ci!eR+2ie-8kupm#eE
zzg$9RE`FF`S2O&?8@jEAiTx^q#(yxNN@T$yT%$EG%#v&^^;5B%OvU<G(tR)}4asYS
zEKa@#l4LZrw$G4FK>M3s#+L~~Dk!g(1Uk-ykwMuhD-nKG4n1um6+O|j7Bl>hgRN7<
zL~W6U9FaG3FZ|N)9^6MND=53fa#-?3_=ODhx}BoI08h@xBg2fcRwlJkegfcy1kU7l
zCBn#BTlHJ*%W#t^B^O`f&ghb^yH$EE<d)X;<MyuTy4S4G{amNRm@{F}U`BjJxXgR?
z9fP81Ehl#&_y@=(FX@+I6GeUCKh)`z=lJ{Ma(R${ewq~F)6i|wIV0vo%}f1jRmI~R
z*I#92!BkRgwW3MRUUGgc6`3`Up}P>?u{d;v${+`C_5d|;b%S%awS<7025?Rs)Hsd-
zlMcE7A?L$So9)ziTSphdaY+;w+ocs@Z%5~^>U-+Z-N0RJ^WkiltFfJMZwTBr=6+Am
zGTZNM8+tO^ls>P?QQve?8bgv9?w0;I_{LEoeOeuTL>D4aaW;#?*=qhVKSkc#$ceEJ
zcGE@HHW<HeJ6E<+yfO8LxABjs)cuxRM_J;5pNYm*dD}-3{#>U|r~Z=)EUELq7`Ok(
z#5S<LJXhZdJB}fXx4iXUe*PWm(IZg#59K5RoUQ_?#S|v!zPjvQpaEkRek@2d&hxKp
z?pOdpE&$02C)<|y`djod^+@y(lKj(YX`d$nQVnJRf({;qadwqxgBmc8>l}tOse+C-
z{0NLwl*<~R76mcpBd~QHBdn+eNFO1htNZl5t=Kd}`n?~cKHH}O&Lc&H6Y2+I+4ORQ
zc#CCc-`3b!z6@a;*0U}a#XEnNa@dpcUdB!YNphY0HD^emamb7+FLgE*ZSk78D=f>w
zkUK~;j?1C$oNW{dVm*?)okUJ<e1~qn8b@$SwMwx;AIPQuf}fIZ^+82e71sSNlX;F7
zv{)@TqQy@qE``|0;(eV?z>avr2H-AX7Rq@Pab?kqD`-xHj|@if-&Ba-xjI@RJ1Etd
z4m$-vpf8Vu5V6KWpY$Xk6Ib^3_!#raxV=b90BBcA?N;b<3J4>Su28QWW{N0z7;_4t
zev8I|J+Fua)Mr`?Rm~}hYj+46#DRH_lz5-1hu>uiT-<2IQmS!aOz@<3AAKYm%g^CQ
zPCX7Ri}`~U*2nF^HLg99)zsXu`lXr=P>43{z0{y`t#{#h4slxZGzhp`i7JcB1PZUH
z9V4q8f`hUy*NrhyO6T2;nYU(8acJ3I6->(l(|3?K)70uw_xEo%h<d_Y-29zahyWo5
z2Q)Y|m1UefuC@$9uphv;r5vPXvmgf_#%6OA(b1VxYr3Mrdc94YTL$3Hub?>KgzUpg
zt873q&?OK+15=<V39Och7u1Ao6sVBBj1K77Kz3szsq$lD#ZFuZgqT1xD9|)xt`4f~
z!;lD!9^}+iDOC!DC}S~PN@NK+ON61QKf<l<xr(5u$!%ITY?Z+Q69EC@M`|!*jsFvb
z5PqHKWA(8AzyYj&`zkPAy3z7wSJX6e{iQ}f3P??L(%!M(2vBc8;ZobUu^2N<!(8yO
zn%UZ?X<}db!_Obd(2C!%=M*-)!rI>$GsjMlDif^_|3>`VoZ>H|Zn3xoqjrF<Emw)H
zCk3k8W1Jc8yjKo45?$pRi$P}-xI?&(>C~9<;<v!(4WOuQbluIk;XO}2p^Y%L2U(-T
z%Iqf9oLIw{kkXkjaF^n-A&0_~Ff8bx*fy243<#bqxS(Q^=R;AVXY<c;uQlZ4P<&v^
zU!(>L6wHTkUp`}Lb4TKgVf#u7#W$QbEMGlb?LnKBD*Ypt;z3{*Q)IK74kNLWxYBXd
zg@6u9wpY><C6iNb&zE5n<LL)|QZgfup6KZXZ)3;311D?<Y@h$r&yfPF|3|$XHon~B
z`~(_@h}CBWLs#cXiT9X19b69#pQY@}!m%?@%H4V9DG4_@9EgtJV2KBfe0&$|6}v8z
zIFHIbLGd4ujGu<_bhF3~-jbJgT2MZa0S)W!XJN!>9f$yWA-HLQVD?Z5<5g@77M_u_
zr#<WmE8I`-S6!1Q=?8!LeWD^j)%eKAj7V#33?t6F(|B99!E%SOpx{gU_;sWVoXJwp
zrQeY02+h}K307?~lPgB#nL6N(w1)IRAKK>&@ebr|<r)0+ZJqJ$yN+&r<xD(|Axsuy
zL+5iIIUmVB77OiB$v@&GPYvtzAlJdWLLRc__mqiVHeCCQiqD6kff+sYkn;@YqplSR
z-#AL2+?XALluWZ?gi428%Bz9Ap@@DZ1+<im!Bq^_@b`G96Q_3@^-l(l;fOKvqo;OS
zBaP_jIcC<>7=21ZP%5m^EEFGB21FY!mwEmCR4sT78nh4iW$~tZ_@AI`cyU57X)Czs
zs}9|f<VxXWDYeg4>)JT6DGq#$9NT#$Hx`pK{e6b+`D;nHR<jocFkZy_<U1i=p4>Wg
z9eisgST0c#xV<QFTAdQgNb1uA|E4^J3+)myc%7r9fw;cFM@n!;AzBuD!f#)9x}VfT
z3p8GK`}(!~+CerI8Trg$eP7fLt<V@NY?}Q;9?|Ei)6M*4MfGU-A=$gnupKs%A0@BY
zl^C@bev)Ch98y;C%!v5B&$;L@{#5t&*o*<(K7a9{)`t=MBQ2G-R(5XA(79w?`Edk@
zBqreCnNChvD)t?xMl3`|SaeRiPefu<)?b`!vQR^!d9~$+Yk9r1J9=IzO89QF4r46p
zp)%iWi@HA0kH$LghS4SW0)0A8Bz&iT3g+jkie=779$MWSs`R_^bqR+tfd_^w={{;4
zzyo3*-quj?68gf|$+^hA|BybBFEW<0FM%tFjO4K&tPoLgqn;1T>796P+cOQo^<~L-
zY77&Hf<bp~Dx4l0mq-t+rLBqtw*!^IP||w#YpQyB-ag4_L8QKkp5S;fE+iWdLaBkQ
zUl%<p=jRl3pA4B44jPCPTb&{~xxG1qAvs7{#s@G>63@db-bUg9KDGw1d2vxg;{sDX
z3{@5VWp+vj%3YZwl-`3t-L*WK>SSi1Gz0%UQeRk6^dKtcOGwWDjz)xca{Vsq<FJQx
z3Qn&cd)F-Xj`kiB#0ta_{sJ5*bSE-oV<YzpPfMkd2=5m}-@{a!lP6sd881}{sxb5R
z#uyf$5Z=AgWHgnbjgz)Nn4iWpc2vMPX3F2=rgY6qxMngsOXatk|33Z$uqb6*T<eyv
z7c+s1Hgz+@<V0GFF)ztL$-<nbsbUp5+iO0wsJl4eTu6KjTN)25IrJdD9FJnp6`?Xi
zs#8wOBJ*ck&XW%LufR_!iCdUyWmff>P9dQfk%Gd5h>P^DtpQxzs`+QfCF!&)fRA|I
z)Tv3328b*B+(bDl4LDK;K>ky<a>8ZXUgyG$$cCb;U(o(@<>$^kgtcQk{J3J-C%95t
zVfCT?bhDu$6O>(AwqzrqMr`x<u`w?tWO+--_dU~wIn{@KFX+}WL)t%kaMaSu#F1!I
zBKPoxs@EeuBDAjqfQ0_)@iL~L@VVqN>Ki->2!t093>|Fcm`pBEot#oUs4;DvT62Nj
zm2x_-uZ{4)K3HkvX6pvmJHMg-5P?bn(E(mxU_b-GyHN`ArR#KRmF!+)$?{kyz6o23
zsB1VHL&5t}fQ!_tFv5fksntUpe0cX`;OVKAx7@fF9~v(7x>w|2)-r}tG&~Ylz?8z_
zC!_l4(@-vabz<lOev6s6<$Y)3LJCP1k;K5eix;cW=&;QnU1k6IeDZ68`E514qS6kW
z+m9Es6O$`pH!SCWtBB}Mg*R_71ypjd=c|>C9Wi6-_3o;mFG!r|$+U&9JS=Ubr3eJ>
z;-O!D60ON-G^D)Nc{C!U^5pkpxnVPnJe%bCXTA@40F)F~9Jf>9zP|-5k8Z|PdNOH>
zbiS+BAe6aW)pj{K>{M!|rf8L*p_EJ*`8)Xn^U<O+it=7~_#|@>05`RuRYaL~&0v8_
zxn64yY?U#stWKpgqHP=Nuni9A>{ASoP`K1AyEKhOZz%|Eb-Ar**ez||L+g%$N0Pgp
z6nqeg9ykLl*sJ)7AIc)NPWc)=(hTK4qgzT9-(2O-v?p%H!%Bh*_)XKr(Wou@ZCgbi
zXk@gX=0FNhHo$d+(ZOw7c0_v?A+B5XdbDARIL(SmD&+S_Cu+X~dPg%ibaV7b)%<D5
z;wh{gty*%EtB77-pe~{n)-;l^H5~|uwJFo#i?hh<yYYgG>DbP75Z=*x87kF-E9<~B
zakO*hj5kS-vJNm;df^0lB!Xz0deh7n0s0vH^VP*q4gT`wptSaz5#_}sE6}$8nn|}!
z%PEpjMXv($fu1ak8MN=aw@n4!XwYCtn;9oKmR~}uzAF%9Lvu24&cVa@n$Ek-4>&Tk
zlRu#Xbohp<{-mQ6(pU@zPrutD6Jgo4q%z>kEYIqf^BVcoq(a5NzuwQ#2>%Iv;_vop
z`zqRiw3Pt{b`qZet(Bx+oc$-rRYt<@`&vk9w}XNv?mm%je@`tKVbkMr2_2Lq#Oc<0
zgx8IzJriS_ht!cdU^79%2<`(-5k(UMLby0dds|Qw5$YqxsyF^!xIN~7;1UHaZgY;^
zWGpxPBJL{3ibX)#?dqd%vc2x~2>hL%aGP;VS5!Ask=>=JnkF|92`tRE5&;_@UISBl
zWu^xVj|US<9Mu`je$Bj1l-gJB0;0*!^q+wwH)G0Ykqmws;JmydwnqO9O40d=A;@Uj
zRv<M{vwaN-k$+Hi^pl~IkYlOF4$IC{6<X~FMft0pPcuyCUUWCSNpF2(1+>Bvh{3vA
zr{~+|*7OIeQ!axcePrxF(1b}&M%uYX#M%B5ITn_~Cy7x@eMo{t+f|q8O&<<eY2CwZ
zeni&U!meM~T6@6?2??IpjCx#;^T71xVP$$kIRh-{y;tG)9RWnt4H!S7@lzG4it{=~
z#h`+O91bh8qe;feQTP&SvD5r%oI|Fuf(+Z+=Z*?$<AZhq2`#4;5IosW%wQP&g!2U6
zwI+TW&Y3;Q@-tw;QBKv@bC4so%t|wJFlvZ0#t7b3JrtqYCxkP?{K<w<6#bD+prq<g
z2fi-z<KIBfFyi~4A!=Ej&*WK8tqRYrnZm*s6hP~J$%bU)<RGOP^#=x%a_b}dbr1Q}
z@?&Did3%H`Pn7&J)j!1ikg{u(W3t#hU8|z~hG{PT7fdi!+6?X;k-p7yRBj|6`K`kl
zHWuj>)@O%QrAan)aJRtAr7VB}xYIpeMH7^<NF2Eb_}&rYyM$<%tokRvKfoqx^<SRo
zX~D<$j*6@zvxW56Ev{W27p!8rAA9k9$Q7_uc&l|=R3c+x7}#1$Nby<n*f(Pc?Id2D
zgu<qSRg_n+${7JBvsTtn5o8-jx@ye|1`9!E*hQ(f_Fs$6DabTgSOB+bkbr4xUYI50
z2tM;am!}N75l;VOHh&ik8m)sD0VO3%;JE8=24$*2Rz5rNcC(YXTrSqpR595lgiqi!
z?(xE|W6kO=^MUg@sP~-Xz7o9LYw_i;2mU5;GYnmYcGou*>6|3SfDJ~BGdkM^pTg}j
z)#G_>X8)M|sEDyN*%oqF733y%We)4f!jkwo>M6XQ#J$<l7*G{6iBqyr^dZ8C`(lyE
z?b%~Z@(gj65bWw_Ei_gDw!PQbX7u<mJWsB^?IDx=VQfqBH57r2t%0C&;vq?%x=#38
zV3zd^Xd6pdEo9``<MWHc%_$;%bA&W?j(bGMr!ROllRTv*@@QPYV_gLXTdu^_>f$fD
zj`&H@ZM=2PZK45wM{wNL$}ic}q7_X3F_<S&!d*b`)opmeGD8~3bN)#A8BGB=p#Szd
zj!3xeM?Kmh?Q<B;kv4~;)gY3El_+c&w2LFhY=YTjQ3%81Bg%hh@L8fMv@%AoWcEM4
z>}&|YpMxRt*BW)s7>PriVLH_rLd{{C>OjBK0x_h-)O$x-0=|R1PC=rddfnn*BN<XM
zsT)wPL~ZOhulOVsdidQv%B-+VNnVqLuYCz+??i_9q?WYoYMxIuoKB=n_66xiKW1!x
zc9_e_J}>pWFC?6@^<#%6nAnKf_zJ+9CnXK_B)36WFgz)l9^we-!KMRyRmdG`dywI0
zhRp?cF)8Gw#T!zFY#AQj!N_K8Z_52Iiiu!gI7kz>lSzb{>#U8HsV*@mwTQ9O_mCwg
z4gOW8+3x<i*}u`ZWC4@3@{swMzyjV2BM?7h_i>?^ZHz_ji&E}Kz8~*0@IC%Z{hcu}
ztE>PFgdp2xwNvIH5vEFe`P+7_n^7Npgt%_Odbu(ZJ+IS9;OAnc#Wq9-rg`xqw(6w%
ze6bu(sO=i^j-|%bsY6_nu(nyaP=l78QT|-_|IZDL|2%vWQmVDo3k*G?nB=MwG1|H!
zz}3@f=l87l`{T?<(RqKBkY?p!C!Z-@dZr2mrYOaQ(TfW*0)!dmFSsIylAUn1zT^kB
zNS_AY0$rFW+4A{GlDv~0l&Gh|?hl3f_TfQPa(~SxA0p0MZ$bY6g4#>x5}@;<nyt+j
zvbRme*eVTfUr;eQ!?#11S!an_hL+;a)g{(e3);D#=>MiZ^d=q)frXrM^jxF9=n50M
zB7Nyci`B8XsLib`Yj#zG-UY-7hXkN)$pMtcFa$t&l7yDl{Sb{2pn@nxi!#CYBy>~%
zU%VEoYWv^}A%@yzyz4+6zD)~)Gt!81m-p8lqp%$#tw0#kJN?4c4fYxNI3BgAvhu9=
zP1P8pag6!fZLtvAT%b)A0WH0eoi2Rp6-*+%LQ;Iv2biGZ0}Y|!C0cHR>gmyo^MqR2
zo_3n-THa*oMPO29YjUH;WO5zVk&gu5jPG{!=j<CCi>nb$El{WJ_(N;5u4%S^49tDX
zPPV4AdJC`s^TcMF%_+w++IH^s>CbLtU-_2Qzm9anj61NU+K{W~VaW94CN2_L->UHs
z3f#z~U0!#M;Z)haOoBQr@Fy9!c(_XkR0T8124=dqojqCoR}vg406!RUBaAwuuqJof
z|F{P}`!MsQr2HS~{QeHrvtUM~vd(m3D!#bIb@*pk*~Fx{l>XTZD`*D)*;}BN!a%XI
zTB;Xn3U56QzxEuz38>3rZeD&Nc_3o84V~xlk3IM2n^<NL%Y)6`BS^+AHDG$lBYc-L
zbf5tW!CQc$4ADt<^~Z=xZ|F_4reH^Wy0c9y<W7$|bceQougir6q)atvJw9xWK-rn!
zLB(NORk>AMO?W=U(vHNbl#84Wq$Ne=8ji`$S);diq}6*iQ6cwDvABy!g(8|BAPMkq
z)uqu)nU0ozWG=;-N=_G#Ryx4zLNCh6DJaKT`k*=KhAb`;xX~Jb<miG}wg<-W;+6YC
zXm!xAfh28v^^HS>A$_0ykWA)wabm9IF(UFm6W0%rmbplpT37f$=+?kQi>IH)yAFj<
z?STcoly``TD@Eh!dH%KJ4n)5?df{Ba>DLN$D^TRLC3l6Vgb*qlc^Ti|N{FtR{;>U0
zqNegDoW1G&{dbPcuHUv2M!qE<twKe3)8G@GztHl@mAi?1jxD7IX~Vj;ZLjcN7mVNU
zn}m+cF05iM092V9a;p$0=Qh68jTOyfU?@u3Ik3jK2HcdgtYi#flZIdjGX!fSODq^n
z;>OkOFVcNN89z~eHAd+|eauWai0Lg6f`zc=F_-414^t`g2UMCQ&vMHr9Phn&ZH6l~
zS}EDcJ6dj4tuBm}JO*N9B0A-(x$pJ53>#A$7Bs!9RM}_VrUSYPLCw)9xNKLy^cPWC
z39NQ!hQ-UMxaO1+WQvYcv6fxJbIYSIM^o)*u0gMp#WpHcSLj6X%;mJ2<)-kg`h4)V
z+0=G)#Z3lP)yE;oP=q3mT0AD$PFQ3)k%|=&P&T8eHZW*3C+1SJ`L9Nb<~`V2a`XcC
zvXrZWSxCgCEPbAZBsMy0)y}nAJ_1w8WAmAEd00kqyLH9iHpK9gsPeI`yVK~Ly!D23
zNBnh>@C472SF+Y*NY?x(J{rG0N3EK0s(7z99`cQey$@UeHks>Raak$zvY<h1XLTIe
znzfCL?FgJF9&5?r5q#YKCUAB5{ic`aeIhn~kTD+kbT6K&%XXCXxC=p`KDVqR_qd+`
z)4i_^#Fg*xpfJR9@)gTF+7>OB9VbPPg{Uk4VK7j8MEI$IkbrhDj&IWV%?S)h7q8wy
zQz*J9#E6$4{&UhTT{o+B7!%3L5{?`eN#UEsp)9^d2%CBB3(EPqsr}#((+Xa<K!_i;
zY>e&^5tVb@1#yRag;)4TLm@M(y_!G#y#|+P#JhgL6OaS`6V%qQ?nCi)iKxfwXHs(W
zf4=v9l5b?cnGyFY^vt<QmTLDC>3N7v5Cn7&tq*ZoERhw?d^Uw@@GI~GBtdQ2HR#dy
z_chMWuocyk4yE>=E4mhceS;03_)z%5@KBFB1wa@Rp3B*fdzuo<BmDqQWELCE69Xiy
zR`CnyMQ2GSw?{v#&(qN=4ug=999g%bfN_7xhc;60J$BxsyrL~uor})x@#p6A^cXB}
zhj5l9J%{7rkHqZ+`*%by|F@gzGS~a<gSmNr%#2Un$?=NDs1+>bzLTM(G|Sh9h|h>c
zP<@ajj<|rAK+0G%yNd_Wpf3xG|ERuprf{1MZ0V@VL9*#_w(czSlI_K}PIrx1f`3PE
zEc!@COce)l?TX?nH>^ygxj-z}3v}_Z0;Xr9xByid5Va5u^Fy0?9hi@fg>H=N-^*^B
zl~)nYsz1!28xp5SwDwDyI;#L!*))YCwx3Yg&@lCSoRKIIdxg~a0E1{w^U&TtcU2Pp
zN9qlY<`~G+;e8~}oLb2qv)ewNNqq>|+fEz!gn5`c+FE4%wr0p}V1^~Rrr(^Ou6H=(
zY_G|D)f<bW>;1P^+tI%cC?lU#mluc@5-Hc`BYn*&aT~rdym{D44nozpY<xe82!Bif
zlwr4{b@E`55u8kB>m1?2-{+P430Y6yhG1W@k&N##)2|qT0>f9~``h2;#u0?q6J^(X
z*oUcVq@G#R;Kz$8IP>tt&@{XO=gh`A**Kq#*em2q@9c2dslj%kf}_PN|IN+IT}RfO
zP;!#k%c^6GC8%!@1CF2NnHz1;0uKtstRN1Jl1nRX5MM9TgGWQ5qJzO`TfT;b)puPe
zPR>7m8lFwU9%mX%#3J;WjY6Rh?lM!2c1p+6c(3G(R*1{MHjH|663!qjtV%U?(Uts1
z+9O`*;g!jgii8bDy5_{2z=#>_t!tp6#5WN6P+xf!E^lFamSqtHWo#j%FT(9PUl1b%
zd)O!uL@#2SKm`*#O5qE<V;|Bdu;Et?aH|IRw+ewO=O_|0PlugeMTda5xFt<%4y#)Y
zO-#=?%s;%6VcdKx4mbxYu)7yIo!if6n}pEyIdi_wof1af;aNW%{(ee$+}qeAA2r>Y
zS;3Vo%n=LzTvU<E7%y;c<)rC<US^%Y4REG`X|0nL$~}-fK07ds<s#-sx@hRpX9TOk
zElT?}s;xy$D5?M%)zWx%pEO@9&-m6c9n>_rrS)0VY|P*va#9i@s61cew+O$O90G@<
zLQ$B1l_bNAo1<%`R9@m&9RWk;OM%P`=o-kjVt7Gub+Y*gRejL6Q)==;_X6(z`+M^H
z>x;b$ZJ?3L+>om^jDnD*4i`DZ;1W2z_<W#Edy*5k9j#Ja(k{+=m-iBEc3<v;W)i1o
zoQNxey=}y9*4%9ZPa|5%2us-clr24SkL^}l^Qr@ESht?%#*Jp^oq465=gX0Xw5a?z
z*M5d+|H@KiC>ItO34?EV&Z_69U&L$fF$S6SF){HrHH7rBVB@XAqR(rmH}5et;rOow
zsl-MZWbDXnSJCcZd#mI~(|c>CC$PQSKmBr_8LU>*tPlEvNbzz^_aUUBy#Pk~wb@^d
zoe{Nx4JUCb;{Vo;$zGWz=vwiMHFVd-WjC{?Fq0kIcj}vYtdG?n^v0>uF_XNHp#?1>
z3F-G>LhTD!Z5qF5C~cs{sT`pK^R^=O`QVR1l3|-B*l8a=Cn=0gu20J?mqva6(hseV
z&6{GFGCs3cj!36ee<GqxkMBQJYq2R6(K2AfIcZAxWCiFR3w!hr)9m7p_Y+&uJ@9=-
za>N?j>~aeFteE?9k6o&3pAb#hBn##(6`+H}XNHw|o^X>X&~wKXr<>6!xz@?RWR)gJ
z;PY4vXV4lBhKjaWicb5Y=!?60;w3zH8AanH2ML1mPW1^{Sn02yQ}HAPNzXic=ftZz
zb;7|pc?kkGgKg=UQhTi-70>cBxhD2}YME*&<rqZk$D~$;%kt8h2CACUN)#1HCHmOH
z`7<b=#5^`?<?4?AZ{1ewF@+jnkb<#J?p}>f|Ipo-<;r2c2pJTm2^rzsRxS&t;~c+>
zZ$dGiei=2da!2q|DTg}_^KXH!Km4mPZE!mmT>EWvT`?kEv`X%gtF~FP^IlDPZ7~}l
z%7xL@OyY#}dt9!Yb(gtyn{d`%Kau;Qd-rB@h!1Uby%7_CT_QMf_E2qBfCe{W&@-ZN
z4lvZDe00n*&ULe8tX0v#y?fMXKw&rIN-xivJ~C!m1B*+dqmRG@0nzqB0!x{lEYbbv
z%k&lhi!ewwZ79rO+B!{u`V~*76=3j!6=#W&B=j_s`{jW}E0a{6ET}Rqml`HH5)!aX
zDChk<PQTPNmFR(cLka)k-q<$U8*jn7Vk2U4AJ#W9Y-0bqt=Z~jS%-uYIq--K_?JdJ
z3n6iG$ZCp%_T0td!9V6xtgG6Y(~37iX{_Z&72i6NmdSTbY3L>ttm+(}_x50`gVan<
zr8`T7#zJsg2Kpio#OLf)^cDndrCRJtu)$j;)qS;T%wgGgnX@54k=rs4ldE+wBdFN;
zyf`Jy*+Ncmcjkih7^0Cq0B<vGqh=jmK{Lq@M1?e5X8^eQV?$_6GWtq1g}1<PgVG^N
z{QVRq4|^txAS=4W6jG#J)2-U<$I*F@jtz(q!R0nkHeWFZF7fqE(C?NV<u^FXQZaO)
zAN0sssr%V2O}<pE<$VY5%j5x?K$!hG6<e**ovrVJ2CUBg3(xEp;--yymD&w^)dlnD
zLP6Tw&<y_!uP3gn(`^Axo_%>A{_w;M8DW$jT<wprFQUP9yNSbJ++}dL_SMhXIUX`-
z`_AeS!R2o=#R#k+O3&*3WK*}`-VyhGZ)3$Oc}393d6_bU2xtvDv*hM(uP%^D?tk79
zw=A+PpSx#$bIqSla_6wZ4Tq}@4L6qQZB}myi3=}&pwSrYdcAMH<?y6wV-0%Pn^&J4
zspdCN1ENcrH^aOsfkJLKg`y&+g(f0Pk@aT(ff4Lma&5Cy;96TEqZU_bN6+KVPhwCJ
zCn9gr9fV6Fz}08s8IP2!W$pQsOAKE93EH%-Egn&%mE&O?JFF%W0zp-ul(46ry^n*Y
zz?S9GP1{*Web)INAQCeguy{{s@#)oVttsROx0R2pD*xL%8B2pR+jV-kB=MakiW!XW
zR2SY<liAHPal4fZ>E7g_un)ynk<_3}E$cjGH>O+<Qg4u&kqXRBq5v}M3m%leYb!s>
z;;77VSx}x0172;Q)5+d@C~yGdQWSNX7?g%i7>Psm4LEY>mdYqFKm)^A*S>@X89EBP
z{sFN0(Z+u0?t?fWXd(jM%!H}CU<2i6dzNi&xp1zR63WVMqrrhEIPZcEhdPRgU)n)K
zVBu>9=dii5&J72p-)$aF2`s6gxjO5xGQayXDi6WdOHQmo2t14DSi@<S<MO+e%Djmi
zcn1V1n3w+H?9qi(YDn)Mr@{iD)8NKgX$hRNI6=7;IObBQnI5A)>IKM3rmLf#Gc305
zRqg5^8!n0d3&GkcrrT7?t-Za5mt*TR)hn>tJ+s+Q=S?dA=Y_~VMMY4JC)FE|uTP1|
zOG7(1><6Xpy6-1XUz2QAIN!R<uVbE-R6SDRyS@*D{nPRIcuC}psKYxe_SOT(IkJMF
z=5Y~K-$>j*^L0?81uV&K_F`gmJk5#m^x-;%56C{v*HZsPGSrf>f3Vvh$F<z-RNMB<
zYay8CC%n}L`cROOB<{^1B#_CPfCX~k>x=WhYq9{v^l962=gYt>h(!8o`Q9j<dpVfz
z?>u{-qT5-=-9@EdA~}wCyk@6Dmda5>q6jD3z5}eOMuCZPz*%wH_H!)v4We#^O{5Ag
z6KGcZvI8*3eBpLvt&hZDiDO<<t06wc;jV!pruQBO?;u1WBihMHhP5j_%r4@S;mEzb
z-zMg&W9r)#FsAZ!pp={63;K~y`r0&s6aQpgp(DJVbI8%N>^6_$!keAgX)+0!DdCeW
z_4Z?#d!4H2(cxK?M{v%p%3V@ZH><c0*dt2&F@t#u+#%{jYlZ{tZy0JzvJ2^@nCI`n
z&Y5TH(0q!WhB|;GOp@Ay)o&eY0H%P>kxWp<2b+M*ptwbLFzEKL#bFF|Do(J2VAd-G
z&pvG+u?SG5CK<czwK4*Wa9_o{zcs$vQd9u}2>$mqADVC3FqiIsL*Gvn!LnArkvF)B
z5%2vA`fYwQ3kP{12UMo|^%;PT{MjrH;(|_S2iDL~W(qZHJ#hRALEd)8!VB7l+uk=S
zc5z_}8%E=|H1hb6FYG?(*1lCuo6S(m23r=HbhU`IsosN|B(z(J`@#bQB7)I;_TZfU
zLd*yZGHb1dSlTqaMXRJ0pTE=WU96-P<lR{xs-VIr&W{54P>aPopg`{Z_79>Bkd-)M
z_ZQr$#MkpYEtF#ld%`8$(C+iUt+5D+DRFaLo<rD_b+fQ@U)=ylTKn!eIPt({FfyBF
z2j|zA7@7yMv;9EPy~evb?4<=(6hvqXIDF2uDmf-)IC>ounXa#xQYK#zJ=?&+beT5l
z(Wf9r_?hyutMeO;nx<sxElp{I@QwPJiPSsod=qY#A2SY%-NlQ>&|7Su(#8XqaT7^=
zQ{1uDR54>PS^kBJKN7GQwG;M6Rawf7Pa6<qn0U%c*TZ%E>r87E#{Ko{)>f1D)9?XW
z4p>!K8H?KtKrwrmVebUtic*XkQwvd4kTpW2aQg~ji6j&oRvy`z=~d#NhEtSqQH?af
zHI+;)nyPuM?QLj&>@&umEpx1$aoYzKe-dRazMB9Jd>DEMJrob?K|V*ax`0&6Y8lpz
z0-1%u+<Q4oRHN@+FNm<G^$l$?+7jTy>REK=j}-H5x8Ssf&i24-HyNbCtF^ihv5ZO>
zbudrlX<##G0NN`Tp#Z>nzp^Q&9!AHgzp<DFKR}UCf-0w#aoGp{&$TD{Q?(Df=@-2L
zp#z0-my<4ClD2`wC7i|Ng)*2=8}%Rkd(I>u6=FB^?jXdc{pQ1z|G7-U$5VwW&?{tW
zEd&0iE4zQMYl=fY|0lZe7;%twSQNAG8x!B+3EQ}L<jQRJ!KzQGE&Ru7dC%wLdp_(E
z!fJrx@SO*pfLoaxx7$E-Q_UER%FPmGFz_!n7w#byO5>EfC^am%^QvB?>>mIOmowaQ
zs|dpP9ZXxA{y_|(%x?`b;iHV{`ajK%bL7l{?8jpfi)g|>eLOaM)X#6mIZM9qpJVR`
z3*5M}2zy#|7I#?l29rva-c2OKZ*=wLxh;$XfWu`YFkT&5Jt)CWPO3Ylc6R!@Qxz-c
zE`4B--SYxLNlDyOZIn{mJ+Uu&i%jzUZyz`O5#<?XBwz(PzP&>l%gd-#P6Kc!U>sM9
z#SE^#xWKuId~sO+A<CR21_PfT98Ernya$&yW3a{}D&5r$(>ybb>Jyo}WEkW-f3{LN
zQ3PDJ>&hboWXih;9E*!1@Q-1zUwSA~SzJ+ksSd94A3B}n8LX+{ne$dr&5^wKvKT9Z
zsd$a~v>GzG6=yIx`z)~ND<jPM(7;9KOXeF&Gz9d58E4;hZycjTB|s1klt)<Bv%`bS
zI#8Artf6t(eZK~kI~2G^rp-kg1iq2uSJ1l`<3;OUBZAA7;6rP$qaUj?=$BK>2F|_k
z9TAm7;O&<n*j>pEJ-$a_n>|gNeV-{R=Zz6VTk%NTqys`{^x+PKCc`XZW*@(SnbME0
zL?$%{6+RgQZkir!qnB9R@_O8qaLvV2{{MGiL$t?Tb^+!KSA7~*$)^6exeP6#%Q7xn
zFXl8qt-ePYe+^XjC7Kw0UX_@+9trAlZO5U}Ird2P4wm%6S@nu{D~_=316a@rMw{ru
zKZUVj)K3C*jxe0OUHz8|*@h)%X{1xHGUvz4z)_Oqq_|ZO&=*^6rlQBz*WrX#ozTMo
zk5je0I#!21mgQb6Z~}C)-g|nrF@vgW1-0wg-inE`eby6}Rr`L${oo(Uj6EVulR;qf
zF~P!ipE85q5B68)&~1dNtuRK?#Tm%Bky87c2tx=O29>?X5`93WG>MUS;myq~@^bY~
z5x=2|FjSvl@}B2NW7{cJl!+EGU&T|$Jkw{!IGZtIrT1A_M(yyeTAS?W&96X&GmQ3a
z9NQn|fOY00XLO4_;3hD16y>t!r34Z}dcdc4AXQAUD|H)PCBd0ADoLkL>|b_MS{Pv#
zm65WxA{b~<ogjmN<rUG=yvlRSjimsS)BjK0^y0SpB}Hs2vvgdY-~Y3o%1FW{I04^1
zrA+tcwzL*9^C&t)zEq(5s$N!833`z^2Z18{o&Xq~zG2>^GPe5=>${I-iN_!`8TK)D
zz8hx7#Cujy1LYW)NO<R!>ww;9a+$97D{9a))4DakFd0F%Jm-O0%ipo+$q=Lb-4b_*
zk;m&r!d$drcoS+l9LiEW1f(J;chNq&=BYql>758_%p4uiVB!Jkb2xka`wvsNEraRB
zzI~+6?*tXx5~z1;c5Z@t$!@#C4(#J%&bsqVl<!oq2Ne*1Eeu40j~g#=VXWs_9plmP
zBcJDliL~Xqs8={5mKDQ9hqlDGGx@O<LF{*1!H;O8!&P==)D#MeFvX8Y`%U})5B-Ns
zZm_yGYcGB@owoGea8B$bY=s;0a!wsIh>kNRYF4OL*bzP`egoO3QO8Q%l|;R#6;j3J
z_Tj<b<zrqc5!(pe?#`StwP&7caF51wLTZ1v?)RY8;X$i@>mg;ol1^e6)v|D^0=!;8
z0Z(_dpG||XfeYi}o9usw5&y_VhZc@^eQIgMp*J%ETh?hJI#UZ#Flj=XSezb|{ah4m
z`vc7(llYQu1sc$_+=;m1%<ww&*%)tE(Xsb$yG8IdFq+wom+$rqqq1pQ$odPq+G42i
Dz3=?c

literal 16400
zcmV+rK<~eit>Gi(NW3ZyS$P(Ixa`hpK<xgE3k6e<SwAG1nINM;y_fdL6M_ZpwBg#5
zi?_k`hs0>wI@@8DZlWgzhA6@IU$ozwfe`nZp0S8-?G`8-FL&GP2L=B5O=$g4eQ#!^
z5AY<M0M@qg?l8aDLu<L3*>zVFc**Ml=_PKQjLjlz5|lRPl*l=bc(k`tzQM%ifBB~W
z2;IY#^1=@UR0eHrmH;Sccx^ges{WnBc$&f1Eshw3h`uXiSmu%w<nAGTomz94lmD)L
z?Y0gspT}Blc>6rL^rmAEOt%BQHhdaay5u)SDo7AU8fl!>)Oud(AS}LHn(33e;40Y5
z9(7sv3&)25+Wk;@q3sB$lt+r1b_UD8K_vYMq@I7m+|akRG*g;(8HMT}W_r>W6f_-|
z9rfj8i$}e2(r4w@E9PvCKy|oA=j+^SND$k1QbX<r5La?n$p2)w?N*Vi!uKXBL6(!|
zVLGa#{eni#vYZ%<5KlNC!RW6HNsIBO=PF+B?B4y&HOXF(0NiN_v+pv<GXPjs{N^fS
zqm-u06a_}v(cV=<Rp``*-;q@9c_&U&iR0vor<F+ICS1yACtpQz);gerE{%Rgi1#X7
z$+0%wb-2{h7qV5~mQVG+k$<&gSwSLkBPP9Gh{&F-ONFN{!i`yjf!iX(;lCB8^Rxfg
zNn1_?xZ7XVy2g^m9uxrr8UWk&&S4Q~7}DOm^udvY>vW9p)?wEmZB~d%gQL*NCAQs|
zw`};ncv|=Usy1p$-<pwF>3Ua9bS1~go79*90u{F~kO$RyXPE9u)BQkV+y7Meh+7Z+
z0Ww_{l-ar7ZTMzK-Y@QjG%PEk`!1*L7LFc0T_59#jR$$hi5Gk+y(>!G2vrLLDzjCZ
zuTuBo#kl}{M*_Utj&`B*-sJ8K$bAJ7Df2fp6_8-xXj)d;#Sej=lR?6K1YGkyaWm`D
z&qlF6<8Nb8W_&!XK5q~rA4}L&p}Y#MNq-JyZ4i;as2QsvVMc5zfOH5EJ_70CLTuZ)
zw~Qenq?LP(C!04Z#b(ye0mbQhl_Mx;(}Y&D26cL7d2AeRu}K!vC`glAKLtj@>iLb&
z5Fl<!xZ8oGORLD6J=y)`>!*mpJA{L5dCfvyT~mC4_gRGk;VliGo`8-H1|*=g@=NIr
z1cuMEJkN;+=g+ye1+jCM5WR;pJ-={K!)MBOCazkXBsesXse7X6*7ZPMWhTlQIYULA
zy!PPY{0ll7W743nbQCkvNDTgR(veP+7xvW)?3=G|tR@I!Xz~ISP>&h4xA8#>*Mq-$
zYa4`uw2AZ}=YCWDIx_>`-WIokFwUZXj;pu8I1JF#AgjaT*j)YNn{Za)7)$92ywl#N
zpaCo>pS^)W41hi`5;6lvU>Ckmt!OX!DfF~uMBBTs;hy+c<5oqyHK{bN=^zPe3-C(1
z>*2=D*h$(R^2g#_bBl)bs6Q<gH-H~s7T+is62xAS%V*}eG6wsM#fBqCp9>Zur2FGq
z`H9(4k(eT4&%@}X3o0=V>@Xldy-uOE$d!u%5LO<q2q1sxRjsBX66Xg;g0?8optH>j
zUgLYIR+(9G#3$`yV=n-Kz+M58V|z<Jr9PYzoaCLZ$sy^@>{y*Ke!7ejGkfD9U3~kS
zJ$R9Fl}=#tup(ejBfjfM@Od%mp|<_MM&Z^IK~8l@oIiE5?{^n;3wE=D+v<siOsJ)t
zw;}u?g)qRj50L;QwlV?@)crugix$EWY9tH3a_}=2>QVgJQbzy%?IaZkc%avjV{e+1
z1j~j>NPiYwk}jOd>+E+=pL=*jL*^tQo;8gJ5G5V&zlbKlqQvjnN@b0l0%c`i2Q-Ch
z3;p?ko8+m_FK4&wm7I;FNyS?;^xC@2@#+V`y}f@FtPeyZ8HZ66p0*#&Kg=A2C;aZN
zb-o54-RRR+U@_=IVY)RGWoR*ysKLmNM^_z|s2eqGz@VP?H9-YFq`(Pt%m%W9-gfr!
z0MSB9YAUS2N^qv&uN^|(uhWn&$rQ1U0ni0ut$oHII+HmRKh;QceCgM7kiFtso?&+@
z3<|-Wd<g~*;Vz@-cR$kP{P40yo$iMFm24wlj+(>~VSHk3eTa1<_AoVD!6g6}*>RM3
z7F*SZra6nqesj7ZxTm@4M`%}PGYK^UMaN}Y<Zdv!sSsRX*R}x{jp2-x5Kkx%an0ox
zkduZTuFjZ&c41ZlV!^5T8+@14oih$0Z9PGSXz`a#f704iD@!_*I}2X_3deumJB%<N
zF*2LdxQP$e;_c+_gNx8+tuSjwe1QXa!977@U;;HSlK%Pqr8~gja$tAyIh*}7o&6KL
zzPL-@)_Z(3Ja8`B@}ZEBNV@r%WD{2`U36H4gXB3=fyG`ce2ZhSo#RzrW2Yf%dMu_U
z@V^yrc13Hye4x%LFG0M}#LlK0^Z1*hJkDKZV1?HfS?gF_+kB7Bt@Cv9VB&Aji4%PY
z($rCJbDn&ddrJ@8=iX1tt28(Ws6f)iv8B%k*z}h^(9|iB#dF3YXaaL+vGz9*Q^5d~
zjaZO8F|t11&n>FWgs*d70O=cl4}D|ATR*dForB9GKIjO3z4bfRWk4T(R7r6m0^LD^
zRoiGwhW`>xrDU;VM^KtK3Ka7wE#Ca`MU_F~qd#1PmxG&*fNOrxJOwvxt%q{iW|NMO
zxTD-!FAeMfZT>yygNoV4GL1a^8B8Y-gTedBfU=88`j0NM&QQ5uTlQ(xOUiN9oR{s8
z%X!61R-&vixjs8T-IM&bRF~=6o9PPQR561vrv9wCn|e}sP-*b6q6c!f<nr4xHnc=e
z$uI}9vp{0fK<Y47<0<?u<UxxDK$+vzE@~_n`QtDjO|0tQ+1B*GBQlQn9+f;oQa7VU
z{W-v&dzVc3q4J^>&OYQL@YPN~t8+wdPux_%VTCaj#WEgBh=ZH7vU&=RnmV#1D7SA0
zRN=N%ttQfE@GFs~D_zeL*v8dZc|)0g(6qY8CFGVPaIdD1KxlS(70oF&kY!TheI{jY
zx(v5!9|-5sqeb25&#;J(R~Qmg7euHP>hSk<3lR4U*cKt=Z@6UxUWF|xCXeGJY#AHe
zym;7n(G8Ai1YtkY(Nl*?r*LsEtU8JC{gYb6os`uIm&&TiPn!=;d16b~WLxnQF+Vav
zC(h00GI;0FeyOs-ZZ`OXpNWDF`)s3liYCL!;;Xdn#CP6&r$8;|xq7MPDLZHnWOaS$
zcNn1jJOl5;QMs(r&Z_V+nXrm^DZUYtq<#r6@1IMa@v6988J?3uYB4&0Ehun#r<qOK
ztzsmwJ?C;-;=^?7El&WU!jZhIfTDx_=AloQoqHf=HN+NmN{Q)8l&-^HE+^+O(7*|^
zVr0!0Kn}t3R1!cEQUG=Zu~C+<rI>}}2Pz9qCK&5QZ!=nlVr52z%|D=<%njG&ZM<-Z
zkVNnJu=nVDCA2he5mK<K{(q_Ne*({!HcET)Tt<guMAGvcDu)%;rfm7g1*E}hFM6F_
zTUheE7WVU0LUCFUkVE0gswYmBjs>#CO;0F{hx~JW3w!Zj&tDW7cIBX8fWo&|V9dzd
z#3YLPTjO<xV$Xnl91yw(xa2uQ>djNz!z97Q?_e;v3@0wG<`ZZ%d$iY9DR5IhSkBI3
zNq@{0I~<*_P>_lfv-mEki%p@W2|?i`av`Y$0+9s%4(TLx;Q14gqDb{0Xq!I`{~UW?
z1s-QZ9=;<Gec-$`&jvTqt~uJn?=9`SM278iU}?>4_l8#jGdC#{5aiO1>AWuZAqv`Z
z6rI*arFG$WHr$WZ8}?E^Se`30KR-{Iw-sdf!-}B<X}v0_GMgm{=+Z$CZuZ6mA8R|l
zTXDI^q-4<Iv+IU@CLgK2hK^2%HF^mq7J~w4nfl~8i#V;n*0dIPyb^X9TRXbRf@a4J
z#;a(oGb;CxfaOtyv6FeLLhnxj0JR+lP4p_w;Y4>9%z#d)p&EK`>6;RTJW0<r>WBTH
z;Qm=$^huCxntZ#VSsVry^5slSbRIqj*W`!+a*FBD`{=*mCuH0~03XYNIFKHn-kr?R
zxhoyemc<db%FQ@?wgzGJ0aN<eTLHA#tAG!vECin(QD23z>i{b7HM0mV#2Jkg1uH&U
zWX=$cbT^i@1C%#lXv{F-4ek2ancz_Zf{93h@yPpW*Ml16fZ{5ISg2G&^!N30@5Bb3
z;R5?^E5lYPQ#;U2POoCFRV0bI^orm=@X4A^gGAwIb{jog)v&`G>q$~Wb8Z4<kC9PV
zU%gxD4C76Suq~kTT`9GQp;Z{!&*>%OG}ecljSW7-dpOCuIRy#J#o49*>)?{GVJJyG
zrE1B>tv?++b^~nH#ByjZ-Yw~nuw7-rLZjKRQbNF9lLWI+H?0&4lhGY!2qc+a3^ahX
z_D1_KwVt6Dj1Z*QJ`T3zeE4o4eo48&(%I+G=v==+AlJoDZz^k46-hM#pIlKun;oAm
zfvJN@(5cOt%-p(#K!Y$tLbG$=|F{d|<<BGuhaB-SI`Rzq2T_?GyHaJYI;1KA!H;n$
zyk@GgG)kz@<X@g60P83oguMsp0_IwBBRM5<;h1Y4I|xQw9}y(29fEBHYZ){kcOnjl
zZJQrbcI-1}Qlr1Wc?*<%DR8KERFCs>A-l7GWUUg0j_mGXc{7IHD-KNhO_BT+eDS3Q
z#&t7Lr9QaMpGgX*aUJpZ(j-tpLLA2<mxRdfke~}pJ_?D`^U~@UnU+4gnVL7us(!1K
zj~!!<F+McSbbzJ^@x9kqaSC0D0mE#cbM;ut+o`-8)}sH<K4U5W?JE>h6@F>^u%l6(
z3}{Zx;jzunEZ;!bv?R^l1^mS1OD$G8={=Yj(OrCPeT~b(Uk4nkseu0WduZ75%Y3HB
zl0B)^{+jq>YJhrR^>7)ML<S?s8(84wo|O|e&Yledz#2-322G7MG8q2$RZC_zbese9
z#8|~7V(<cdlv7OutBZdDYKb}V(5=COFm$E_NqZ5@n4A{J&Swno=5P2dvOU>ow(N$H
zsru?uSg@aPJMn6p<>c~Q{~g1xGvM_D)bOrUx=ksDuCi6p4@{OHO}NWM%!4M7f-s$)
z@BV{Z#oGa@nKEwW{qawa+6@yaueI)b^WFES(&RmsaQnCMw{_XLERv&gj3K@{3xQli
z(s;Ov^?Bxzzf2z?rtP^Iaa^<i2M|1=5TYI<8Y}bjHT;2+JZS;F!HLbZqxY9;{|va;
zup{^?!?M6LMKA5vSQ^BNGn`q$p%Cro=jQW~l_0_;F<9*QxT`M1Hxo&60;l@iZPSMd
z@jVYLct_b&;Q2=pA=>}`iuSUbe9@2AoCG9`D7_Oy!08s@YM3>ujB5jcS%ORlO|_bp
zVG3_2vuTci2W;K?E+cV}Ca%l2#T{D$?#Xz~0w3ZRX8p4Gx~bBWhc~gMNtmx0{sM60
zZf<(V0VJluz(5ygI;<Ikh)(8DOm<lj)d#g2dlz*iCz*eSi|B_sePybW8pR4l4hUNT
zF%F86_arI`k|W)0Fr7heebHS8*2qd0(R9|^%NeIRWpTUtEUnYh(^k(`U}Q0obJPzj
z7S4d;CqETZe*XzSvre9)y?PG*C3odUo?9s^$HHg6iz0<95{=cos6RD3Xc~mYnsA9}
zA(S*@>pcdGc$TT-R;KPfETGT$Y`n2?14LJR?emVwqB!7w2b9Q2zR;xOA%B;@kV8hn
zr_zmJY9S}Nm|K_V!YBb@TM<~_52^lR0|>n|$x3^U+&QyU%w`HU^H9|dwSD7B08j2$
z^<y`hwRWB$7J^kHyNM%mL}T8J#~f8pI@_<y-84FJqwg(+oSKdJQ3t~;W+2|4q+y3F
zQiDBz7%9v_VX@OfzDfF<^I!dDcrqJbw>(_L3P)5_Rq6O={$mG$sou}o%fomc&g^WW
z&<QcUMFLU+hZuxVN-L@fNBqRrB}I*9DB|(syNs{s9jOsKAIfXq`g;V~?rfI_MQ^p|
z;RYr84ESjC(VH$v#3vi6K&49wo+i$vFF_Sn!w{3%Z#Qz<_^ydeX~-rNolq{U4>tv{
zh1oV{OIcSs&hD0^8;;kLHu~&P4s<{jq39JgUX3AK17@xcFFXLa%4{uZrS~B^6-%p3
z+6*t2UB3l1pReJi<I?%hah7y2cltei=22vKZ8ltNk}CobP+vbf9&++W`CS$ZSok*&
zAnW(|dDS0S84cZ*@<%-;-*)km;@~MrY$Qz@r&e(AF`YClxqt&NM6_}dgibY3rf^!E
z+Zcy=a%f$3r7&Csj+?Sj%fT+UeHT~iY(F0YmPTi9ht9~-j;PX!k$v9?FL^(-910Jv
zZetPFy+rH9o<18$enI7W^YPxbPa^Vq!Oyy@&#vtkhTQy#)dFrMf6{XEf1HlL-$MMp
ziWv&DY@Wzd0IbAs+J&ee&rC5kJzK6)Cy(8TRspPxJay2l6xXYcgil@@&bawGm8cZn
zNg%_tL7(=`RT;9SYIX#2S=nZ!0S$8H2`6h7ZC-%X^CmH5RIDgH9FnoP+v8+lSoh-x
zPgwn)Hn53}Y7yMt&KV>5pH(<Qarn|n8+xBJ#=vjHIfrFQoOGeDH%;<C*klDJL5OUf
z>J{!EsnXA7`IE@Z-AlbFv$In67~B}QG;}A{e>(fsikcNILfI0APG2CqHu6adMoP4i
zh*wHqnVhMrTP#BgB^B@dk)1uTs1$AZRkCPjDOI2sVAJ1I7Y%KvVU1d_Y`2Cwm38%%
zDFJIS7~|OwZv+LTes^TEqNDp9-dBiWDh|R_ttji7ZQHSi#+qQKRVBl&|6vzI5C4?{
z$+g%s{F`>ESPZyh@DKLpNATeN9*RDh>V_h4?y1MH=n1CX#;18zy^lBESr6E<op^Q)
zw^2Kr(NuYWn_DrY1Ho^0$){=k{8H;I#pJ#mU_wW`3J*>4fvgIkITduA#cN=QCWk+l
zSkA%DC%|t=f+o1N!7-7!c2W*51~Z7Zt(*+t%d%QDTD$~{JPIxud$nW+zh>p+4^p`N
zf&n}+F2x1u|EWy~chdm@h^x+3;cQEh0<x-PftQea7uV+uc50L%!#vq%#0N<X&f!EB
zNY>t73>OH}zFz~z5iAQ9w|$?&e`Ip9!sFgBUuY%&wL5ISx;}-Qrpj`bWvIYds?>9h
zXW8N3sg^cn(v7#(p_{JbFQTn=PAPtGut@<En#z|R-mPDAclhu8?qKE9zPtoVI*rZh
zNvqmPQm9E5%7q@~wDtzGxH9YI<x(<G(kVcd(S!XgJ`rV~+>jFmHse9z<(`ewrM#u3
z(vyLyTJ%6I{#tT)h=VRdJ{|GOeSs_~0ikfWlZ@-%s3x?e3t-6fC+aqz0!co03Z5Fx
zkUF!KG6jfIMVdXm6wnhqFQ6Dn#)x#lc8v{7q@zfiv8^n~VD}o#sqwi>KaZR7x^<1H
za3&2*!yMs-AEeLK@Cd+K?-89f`6YOICu`d<=Pf9ad0+6V3o|a|lJYKiBGX@lpUZPJ
zz-40fgT>z$NPha%P+LP$@MPK~R5V_1rgPtc1yR^iS=(oilcY`>%toqvZC9H{aN@nR
znS_uR>TYuyb*p;yy3KkQ;>#D=A3-C0x#*}D>f_a+S2)90EofYE)+X#&CEoR^qL(x)
z(xQAo<NKnyy7oos>Jw;wpt1D`uSB=<F-ZER1YAe>H6SuzF0z5FFWw0+bL?A!Tdr~5
zyug&C(UvV$kQTqxVos=3Jy^J7>}|GzynPiC%pal0jCqux=CojCwoXai0=Az)q=9b9
z^|GfgMIq_1C1G?4Jc%DtDg1Y9p--o6`!Q<a60Z{7|M!(?0FmyCSwx>q?d(6T73e#p
z(859<hE9&Zy<g?or(M3$!j+X6989!kX3y7kP6<8%5Cx+*<t#)%i}G9BOhTHGWBlWA
zb2x`~Kam0V8>{b8`%Vf0hcLf_*l2&zq_p`-wD_+*`75L7lJsLscN7MGF%G61Y3X#|
zak5<GV<8*K_)Q3;Fr?RJR)^(;#d{hbg<B(RjFw#Ds43Kj#B=gn5w839z}y$QoKR_*
z2CoWwZ+-$xNihK0w{+U3Zi*cudPeuqScR4FwElJ?B1XG<r1ZFZ=Mp<ATr9<@+_FX;
z2bJY10H`_^1CR>8#@Kq#cfz~0HWgnAVP=3*5IbN;tYEW@**FUWtbKf-qC|t9F>TqB
zb}zBbtmnHo&&N-%(WP|)zs*XUAqKedr)#LuJ#W?sAbAKF^JCdo#I@46>uKNEZz|Wx
z@R?XOJDd%g*En?NYxHYSk<%u`liO^)yl-CeNiWn-c9`(_E@Lexl<^bC#wtP*#V&7X
zG|El#;|jYdqI+(AqGk%r@m5e*om3kHJ7wy`{R!z9P}KQ>DFLSN_)&TCus{0ra*aBO
z8wiL+7^u+NkSX?N!Lw+lHUwxI<N}|duxA;BxJ#xcO%Z_gGfh5mu;%n3MhBi(afe2x
zQ?rbeykgX;PXif-<fUslyLG-KqyFsfxm0|B&r_C?PRZ7Hy4N}=`d(i1RaHy6xqa5&
z&JOXWaOt1DvJs)@MKr(e1%O@2>3}^=`U3Q=CbFmG<DjuOs<VSFivirUiahDr>pVZT
zijpUg&_l>$O)F7Fo%j$E)Ko169^MCDW-gZhmssB1oLafA-M2>rp!H-qb{Ecd2-xu+
z=i~T&Sy-$u`yd^Co@g_|3(T2JjqCQFVcnDK)ag&|S-jyMaH#C!p!d9{yv-Qg9@Ack
zp2vsSCpCPA*EW1^`G*_#g08Zenq0~z4fgpw#Kp$pYQR;#^n3)K>oIhAt`y&{%!-#U
zZ=y$mG_C%*+xiWRwt<gP5`uSRg0-wm%91=>A4m=~mU0erjk^N3(#aGcjvLHBCfjwD
zY+P7m2H7IB7eh!+4o83XXA%H~ieAN`q)@<g+6a{$EZMBlET&{zTsSLU1MSt?%!#{E
zFIvb`e;$2&Z-1}RSE_4uX!)MrJhmjs-cS<Px8QgoQOKhg+S@kA4c0`Nz-11>uC0pN
z205+B4Nsh<6AUG=EVf;GAtO{3>ly`gWk=7wmWPvMda*_N@PkqHV41w5@;B9o%ulA`
zjXTlD*D39oumV<3Ch&N~sfl#QtP*cgU4G|D$4o6loj5vL=ld(g$yHFjE=!IUG0-l^
zbO@GC|2^VXE+?@Fj?@_2x6QBvq$aYQ<e0jmIhpO<qfg>P5{_+<Em`N<YtE>nZ%Lce
zBQuM)Qr;<eEn+CkSI|UBPnvG+az&Uk^^?x6DHh+x5?H3sn-*5IaPk}lq(*<+y>4S)
zS2^>UeaAXdcMpcLAG=sw@PhYum|Cd_peuT>G`HwJkMF`tcS3D$@e60QEYFys@Oqn0
zM_$9g(XU)&06BGFth03<F&gE3)dz#{iC_0#s0D~fgPy16oAWX>N7=ZR-Pmtv#^~+K
z;_lY4S~H`Vk=ul|6JSGl3!EoxjjssR^X9_I)<Pv1oa)^Yz6l+0*co2y1(XMG>#+8t
zCrnQ?Y|kg6Y)rB;!68$zapj^g+$f!UI}sM(<6O*GQ~<BAiT!GnZ^S-BWTvyI;){}2
zA6Le<fz3=h#sTN;SaJ~PCGUr%6dpkb5hmbUmFLLZr)OhqS4B*5t-|96OD}txRA1P&
z2zG;1TB%I!o|h{W5874nZNrsc3LcTLhTUK?<(>cqM%YA?<(5ntol)f%jzP2S)_)`4
zcmM*&BQASs&PNI>vK0ya()BZy58PM$N{w%gAr@~xj(y4KxD@34nb1%<(XMZJ6Er_7
zbP9-G$Bn;a%wOJ-8A(AHJ*86Em0Az}&|+ng@DK(xy%qUoK2zy@v9%q;iv32qj0!+m
zErs}23C7UcxTJt1i|(-HDXlV8$VLvXCcn7CyOs@OWIlGv6J+7~2NfRg7&LW{16h>d
zOBIjSqQL9KC~rWt)8a%r68p&o`pf)(_D&a8ub%BZmeN}dH}+eE79?-IipRmeO!`Ot
zhwU`0{#NTi@d=trz5hBs0vIVa*86Iw*hRTl9OTo)Gb_iuTU7D-Fp<fgLmdKjb}Nw1
zv*T~h-c$ap=HZNOP5;<zvok1K#FCv<{p*$Va*d;TJc=cypk9-7OgvVI!b-AfaYv$U
zs`3K@stTd*-avavUufCw=;@Mt7iI%F+2#{9QSZ@8-V?)0*s(Z+Q3m#_7mu}#7R6Bg
zu%%uAHmyHY`j10gicTb`{5*3sf?_fMS?A$!QJYHEMLQ;5e*v~B;?=a&(UmtwRX+-;
zb@BW$lS)D?Z42hRTP_%DN2a0WElJoAsEwV2D~kL4F|HL1WRQX)l3EH*Ps0~fUN!`B
zfn!dZmU7pr4dX?)gv;=C9iAU>e&*#L&)OexOzxl5KrelkPHh9Db>Z^A{#h`$5IF9z
z0sk++8<k`yUWt4jN9W&^ar{Fase;OO2>R$Uj;)x>5SVqvpX2>@1Z!M+Rg<Xaa=z#`
zzC~rq#>1u#{L(cl^v`9@f+5Kr`I4!!li*-lYHVi|7ILr(Ck3I}PuQ$-6eTu^@a4g+
zz;lUd@VX47B=Y{`P5O@q-_*$f?uSVX2BNjnTC8>Ph{Fbwbbl;MDaV6q0`UqIEtMO>
zFGrZ*I~>&(T$SuD1B!Ajy<CoFkY8Cm^kn4)yCY|pR>O@}uq%o#oD22Z4o7!-;e~>L
zf8({fK{db}8blZ5M%tX(F=)I>Ge)vy+h-34gPg+u^@<a1ws2^vim3w*+9Q1*xSR#K
z*cun14SodY`NWov%wo2I&{n8!w0feAE*UZ!buQrek(F8z8an|fub!-teZ>H@cr{Mq
zl^8=kyJU{;>8&BAKrk7JbOVmBScOR7gNAD$sRt2QkTLfkKySA*&TcgN{1+|B)1vOS
z46W=x0+i8`;^cA1RMTtHUT-+~`z;>}J7wwlY%=teU#~X$ZsAq-E2{B#W}CJ-jluKk
zlEiQ_OiVi1qXj%!Ju5xwMU^^M#a#7}TYS|U6+Mx%F>GcKa4EG|4gRY6qOB!6c$xvg
zmG^OVe|e8NRl06Fsg1@D13Z=98PkR3IzL7bq3(2e<5Vml5w4<)jb;ua+Uh&y4THX%
zv}S@pjL{we2lY4|lJyaVFuuIjgRfJJ*OgAwQm+Ce5v2sdP={UUk+Ay+NOlumelVL=
zFrYMM%*Tk^Z?4&GGM4T8*nT-{Ik69aU5#x7b;y);b>u;0%Zt{rvUje%d`~bGQAReu
z*PAcV$OYl^v!02|YA%ts#WlLjil-dG1;CIjobks<x7io-uDN|jYQg{TigWegAfj#Q
z3LD{7P|LG|NlG1piWL_?n@Vg`e{h1snLrNu`KKGSnsuhRQjH?Ub5KhGPt?Qw29fGy
z=u{`Yy-T+Kt(J%(D~qK%^ZCm_<1nc6hBdqt&ePTY8=P3bT&lU5K8$Rz*QZ^)GakeX
z$IRxbrqrS}nZYP0Nh3%R4K~f8kNy@%*?F^IN3hOs5tDUXNY=!tpT0_mQ4Dw8KPSuY
zpWKrF!i;}mT~SOLHkBt_7MfQ9*JU4hex_1tf(z9q^n66do$pN3$1ad}RU<h1089>Y
zK|kxfZ49w;t`S3dHOKs2^h^amo}KjPWgjvHrV7fN8ItyM-QQF25i>OQtg$eR1AO~6
z;+B`%N&r)Q%j(-e4$%r8t?VIMYu1?*x;l6oKPmy%0$MKo5t@1Y69E}UU8rH1>*1fU
zn4$PX_tKSXNLO|C@`~+CWgskZS&{pYs+A#0K+5d`qaRcsgAg1oX$9BI6xX+Hs`PzX
z?wT0a6Qm8&KG1X6Utxw3vq5m&@UdAm@tnX25ePTU=`@28(<2G~M2`nPPUqnGWnR{G
zOnk~cT4z?2tsl%!ra}f~vTNuv;MUPWfepRcHb*=mcz&>7q#PS)P>9!Mro<p3*)k60
zi_H=Z0s}=b3djBXx`Jw;!Qv}T(h1G-Rjt_D5U0?Wr9|cA5z;%RtNFJ($jMgizgX`2
zES|#{+Z|h~V{_q2`6kva=lgX^lW9=nV@w?mdkO+rnKH9_Q1X#$ZTmi$(S9U#rhqp$
z!qJ907J{&PtUJmy@q27IK^=Ae_kGS0cZn0C;n)bUai=E0nQKxE@#-5E-d4Z72|VkX
zJK`!9qApqKbqdI3wCUR`yDe{kh8vXF7g0E<x#>#kAX48VKtCCS$_A!i;(pZ;V<z0I
z(D0LOKPy+BEqs){4iz(l0$|9~B+x&*4|K!BN}gVh$rjSQl+rmpa?v05W!+}-uz)v*
zfLC*PWTceeF<|j&utP`}bh?6eo10s53uDFa9;Dn*x9v4Hf!`9FHXn&uTR&u!O)aCx
z9`u8${XTH70+ou;b`qizsmBj+wUEs&M0p)VSK~`_PL9zJSCG*q;NbcgWcpHh1DI&r
zuQ=%=2L@GzcB4t1^vGKcmE|}}eyhty81_arnu7e*pKWtn6um??QhZNSAjhxFNpUjB
z9n#-SwwRag!;N@)&~PXSqeO}#^YA{21V!3?Uio3&8>y5vW8i-mJBNYtm#wdO-!b_l
z_oq)y9E5>W;xi!ksg$9z#-2{+P}M2Sl|vmacvJTd4^px`WPVL*)X8z!s&*fW8ns6?
zZPma<B~?}juh$C4468EqSbm}N*W3%al{Z{!y;?u6jhxFET8zPb9NW7>1aB&i#WW3q
zv}rfH8(MUDUy2`K<Q%lQ;oO86{S{G6kk8lKpPI1(;5(yDu<OM{EzaB|)i<^P$_6}v
zaX!^|xSu}^u3ESugY@0b)t*xajzL`DI*>|l$u5P+8H>%dUJXCPX4HWUl>%IiTM6f8
zEqWGwIKFXwPWg&lp6WfvI(b<yiwb~MSK`V4vfOZXT=&=FI}~dal#O4`3nIn#7AM@B
z%%{t#&g;v-3OYq|(+x5w;9!GDQw7AiETYPGBlI9^7DfQb4BMI|xH_z($NY}f*DIq}
zD&U#diMd*g7ydtX@cCWAPBbBHcG878Zfz>Vj8E2`Nb<4-#2x^+ssy+tUI&3D_^&(9
zrHkZf@Wh^yoz=bRI$x4(jsG}nOz)_!Q`iV+T5OElQV_Oq{P+SO4`oh=x#r0sl(<Lc
zV8le`x?kH3N2aXZtB7KkQa<ln>42Z1-cQ5SJ09dWZm|t}#YpDjx!Xd}-GeDTn%>{-
zqG_E~^=8H!sqmI85jk4qzGP#3LK5ZSAEjX}$w6?XB#1Zf%f@{%zGZ(}xyg_iH{F{8
zLKWPM)nK?>EP&>^pB!#VuP|4WEsB3!sJq>U`o%Ro-n>SZWympjM%>2uj9D6um5Pf+
zUMN^-3f!x2A_Hpwy01Rz6Lr|nYM03RVf!_#_Weq=?*)0pL1reuV*~XUZj(aOB<)np
zg=4=Wx4HT%t2yqvzK?Gd{mL<;tKs~y@-5d|n*@vT@dmzwadKwy!1`tWrFHx64_7zQ
zBMySZ{SJEhCpeyH+A<UBkpHbb(ceM6RA%SHUa*&YJkBf@C3gwpA0^W50mnw`yOi(s
z?|==szbBg@x&-Ih)hCj^SF@@KQqRQ0+Inz&{h6VEmObrL(X$RvH}u%^qNf9;c+@FW
zKAP78Ok7BbsTHcoG7a0G>{&DeK~S||!bAKMtAloCDZc;0`)&Uc-Yi5V4lDY-{$T>S
z`gFfZKS&@bfWIP0y#kFnFe4;9j{;dV%Z~b6S#d!;?LW2mtXd2i;4qct{Uk8K#&?zR
z$%ZxpT?u$(Edu#5|2N`YKryF-Dp~Ofmjax8S64ZGUG6bgT9}T(_;s)HAaV{A0Mh{?
z+^iaz5>qdM-;HZO@G=J+eZ|}*u)F5`5XsU=6QSQ^+%<<R-_D;&_t&)NLLLyybuTnG
zu(D|tM3}FXM7Q>RuSUqD7h%@+u3T(v{APD3lVW6-T~IRiL0BJi*C`qN?Gdv`0yK&P
zEtkFNa#JjW9ftQOQSJH2EnLH*Z`M-TTUrANyR<WpQ<WnTnUpp^^R(Nwq+<0b3o%;Y
zkg`yVe<crUFPCNryrL4}DymTJr7D6VK74lCwedG4Xv6Zgpc$DM1rZEwi*_zFgK*f7
zcO_*9uHBacxRj*lb~$_jZcd4e{p*$jccq`r0}Ju3?L{Lt=i9;-;nC|xt~D+?uY95_
z3uD~%CPta%dPr(Qq0JTH+hDO2=h^4Wu`)Uv4Vh%bI8l+x|00^kLj>3oRaytNPOik)
zIq`Y>O1DpOy^C)o%_n=1Xg}Jt2B3J*w7&>g22qT4jl7Rd>TCMup2k#P0hM<aJV{~p
z(8ikKd%nqpH|8aR6B8-wIAgBia9=AN$Luh&&t%0SW_J5Ra4)d>R9%cFXUOJ(J&OtA
z>f~&VpZaT9Ect1do26=_;-CsNr2xdsVS6dcmcN;3e^eII(~S=g;JRV<x!Dz&&jWvs
zme!ue8b=6&)HU!~NBP}C2{U)R6#chx?A$#!&dq6g6jQM8X2FoWzy^L`sJW-j{>osP
z7oaW2UR@k@2+**lyDJj7kQ34t7<J-7@`Q7se#z^wJ*uFc3?3?DUm9v9ESJxTp0tx|
zj0t;HlG4)tGw_uJ5y}^u#fco|O5w;8h!nChs}QVmDgc63v;ns|6^QX5R^TmD;S-O3
z=){{cQ^dCNHYM+fkpco$F_%sKTOG9plqf>V!-W|DrdS{EU!Rlepg&mdlk?jyKfpYa
z#W@b5puF<m2t9~<1T1ijlwP|mI1f3##G%gcM$Y4#2QB>M(w;IKO18~YpGH{gWw+Q`
z7*N%Iq1)(C%%O2CK(UDY)+C$K$Gy~gQ;iM{H3^EU+xHL!hui>*GA5q8+)0*5Eqz3J
z)5LfiyJRMJk;NN)+`Ot+hne6r5c_ia(&BG?pW*A@Jr*@qI+nrXaTB!$z`;K-5xv&0
zjI=ifc)BZwvjMP67hYi;YgzpF%pdb%yy}mm#n;<Hd@+0M+U9KI`#HonpiC~&r$s-D
zd^)ICA!V8fKNB?35JQn<SL+;xK2_vo%SSu_@k}7Lqywl+g;9{+SVVr@)J^%;wFaU5
z2@;wuPA4}u;)QE0j?V|Jq@UUqA2l@0POy#GK}Ofsc%DxdeCEAxjD);Y)#7s|9_4QK
zbEj=&YD)~ysmfN4MWwk#e^b6>RM3uXnF$x{UI=5LzuhFOss~|jQ8{OEx?^lK-!Xh%
zQbId@#d?l;Ppvaxc*y7$7b$QbdaHoqZ+fGQ%zR+Rfk`UmNYb0zn^FsFZ_mPn!tG4I
zp!-}`xHJ-$(|*Eoe3TT=e26%lhPCU<5uMRc>Ahb9M`j*3YOb$0C<xuiO#m{&bA^np
z9`q2ss;==gB?VE*9hC@lC&S$5m4?WPcGL7u%VZRj5De9OR1^rBsFD`yviz}{rv_#h
zGLi;Mu5s$&=N2vgB>Lsw`tmU2c{_NYsV7PaQi<h@LhtGNx}l7=<P#0nuC8%ntY!V|
z1s5@H5OC~otoIbTEcccL!uW{a^Zt9k2!PeT=(Sz}Keux?zeDAF-Vpr`y<A2WPMB}I
z{_8SQyy5;oHlfVLHs=d&qlM62{!@#8F>FU0en6i`f0%{Qq#gi9<(Fy&EqRT(cgYlo
z%x43rBAVPa#;UZ|+B9<FGsXYXlx1w<5J&G0FN8!thAp%?D|Wh-$%SRE&RQ^qeqdyQ
z+w}>v{}2UP;O=wRDd#x`=2z{%0$r)13@?4>-OfcH4LoLed;^`(ps76eIPa;*!ox^S
zrv3`W_u+%#mCrG>qO$u|-^-l-F;-R>9sn?>$BH5Y;xu>~4~6!NKJ9NB!e>L0nd^EU
zO|(h7iYB*Q^HOn}((~|y1FsU1!~m*EJ&LQ0C_X9dQkpo1{7IkP!bMYd$v*h4Fmm5S
zL%66v*Hd|?dWIdGDLVa2zZ|c^?J5|M8WN?S_9xdX69N|pH&ywI5o#tsaD*pUy41b0
z!w<bCeJ#+$>QR_@)^OJHW@SmG6dd#O880~|-j7LuJz)UG-&G|~X0W0gv>p|l%_Wu7
zf%k3NfuASQu(-L=M+PA?sEFmXM9mV6E5ORM#PQFB1p%R!PDri;%=_mfD*hS}&k8gJ
z>M#GBSD@Fdp|t|F3;LFB`xtbGA`Ljfe}86y7T@DBT4x}Db)ithp)HwH)q1bSMI?Rt
z7=EO0%S!yMF}5Xg&Hir|BnVq-mZ$^g4Sa;1+WKONfr~4p^pVW1#9#?DP$xIXg2|da
zme1{iBPgV|^agVEoP*YeMe|^GX4sw&HA!3qjM#63J^<BTbGCB1i-9+-*PnaFw}srr
zn08L`Jjlz1S^HWUE9D9j?<>O!kzvXB&|->E_ohAS7xf?FC4#D24JgHlwnEkqepzb9
z;(Xz{Tc>S)?nVB9=p!|<qTAIJXaCGKtyqyQmo2S5_jEm_upgAdUcFM5xWQeGDWa5e
zBXb-Wg$8Om-?CD{H&P$$@c$TNo_UugkROs2G-J)sfe*9f=S*_>AX~3!=a`njfaYQe
zd|aVEub^Anauc)C8{trO?=RS{iJ@9gi=YV}z|r_iONALdv4Sc1#b}ZxQWGvWBs`s$
zdWX@7M%L0Z3PR9Kw$Kso`}+-=8>o@iaXupvw(EqB&aL1E6+wZ)N1)fu_wf}F!)vGf
z_+F!7gz_QN*!uaM3ur!%<cq^k;<rnU>kWSu#rhF=MYCNcUU5Er;nr-d=v!t*us<84
zWNf;=oq<Y)|75)LuIP0SF|vM=zA}#xHmYw9zf)10$~p)6gVqdkTcdt++jYI)sQdlK
z2YX@$y7)H&__MBgRx=)Y3MVCo>kd8!)DK$!G#~O%JA-}!i)*c2pCp__;uw5bqKWbf
z?IATB06Rx>o6_u4t6q5$JjzZZJvL7Y@99Ro1IowVlIU6nQTAZdOA#fL5W_c#r1BAC
z_~<9^f6f<~gm$z_D7n66xeF(I+FP26txHO>73O~io7UK+gkAUISAMtIy8ah#dmThi
z6VR7ve$y@TI19P#MJjNIziY1AU$@Ys_iKJ0cV&0-A(_fii~<I6Lp{f|2xtAcp;7>J
z8Y+g5(oOlNg`91)WmZr_Z0E+mB_M9?MYMn{)Jh}iv0HpG?~W7yDG&P)Kqaf2*r+&-
z=sGubJP)^5OG<;;qv)3P(^A0E-`3gI6F5J_Xz+phqg&J;b%ZV1{o01>`}rHCv{a0q
z_7j~mFt0bn>E$GN&QKJUYJ-A@-0Z~&`5e~DQ%MvMT_gE5B@EMVp=>n4#3(-hE;O}j
zsYw%@w6h$K=h6y>=cHmZ>P$%2C9GkaCOY^T0s{u1SNB}>)pAB)Zv+!b**^@SeW9I#
zdv}wABBcCou^@9uqWO?7NN^CccQe$f{h<As;oqk``LP?QDBPVL{mOdn&@@x|8931`
zh}}C-xT^1Hjk!&mW&AYwn3at;l9LDxV<7*`IJ#eS-ovqIU5X%~6KQ%fnQysvaZRA0
zYsmXiZmVzfXhL&qLR!n|c?`}8xP7tbmjN4Tn}!wGit}R`4Y3=JBKsqA?c@N+`w9>-
zM6)+NR7W9}8zMuXHTbu>`Mjx$Ueva^`YLLqtj1H9U?w;sf7)IxM8hXi5*!hdZaZ1R
zCM4SF>9-hZ0R2k8H<XDhY4E^4XY^t9hox#APnw%Ur2T_@u1<FJjCM46j%JXN=)mg1
z_0Ka+;6)9~Ai@K!<@v2MIvus28RvYvFpUmp1m>%-IA{hrLi;gDna~X1vmfLs1ue>h
zSHRT%^R`=w6kGgoz0RxlDEw7C>41mLkAy0Oj&Sby6@I|jqi}EVHN-wSebz!&=m11z
ztIa#dLsX}vNGj<T{vmjk6bz(Pdna^?wMc1vGj5be;NzJxwuYuYpEl#9gKr|p3-20W
zbO_flz2T=dsedf!qKYAKPhf`_Vc0$C<j!G{s3fG<^&>ukwiiX_@>O~*Qp!}6B}fvB
zUXhMWbPjYejMY&9B~PaO*tideQP#jhfI-rhRwH}Y%s^|-g)KqreA{f>ZQPnD&<N6E
zFP(GBP*d$&xSqwffvdL$(q3~`vKH$z^_#09H}PG4i<1}1W*oXfxO0+}CO>oP_PGN}
zwt(o9nJeqj8(W0iA(I%0Z7rgsSQtG}5I~F##9>o9=My=X%Q^&9{<)xswG2C5X+Spz
zY|~DVg{7}KD)=Zrt))ABj)H7@Y^I;E=Nw=ELsn(mBU~U!B3?4M8WXrNV<*I}YtoSJ
zF_jfd=XS`xFHWD4#1g{T`Mlz@A9d_U%RDp^qPxsT95eupDT_RXG*0}K<xf%qRaDh(
z-{7#X;Is>C!fA==a?dy0t!U)tnv*#KEa<52(*(B($jEebj1clWB7!WiaJJRBl-^va
zW`Yg3W9g6OY{r=PU&@PHEcSOLSm=~<Bz`z2XsZ&rOmngXLbMQnHw}n5)cTLf8w}pE
zxV+FS(ffm`u6kL~x+y31w8cB8LeYZbirmPwC)o!`w1smgrH~X(i&6#$bpYLTV)SPE
zlR7+*7<pQ__wd)Nv6W~68O5DHQ&!c$D5Vq0KmUAyFmTpf*q?|b2jhbZWg39)<tEcE
z&;42g1<W^I0i7*Vgzo$U?cS~t>58S&e75Nt?T+?Nvh2nBcQQU11*G&)kl|7bO3tPI
z$jCtM8%QbiqGR=N`d^rXQEi_^5oNCl*m7nJ-H#Y$YR30WWnuABB%<$2q6KRrwO@Ev
za!ePbzgagTBGhB3tY{s7K^b8#o06Ta(1hI+UDDsuP#G<Vyrs~YTuXW>6u2!ylDZU8
z^s+0Ougpz=mLU!sG^6fS5h4ydTY>7vzkbr+Q+{MepvT;Tl00T%YTfS+3Bgnq*Ny#w
z#>i_uBgo9)Q4ejLo)x|h)Wp=r(rD&5!3n(#n-PlSw5{2f(N^V7x}B3f+wtbzs26wx
zpa%kr=440{MxAn-VyyKwXa>QbajXA)&;GQ}S+=J%1tk1Jl!OK!tVEdLzDZjxJTehv
zv34BI&{e}Bg|Gr{5_mW>jZ{bg3JqwN9fdJJ`IV27DHa#YNm=QpYV$}WWL@JahSm<}
z62&T`ih_YqsN1uR8w{flYBRQc*H)N1MLhF)0S*^y`(bP!oBeEA!+`!hzX(%iZmSHe
zN<<Ebav(;6r-^`NQo#NhZF^xnqd#fisMW?_>H{BM8(OI}Hd>2wK;12UfugFE&uz=x
zrwg)gEME4D@NZ1Zi7^6@y7nwys}R;X(E?W+4Czzeu%d8s+K$%Rr@|fk4jfy9Zj`jX
z>8ySU_YwPjr1!21K!y#eM^{p}2Xiii7?bdkt3O+i1Df)85D*I$oS_YTuJ6+O1VOr)
zW2A&b>cDkiltrmZJ-#j?C=4+a`%#*L`v}4tRsCl@19oc&2VR)O*DbB#M~Pr0>mtC0
z&POs#!KM;HE7mafPk-k+HESpvcXVV(J|>*onVb?~%EP#6zwTH6#1uAI!g;ZP9|&Uj
z8GC}{X?zbz*Vnt2fKSsr_6}eX+F%Jj5o*6q8sd0ng_*qU*Pfq*B<(@i{?if;5TkYP
zr{$&I${97(Tbnn2;B3r1&0Pu>#z)e%g9W*ZRrGV^Nu^`n9<T)Z0d-iFL8k8Y|1ln;
zR$H1GZq(wO(F$#h;6{#;13txdSyi<4aID1FM4@n;7qU|4B5e`9jCqed$3!{#@E!AL
zz2W{TtZ*dy=$$dX;|(A3Hs{E5OKYBgFt8hO(k&t>EAkcgh+6fJ`7+I)EoI#Hh082y
z_zSEvPdX)s^iI*C_aSD>x$RL4q6;qJpN9`nrkrS;(`|sPW{0&F3-(c^p*c+%$&2~h
z9?GiNnlL-M5wQr4fq%-5BduCQrY`y7ib(0gTRUz*$Z}<Hc+xFo7(fSCNzgaty3>ar
zmx9|~_FZq`SmaO-7SjM6P=xP3{++q1Rso;^x2(R4o@izo%kiDrzwIT^4A~$1ksA=%
zF>+(anw}ec{?-)Ve7#w$#9bc^SVkBm9!0aueDKs%{^NPa222bX{iLv>u!QH{K69)K
zhVaQXy%gjKFRv&r3@tw}0atJFX4dEMgu*e5AvP{AIb}}~XBF)h$k|VYe&T4)#paS)
ziiyNX8WrpgSchK2*RhapxzG711BFd5kIS3(i*suA2tVq$*xoQ3MRqx43_v*!bwD4+
zI{hD!%7AM@O8L7~I4lvU5aE9nR5)Z28M@kgi*|^@ahsJGVU8U25ix6ns=vM(=S^pR
zF!QHelwrbiN?oe*boaCt3OoV>>k|K#j1>vpy~TFy07Z1+nwXI86Cadrc{5&eKAxPA
z!s;5mFMp&|8|$n0Ssd*WT`rZUBi%+<2^tm9d`zxvj=Qq`+9S&D(-R>}i55t9;c&#_
zM_p`#U$`%9w9}?pB#kv<hg<AErHg0t)*CqQodl--C5g1};YS82IHy*!r-a{fHHI-#
z<_*Q~vPgBrIfefb#Ns87h^%*HK_>)24HY6ge29Xx>%-7gq5a%1{1cdJ)a!on%}Mpi
zSM#Kyn;K&F%=i;GQ8T1_uY!3r7#n+dG(#-VQnImnXHA^RO&E*$#r<Dae~`gRq5cib
zlJq(!s)+g&94tw>W>f?r;bt*&dn8O6kCLa?AnJwDoZc8)ZkBR_{i@n?@374VON#0D
z)$5>s-r|cx^FS8-5Phfzdv|8Gxp`{-4i_#THHENBBPg=IWvhW8gy%2F!}hMayp2Dp
zpB9Uhk=+T~_3&g%p)AzFR+ot!LRKkda*98V%Zebcu3VXI`sQ8m_;3yGq;c`={1@|H
zM{MI(=!LZZi7ghJ?T%1B3r;<K=x6YgRj&r4gxeC)_(2dwgbh1QG({joYA|6$aP>rn
zRAb<t#E5Vr5ztwIpPAVx2cvWx`SCwpvnf3)OSCFU>fhd+PyMr&{kFWO{p|u@b?KC(
z^6|;CUc8sX+!h~Cq$RPQiGmW?yYX+oAUXT%4b1IrY`Cl)aRqCWq$L*WEfA=rS8f!8
zKd>|4I*owsWYuJZjg^-Np?*5re=z&2n;FxyXWAz{y0zm3l#Y$QwrwD{3l}(qKFq@|
zo0Rull_8cpNa|N%E=))Gp8a@|RtKRlU>!E2bd*~OfL+`AvWbI6&;x*?S2m^+{v5R+
zyU`=&6uR6DCfuA^ly!V&{iFmCG9YGLDg?X;>w)^P;I5vNzy;-mJ27(tF*x#%8?dQf
z%1z8WWW0*afaOJ9Z<DPYS#C@e^3Y_>fZ_N?t&zW@HA$xcZ*MKhmD9fPz%v`=yKJ|f
z9A@X9sE*&fGopb)FiOHF6pW30s12!3*W+k3M@}qvr4_wG8%H*)V4K=f0}A5M@{nmS
zJ^lrntx*tVsVNN6(hZnmipWnBrEhZ)i4;h@)*B)vtfjWC&%s`#@A84d={+VjWrU}}
zlM{FsbD_9R3JJ?&FxdEMj2vbim`ip{r#1<Y&;tb@hjA6T{)QClhJ!S~rz8|r!1K11
zu-iJ_bi9J*&p_#jH{E>{><R&NjhZ<pZE_Lv46p@CD|=M+SKX+Iu?F77&A;wLyEm`y
zQr6Yz(a}rdvEFnUyOUTo<pfLu!|@*4j9@`m1szKi^nb!DHO+YIZtmOcRIp?fu8X@#
z+}hC!(E#IRa-Vi<f0V83T@qN!p3@WC9$&qx9%1HuAwMUJ`#oE@**ET{2RrTn+!GTd
z$Y&o5Kdz-*kZ`gYzRT2rl8ct#ewRJUi9>bd0wt>kQmCy2H~jub(m5X*$2gJbnFtiy
esSY2<4nG!n{!=R_PXe;4b&xTotZTYKLYOUKb<pSl

diff --git a/src/bin/travis/testCentaurJes.sh b/src/bin/travis/testCentaurJes.sh
index 3cbd3c3f5..711e69512 100755
--- a/src/bin/travis/testCentaurJes.sh
+++ b/src/bin/travis/testCentaurJes.sh
@@ -47,7 +47,7 @@ pyenv global 2.7.10
 sudo pip install --upgrade pip
 sudo pip install pyopenssl ndg-httpsclient pyasn1 --upgrade
 export CLOUDSDK_PYTHON_SITEPACKAGES=1
-gcloud auth activate-service-account --key-file=broad-dsde-cromwell-dev-ed9fc19865dc.json "$CROMWELL_SERVICE_ACCOUNT"
+gcloud auth activate-service-account --key-file=broad-dsde-cromwell-dev-d71ad10e17f4.json "$CROMWELL_SERVICE_ACCOUNT"
 
 echo "RUNNING TRAVIS CENTAUR"
 sbt assembly

From f2ad4ea2bb3842976ee2a253aeb777920cd47cee Mon Sep 17 00:00:00 2001
From: Khalid Shakir <kshakir@broadinstitute.org>
Date: Tue, 21 Feb 2017 11:45:02 -0500
Subject: [PATCH 65/84] Minor Tes updates. Added note to the changelog.
 Replaced usages of `cromwell.core.path.Obsolete._`. Moved travis centaur tes
 back to using funnel from ohsu.

---
 CHANGELOG.md                                                |  1 +
 src/bin/travis/testCentaurTes.sh                            |  2 +-
 .../backend/impl/tes/TesAsyncBackendJobExecutionActor.scala | 13 ++++++-------
 .../cromwell/backend/impl/tes/TesInitializationActor.scala  |  3 +--
 .../main/scala/cromwell/backend/impl/tes/TesJobPaths.scala  |  5 ++---
 .../src/main/scala/cromwell/backend/impl/tes/TesTask.scala  | 11 +++++------
 6 files changed, 16 insertions(+), 19 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 48ae8b1cb..2285204fd 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -38,6 +38,7 @@
   or the Config backend could pass an optional `"-u username"` as part of the `submit-docker` command.
 * In some cases the SFS backend, used for Local, SGE, etc., coerced `WdlFile` to `WdlString` by using `.toUri`. This
 resulted in strings prepended with `file:///path/to/file`. Now absolute file paths will not contain the uri scheme.
+* Launch jobs on servers that support the GA4GH Task Execution Schema using the TES backend.
 
 ### Database schema changes
 * Added CUSTOM_LABELS as a field of WORKFLOW_STORE_ENTRY, to store workflow store entries.
diff --git a/src/bin/travis/testCentaurTes.sh b/src/bin/travis/testCentaurTes.sh
index 0414059c1..caff9cbe0 100755
--- a/src/bin/travis/testCentaurTes.sh
+++ b/src/bin/travis/testCentaurTes.sh
@@ -40,7 +40,7 @@ git checkout ${CENTAUR_BRANCH}
 cd ..
 
 TES_CONF="$(pwd)/src/bin/travis/resources/tes.conf"
-git clone https://github.com/broadinstitute/funnel.git
+git clone https://github.com/ohsu-comp-bio/funnel.git
 cd funnel
 make
 cd ..
diff --git a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesAsyncBackendJobExecutionActor.scala b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesAsyncBackendJobExecutionActor.scala
index 58b614cf9..a0a9c5230 100644
--- a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesAsyncBackendJobExecutionActor.scala
+++ b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesAsyncBackendJobExecutionActor.scala
@@ -6,8 +6,7 @@ import cromwell.backend.BackendJobLifecycleActor
 import cromwell.backend.async.{ExecutionHandle, FailedNonRetryableExecutionHandle, PendingExecutionHandle}
 import cromwell.backend.impl.tes.TesResponseJsonFormatter._
 import cromwell.backend.standard.{StandardAsyncExecutionActor, StandardAsyncExecutionActorParams, StandardAsyncJob}
-import cromwell.core.path.Obsolete._
-import cromwell.core.path.Path
+import cromwell.core.path.{DefaultPathBuilder, Path}
 import cromwell.core.retry.SimpleExponentialBackoff
 import spray.client.pipelining._
 import spray.http.HttpRequest
@@ -71,7 +70,7 @@ class TesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn
   // Utility for converting a WdlValue so that the path is localized to the
   // container's filesystem.
   override def mapCommandLineWdlFile(wdlFile: WdlFile): WdlFile = {
-    val localPath = Paths.get(wdlFile.valueString).toAbsolutePath
+    val localPath = DefaultPathBuilder.get(wdlFile.valueString).toAbsolutePath
     localPath match {
       case p if p.startsWith(tesJobPaths.DockerRoot) =>
         val containerPath = p.pathAsString
@@ -87,7 +86,7 @@ class TesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn
 
   override lazy val commandDirectory: Path = {
     runtimeAttributes.dockerWorkingDir match {
-      case Some(path) => Paths.get(path)
+      case Some(path) => DefaultPathBuilder.get(path)
       case None => tesJobPaths.callExecutionDockerRoot
     }
   }
@@ -110,7 +109,7 @@ class TesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn
 
   override def executeAsync()(implicit ec: ExecutionContext): Future[ExecutionHandle] = {
     // create call exec dir
-    File(tesJobPaths.callExecutionRoot).createPermissionedDirectories()
+    tesJobPaths.callExecutionRoot.createPermissionedDirectories()
     val taskMessage = createTaskMessage()
 
     val submitTask = pipeline[TesPostResponse]
@@ -187,13 +186,13 @@ class TesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn
     }
   }
 
-  private def hostAbsoluteFilePath(wdlFile: WdlFile): File = {
+  private def hostAbsoluteFilePath(wdlFile: WdlFile): Path = {
     tesJobPaths.callExecutionRoot.resolve(wdlFile.value)
   }
   
   private val outputWdlFiles: Seq[WdlFile] = jobDescriptor.call.task
     .findOutputFiles(jobDescriptor.fullyQualifiedInputs, NoFunctions)
-    .filter(o => !Paths.get(o.valueString).isAbsolute)
+    .filter(o => !DefaultPathBuilder.get(o.valueString).isAbsolute)
 
   override def mapOutputWdlFile(wdlFile: WdlFile): WdlFile = {
     wdlFile match {
diff --git a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesInitializationActor.scala b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesInitializationActor.scala
index 67a177b32..773522f43 100644
--- a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesInitializationActor.scala
+++ b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesInitializationActor.scala
@@ -3,7 +3,6 @@ package cromwell.backend.impl.tes
 import akka.actor.ActorRef
 import cromwell.backend.standard._
 import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendWorkflowDescriptor}
-import cromwell.core.path.Obsolete._
 import wdl4s.TaskCall
 
 import scala.concurrent.Future
@@ -33,7 +32,7 @@ class TesInitializationActor(params: TesInitializationActorParams)
   override def beforeAll(): Future[Option[BackendInitializationData]] = {
     Future.fromTry(Try {
       publishWorkflowRoot(workflowPaths.workflowRoot.toString)
-      File(workflowPaths.workflowRoot).createPermissionedDirectories()
+      workflowPaths.workflowRoot.createPermissionedDirectories()
       Option(TesBackendInitializationData(workflowPaths, runtimeAttributesBuilder, tesConfiguration))
     })
   }
diff --git a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesJobPaths.scala b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesJobPaths.scala
index ac7c9aff2..928fd0fa8 100644
--- a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesJobPaths.scala
+++ b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesJobPaths.scala
@@ -3,8 +3,7 @@ package cromwell.backend.impl.tes
 import com.typesafe.config.Config
 import cromwell.backend.io.{JobPaths, WorkflowPaths}
 import cromwell.backend.{BackendJobDescriptorKey, BackendWorkflowDescriptor}
-import cromwell.core.path.Obsolete._
-import cromwell.core.path.{Path, PathBuilder}
+import cromwell.core.path.{DefaultPathBuilder, Path, PathBuilder}
 
 class TesJobPaths(val jobKey: BackendJobDescriptorKey,
                   workflowDescriptor: BackendWorkflowDescriptor,
@@ -33,7 +32,7 @@ class TesJobPaths(val jobKey: BackendJobDescriptorKey,
   }
 
   def containerInput(path: String): String = {
-    cleanContainerInputPath(callInputsDockerRoot, Paths.get(path))
+    cleanContainerInputPath(callInputsDockerRoot, DefaultPathBuilder.get(path))
   }
 
   // Given an output path, return a path localized to the container file system
diff --git a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesTask.scala b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesTask.scala
index c5f540ec1..f6645f009 100644
--- a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesTask.scala
+++ b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesTask.scala
@@ -3,8 +3,7 @@ package cromwell.backend.impl.tes
 import cromwell.backend.standard.StandardExpressionFunctions
 import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptor}
 import cromwell.core.logging.JobLogger
-import cromwell.core.path.Obsolete.Paths
-import cromwell.core.path.Path
+import cromwell.core.path.{DefaultPathBuilder, Path}
 import wdl4s.FullyQualifiedName
 import wdl4s.expression.NoFunctions
 import wdl4s.parser.MemoryUnit
@@ -55,10 +54,10 @@ final case class TesTask(jobDescriptor: BackendJobDescriptor,
 
   val inputs: Seq[TaskParameter] = (callInputFiles ++ writeFunctionFiles)
     .flatMap {
-      case (name, files) => files.zipWithIndex.map {
+      case (fullyQualifiedName, files) => files.zipWithIndex.map {
         case (f, index) => TaskParameter(
-          Option(name + "." + index),
-          Option(workflowName + "." + name + "." + index),
+          Option(fullyQualifiedName + "." + index),
+          Option(workflowName + "." + fullyQualifiedName + "." + index),
           tesPaths.storageInput(f.value),
           tesPaths.containerInput(f.value),
           "File",
@@ -92,7 +91,7 @@ final case class TesTask(jobDescriptor: BackendJobDescriptor,
   // if output paths are absolute we will ignore them here and assume they are redirects
   private val outputWdlFiles: Seq[WdlFile] = jobDescriptor.call.task
     .findOutputFiles(jobDescriptor.fullyQualifiedInputs, NoFunctions)
-    .filter(o => !Paths.get(o.valueString).isAbsolute)
+    .filter(o => !DefaultPathBuilder.get(o.valueString).isAbsolute)
 
   private val wdlOutputs = outputWdlFiles
     .zipWithIndex

From 74727a7160a8db682cc28af3e17280f3c32e3f25 Mon Sep 17 00:00:00 2001
From: Thib <Horneth@users.noreply.github.com>
Date: Tue, 21 Feb 2017 13:28:44 -0500
Subject: [PATCH 66/84] Throw if cache hit copying fails on SFS (#2003)

* throw if cache hit copying fails on SFS

* PR comment
---
 .../cromwell/backend/sfs/SharedFileSystemCacheHitCopyingActor.scala   | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemCacheHitCopyingActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemCacheHitCopyingActor.scala
index 3821d02dc..396a33849 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemCacheHitCopyingActor.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemCacheHitCopyingActor.scala
@@ -6,8 +6,6 @@ import cromwell.core.path.Path
 class SharedFileSystemCacheHitCopyingActor(standardParams: StandardCacheHitCopyingActorParams)
   extends StandardCacheHitCopyingActor(standardParams) with SharedFileSystemJobCachingActorHelper {
   override protected def duplicate(source: Path, destination: Path): Unit = {
-    // -Ywarn-value-discard
-    sharedFileSystem.cacheCopy(source, destination)
-    ()
+    sharedFileSystem.cacheCopy(source, destination).get
   }
 }

From 996acc23145bc8ba2b19d089c8dfaaf17f914737 Mon Sep 17 00:00:00 2001
From: Thib <Horneth@users.noreply.github.com>
Date: Wed, 22 Feb 2017 13:32:31 -0500
Subject: [PATCH 67/84] Update allow result reuse (#2010)

* update allowResultReuse

* make job attempt nullable

* fix compile

* fix EJEA variable init order

* init in preStart

* fix test

* PR comment

* custom centaur

* centaur develop
---
 .../standard/StandardCachingActorHelper.scala      |  5 +----
 .../migration/src/main/resources/changelog.xml     |  1 +
 .../add_attempt_in_call_caching_entry.xml          | 14 ++++++++++++
 .../database/slick/CallCachingSlickDatabase.scala  | 14 +++++++-----
 .../slick/tables/CallCachingEntryComponent.scala   |  6 +++--
 .../database/sql/CallCachingSqlDatabase.scala      |  3 ++-
 .../database/sql/tables/CallCachingEntry.scala     |  1 +
 .../execution/EngineJobExecutionActor.scala        | 26 ++++++++++++++++++----
 .../execution/callcaching/CallCache.scala          |  1 +
 .../callcaching/CallCacheInvalidateActor.scala     |  7 +++---
 .../ejea/EjeaInvalidatingCacheEntrySpec.scala      |  2 +-
 .../services/metadata/MetadataService.scala        | 12 +++++++++-
 12 files changed, 70 insertions(+), 22 deletions(-)
 create mode 100644 database/migration/src/main/resources/changesets/add_attempt_in_call_caching_entry.xml

diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardCachingActorHelper.scala b/backend/src/main/scala/cromwell/backend/standard/StandardCachingActorHelper.scala
index 52717df8b..7c72d6423 100644
--- a/backend/src/main/scala/cromwell/backend/standard/StandardCachingActorHelper.scala
+++ b/backend/src/main/scala/cromwell/backend/standard/StandardCachingActorHelper.scala
@@ -7,7 +7,6 @@ import cromwell.backend.io.{JobPaths, WorkflowPaths}
 import cromwell.backend.validation.{RuntimeAttributesValidation, ValidatedRuntimeAttributes}
 import cromwell.core.logging.JobLogging
 import cromwell.core.path.Path
-import cromwell.services.metadata.{CallMetadataKeys, MetadataKey}
 import wdl4s.TaskCall
 
 import scala.util.Try
@@ -75,9 +74,7 @@ trait StandardCachingActorHelper extends JobCachingActorHelper {
 
     val fileMetadata = jobPaths.metadataPaths
 
-    val otherMetadata = Map(CallMetadataKeys.CallCaching + MetadataKey.KeySeparator + "allowResultReuse" -> true)
-
-    runtimeAttributesMetadata ++ fileMetadata ++ otherMetadata ++ nonStandardMetadata
+    runtimeAttributesMetadata ++ fileMetadata ++ nonStandardMetadata
   }
 
   /**
diff --git a/database/migration/src/main/resources/changelog.xml b/database/migration/src/main/resources/changelog.xml
index 851589da4..dc182d106 100644
--- a/database/migration/src/main/resources/changelog.xml
+++ b/database/migration/src/main/resources/changelog.xml
@@ -58,6 +58,7 @@
     <include file="changesets/embiggen_detritus_value.xml" relativeToChangelogFile="true" />
     <include file="changesets/standardize_column_names_patches.xml" relativeToChangelogFile="true" />
     <include file="changesets/nullable_lobs.xml" relativeToChangelogFile="true" />
+    <include file="changesets/add_attempt_in_call_caching_entry.xml" relativeToChangelogFile="true" />
 </databaseChangeLog>
 <!--
 
diff --git a/database/migration/src/main/resources/changesets/add_attempt_in_call_caching_entry.xml b/database/migration/src/main/resources/changesets/add_attempt_in_call_caching_entry.xml
new file mode 100644
index 000000000..ff109b721
--- /dev/null
+++ b/database/migration/src/main/resources/changesets/add_attempt_in_call_caching_entry.xml
@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
+
+    <changeSet id="add_attempt_in_call_caching_entry" author="tjeandet">
+        <addColumn tableName="CALL_CACHING_ENTRY">
+            <column name="JOB_ATTEMPT" type="INT" defaultValue="NULL">
+                <constraints nullable="true" />
+            </column>
+        </addColumn>
+    </changeSet>
+
+</databaseChangeLog>
diff --git a/database/sql/src/main/scala/cromwell/database/slick/CallCachingSlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/CallCachingSlickDatabase.scala
index deb7b6d5e..518939697 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/CallCachingSlickDatabase.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/CallCachingSlickDatabase.scala
@@ -3,9 +3,9 @@ package cromwell.database.slick
 import cats.data.NonEmptyList
 import cromwell.database.sql._
 import cromwell.database.sql.joins.CallCachingJoin
+import cromwell.database.sql.tables.CallCachingEntry
 
 import scala.concurrent.{ExecutionContext, Future}
-import scala.language.postfixOps
 
 trait CallCachingSlickDatabase extends CallCachingSqlDatabase {
   this: SlickDatabase =>
@@ -49,10 +49,12 @@ trait CallCachingSlickDatabase extends CallCachingSqlDatabase {
   }
 
   override def invalidateCall(callCachingEntryId: Int)
-                               (implicit ec: ExecutionContext): Future[Unit] = {
-    import cats.syntax.functor._
-    import cats.instances.future._
-    val action = dataAccess.allowResultReuseForCallCachingEntryId(callCachingEntryId).update(false)
-    runTransaction(action) void
+                               (implicit ec: ExecutionContext): Future[Option[CallCachingEntry]] = {
+    val action = for {
+      _ <- dataAccess.allowResultReuseForCallCachingEntryId(callCachingEntryId).update(false)
+      callCachingEntryOption <- dataAccess.callCachingEntriesForId(callCachingEntryId).result.headOption
+    } yield callCachingEntryOption
+    
+    runTransaction(action)
   }
 }
diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingEntryComponent.scala
index 5e37932e7..d692e57c1 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingEntryComponent.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingEntryComponent.scala
@@ -17,11 +17,13 @@ trait CallCachingEntryComponent {
 
     def jobIndex = column[Int]("JOB_INDEX")
 
+    def jobAttempt = column[Option[Int]]("JOB_ATTEMPT", O.Default(None))
+
     def returnCode = column[Option[Int]]("RETURN_CODE")
 
     def allowResultReuse = column[Boolean]("ALLOW_RESULT_REUSE", O.Default(true))
-
-    override def * = (workflowExecutionUuid, callFullyQualifiedName, jobIndex, returnCode, allowResultReuse,
+    
+    override def * = (workflowExecutionUuid, callFullyQualifiedName, jobIndex, jobAttempt, returnCode, allowResultReuse,
       callCachingEntryId.?) <> (CallCachingEntry.tupled, CallCachingEntry.unapply)
 
     def ucCallCachingEntryWeuCfqnJi =
diff --git a/database/sql/src/main/scala/cromwell/database/sql/CallCachingSqlDatabase.scala b/database/sql/src/main/scala/cromwell/database/sql/CallCachingSqlDatabase.scala
index b2b3d221a..ee38700ac 100644
--- a/database/sql/src/main/scala/cromwell/database/sql/CallCachingSqlDatabase.scala
+++ b/database/sql/src/main/scala/cromwell/database/sql/CallCachingSqlDatabase.scala
@@ -2,6 +2,7 @@ package cromwell.database.sql
 
 import cats.data.NonEmptyList
 import cromwell.database.sql.joins.CallCachingJoin
+import cromwell.database.sql.tables.CallCachingEntry
 
 import scala.concurrent.{ExecutionContext, Future}
 
@@ -15,5 +16,5 @@ trait CallCachingSqlDatabase {
                       (implicit ec: ExecutionContext): Future[Option[CallCachingJoin]]
 
   def invalidateCall(callCachingEntryId: Int)
-                    (implicit ec: ExecutionContext): Future[Unit]
+                    (implicit ec: ExecutionContext): Future[Option[CallCachingEntry]]
 }
diff --git a/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingEntry.scala b/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingEntry.scala
index 95a018256..f06bf96b6 100644
--- a/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingEntry.scala
+++ b/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingEntry.scala
@@ -5,6 +5,7 @@ case class CallCachingEntry
   workflowExecutionUuid: String,
   callFullyQualifiedName: String,
   jobIndex: Int,
+  jobAttempt: Option[Int],
   returnCode: Option[Int],
   allowResultReuse: Boolean,
   callCachingEntryId: Option[Int] = None
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala
index d2e76189d..20a2ee40c 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala
@@ -12,6 +12,7 @@ import cromwell.core._
 import cromwell.core.callcaching._
 import cromwell.core.logging.WorkflowLogging
 import cromwell.core.simpleton.WdlValueSimpleton
+import cromwell.database.sql.tables.CallCachingEntry
 import cromwell.engine.workflow.lifecycle.execution.CallPreparationActor.{BackendJobPreparationSucceeded, CallPreparationFailed}
 import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor._
 import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.{CacheHit, CacheMiss, CallCacheHashes, HashError}
@@ -57,11 +58,16 @@ class EngineJobExecutionActor(replyTo: ActorRef,
   private[execution] var executionToken: Option[JobExecutionToken] = None
 
   private val effectiveCallCachingKey = CacheMetadataKeyPrefix + "effectiveCallCachingMode"
+  private val callCachingReadResultMetadataKey = CacheMetadataKeyPrefix + "result"
+  private val callCachingHitResultMetadataKey = CacheMetadataKeyPrefix + "hit"
+  private val callCachingAllowReuseMetadataKey = CacheMetadataKeyPrefix + "allowResultReuse"
 
   implicit val ec: ExecutionContext = context.dispatcher
 
-  log.debug(s"$tag: $effectiveCallCachingKey: $effectiveCallCachingMode")
-  writeCallCachingModeToMetadata()
+  override def preStart() = {
+    log.debug(s"$tag: $effectiveCallCachingKey: $effectiveCallCachingMode")
+    writeCallCachingModeToMetadata()
+  }
 
   startWith(Pending, NoData)
   private var eventList: Seq[ExecutionEvent] = Seq(ExecutionEvent(stateName.toString))
@@ -123,8 +129,6 @@ class EngineJobExecutionActor(replyTo: ActorRef,
       respondAndStop(JobFailedNonRetryableResponse(jobKey, throwable, None))
   }
 
-  private val callCachingReadResultMetadataKey = CacheMetadataKeyPrefix + "result"
-  private val callCachingHitResultMetadataKey = CacheMetadataKeyPrefix + "hit"
   when(CheckingCallCache) {
     case Event(CacheMiss, data: ResponsePendingData) =>
       writeToMetadata(Map(
@@ -300,6 +304,7 @@ class EngineJobExecutionActor(replyTo: ActorRef,
 
   def writeCallCachingModeToMetadata(): Unit = {
     writeToMetadata(Map(effectiveCallCachingKey -> effectiveCallCachingMode.toString))
+    writeToMetadata(Map(callCachingAllowReuseMetadataKey -> effectiveCallCachingMode.writeToCache))
   }
 
   def createJobPreparationActor(jobPrepProps: Props, name: String): ActorRef = context.actorOf(jobPrepProps, name)
@@ -360,8 +365,21 @@ class EngineJobExecutionActor(replyTo: ActorRef,
   }
 
   private def handleCacheInvalidatedResponse(response: CallCacheInvalidatedResponse, data: ResponsePendingData) = {
+    def updateMetadataForInvalidatedEntry(entry: CallCachingEntry) = {
+      import cromwell.services.metadata.MetadataService.implicits.MetadataAutoPutter
+      import cromwell.core.ExecutionIndex._
+      
+      val workflowId = WorkflowId.fromString(entry.workflowExecutionUuid)
+      // If the entry doesn't have an attempt, it means that this cache entry was added before this change
+      // and we don't know which attempt yielded this cache entry
+      // In that case make a best effort and update the first attempt
+      val key = Option((entry.callFullyQualifiedName, entry.jobIndex.toIndex, entry.jobAttempt.getOrElse(1)))
+      serviceRegistryActor.putMetadataWithRawKey(workflowId, key, Map(callCachingAllowReuseMetadataKey -> false))
+    }
+    
     response match {
       case CallCacheInvalidatedFailure(failure) => log.error(failure, "Failed to invalidate cache entry for job: {}", jobDescriptorKey)
+      case CallCacheInvalidatedSuccess(Some(entry)) => updateMetadataForInvalidatedEntry(entry)
       case _ =>
     }
 
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala
index f02339720..f165fc5ea 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala
@@ -26,6 +26,7 @@ class CallCache(database: CallCachingSqlDatabase) {
       workflowExecutionUuid = workflowId.toString,
       callFullyQualifiedName = response.jobKey.call.fullyQualifiedName,
       jobIndex = response.jobKey.index.fromIndex,
+      jobAttempt = Option(response.jobKey.attempt),
       returnCode = response.returnCode,
       allowResultReuse = true)
     val hashes = callCacheHashes.hashes
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheInvalidateActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheInvalidateActor.scala
index db2465f9f..ebea293ea 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheInvalidateActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheInvalidateActor.scala
@@ -2,6 +2,7 @@ package cromwell.engine.workflow.lifecycle.execution.callcaching
 
 import akka.actor.{Actor, ActorLogging, Props}
 import cromwell.core.Dispatcher.EngineDispatcher
+import cromwell.database.sql.tables.CallCachingEntry
 
 import scala.concurrent.ExecutionContext
 import scala.util.{Failure, Success}
@@ -13,8 +14,8 @@ class CallCacheInvalidateActor(callCache: CallCache, cacheId: CallCachingEntryId
   def receiver = context.parent
 
   callCache.invalidate(cacheId) onComplete {
-    case Success(_) =>
-      receiver ! CallCacheInvalidatedSuccess
+    case Success(maybeEntry) =>
+      receiver ! CallCacheInvalidatedSuccess(maybeEntry)
       context.stop(self)
     case Failure(t) =>
       receiver ! CallCacheInvalidatedFailure(t)
@@ -33,6 +34,6 @@ object CallCacheInvalidateActor {
 }
 
 sealed trait CallCacheInvalidatedResponse
-case object CallCacheInvalidatedSuccess extends CallCacheInvalidatedResponse
+case class CallCacheInvalidatedSuccess(maybeEntry: Option[CallCachingEntry]) extends CallCacheInvalidatedResponse
 case object CallCacheInvalidationUnnecessary extends CallCacheInvalidatedResponse
 case class CallCacheInvalidatedFailure(t: Throwable) extends CallCacheInvalidatedResponse
\ No newline at end of file
diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaInvalidatingCacheEntrySpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaInvalidatingCacheEntrySpec.scala
index 40e43d77b..ce6537824 100644
--- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaInvalidatingCacheEntrySpec.scala
+++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaInvalidatingCacheEntrySpec.scala
@@ -14,7 +14,7 @@ class EjeaInvalidatingCacheEntrySpec extends EngineJobExecutionActorSpec with Ca
   "An EJEA in InvalidatingCacheEntry state" should {
 
     val invalidationErrorCause = new Exception("blah")
-    val invalidateSuccess = CallCacheInvalidatedSuccess
+    val invalidateSuccess = CallCacheInvalidatedSuccess(None)
     val invalidateFailure = CallCacheInvalidatedFailure(invalidationErrorCause)
 
     val metaInfo31: CallCachingEntryId = CallCachingEntryId(31)
diff --git a/services/src/main/scala/cromwell/services/metadata/MetadataService.scala b/services/src/main/scala/cromwell/services/metadata/MetadataService.scala
index 332793d4b..3eb13fbe3 100644
--- a/services/src/main/scala/cromwell/services/metadata/MetadataService.scala
+++ b/services/src/main/scala/cromwell/services/metadata/MetadataService.scala
@@ -4,7 +4,7 @@ import java.time.OffsetDateTime
 
 import akka.actor.ActorRef
 import cats.data.NonEmptyList
-import cromwell.core.{JobKey, WorkflowId, WorkflowState}
+import cromwell.core.{FullyQualifiedName, JobKey, WorkflowId, WorkflowState}
 import cromwell.services.ServiceRegistryActor.ServiceRegistryMessage
 import lenthall.exception.ThrowableAggregation
 import wdl4s.values._
@@ -46,6 +46,16 @@ object MetadataService {
         }
         serviceRegistryActor ! PutMetadataAction(events)
       }
+
+      def putMetadataWithRawKey(workflowId: WorkflowId, jobKey: Option[(FullyQualifiedName, Option[Int], Int)], keyValue: Map[String, Any]) = {
+        val metadataJobKey = jobKey map { case (fullyQualifiedName, index, attempt) => MetadataJobKey(fullyQualifiedName, index, attempt) }
+
+        val events = keyValue map { case (key, value) =>
+          val metadataKey = MetadataKey(workflowId, metadataJobKey, key)
+          MetadataEvent(metadataKey, MetadataValue(value))
+        }
+        serviceRegistryActor ! PutMetadataAction(events)
+      }
     }
   }
 

From 5588e731856ad5044f7a45bc952480071f727678 Mon Sep 17 00:00:00 2001
From: Thib <Horneth@users.noreply.github.com>
Date: Wed, 22 Feb 2017 13:32:46 -0500
Subject: [PATCH 68/84] =?UTF-8?q?Evaluate=20imports=20file=20content=20eag?=
 =?UTF-8?q?erly.=20Fixes=20"workflowName"=20centaur=20f=E2=80=A6=20(#2013)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Evaluate imports file content eagerly. Fixes "workflowName" centaur failures.

* add retries

* fix compilation
---
 .../MaterializeWorkflowDescriptorActor.scala       | 32 +++++++++++-----------
 .../metadata/impl/MetadataServiceActor.scala       | 17 ++++++++++--
 .../metadata/impl/WriteMetadataActor.scala         | 13 ++++++---
 3 files changed, 40 insertions(+), 22 deletions(-)

diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala
index 270140d06..278edc2d7 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala
@@ -335,11 +335,11 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef,
   }
 
   private def validateNamespaceWithImports(w: WorkflowSourceFilesWithDependenciesZip): ErrorOr[WdlNamespaceWithWorkflow] = {
-    def getMetadatae(importsDir: Path, prefix: String = ""): Seq[(String, Path)] = {
-      importsDir.children.toSeq flatMap {
+    def getMetadatae(importsDir: Path, prefix: String = ""): List[(String, Path)] = {
+      importsDir.children.toList flatMap {
         case f: Path if f.isDirectory => getMetadatae(f, prefix + f.name + "/")
-        case f: Path if f.name.endsWith(".wdl") => Seq((prefix + f.name, f))
-        case _ => Seq.empty
+        case f: Path if f.name.endsWith(".wdl") => List((prefix + f.name, f))
+        case _ => List.empty
       }
     }
 
@@ -352,19 +352,19 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef,
     }
 
     def importsAsNamespace(importsDir: Path): ErrorOr[WdlNamespaceWithWorkflow] = {
-        writeMetadatae(importsDir)
-        val importsDirFile = better.files.File(importsDir.pathAsString) // For wdl4s better file compatibility
+      writeMetadatae(importsDir)
+      val importsDirFile = better.files.File(importsDir.pathAsString) // For wdl4s better file compatibility
       val importResolvers: Seq[ImportResolver] = if (importLocalFilesystem) {
-          List(WdlNamespace.directoryResolver(importsDirFile), WdlNamespace.fileResolver)
-        } else {
-          List(WdlNamespace.directoryResolver(importsDirFile))
-        }
-        val results = WdlNamespaceWithWorkflow.load(w.wdlSource, importResolvers)
-        importsDir.delete(swallowIOExceptions = true)
-        results match {
-          case Success(ns) => ns.validNel
-          case Failure(f) => f.getMessage.invalidNel
-        }
+        List(WdlNamespace.directoryResolver(importsDirFile), WdlNamespace.fileResolver)
+      } else {
+        List(WdlNamespace.directoryResolver(importsDirFile))
+      }
+      val results = WdlNamespaceWithWorkflow.load(w.wdlSource, importResolvers)
+      importsDir.delete(swallowIOExceptions = true)
+      results match {
+        case Success(ns) => ns.validNel
+        case Failure(f) => f.getMessage.invalidNel
+      }
     }
 
     validateImportsDirectory(w.importsZip) flatMap importsAsNamespace
diff --git a/services/src/main/scala/cromwell/services/metadata/impl/MetadataServiceActor.scala b/services/src/main/scala/cromwell/services/metadata/impl/MetadataServiceActor.scala
index 34e6502f5..4c343d173 100644
--- a/services/src/main/scala/cromwell/services/metadata/impl/MetadataServiceActor.scala
+++ b/services/src/main/scala/cromwell/services/metadata/impl/MetadataServiceActor.scala
@@ -2,7 +2,8 @@ package cromwell.services.metadata.impl
 
 import java.util.UUID
 
-import akka.actor.{Actor, ActorLogging, ActorRef, Props}
+import akka.actor.SupervisorStrategy.{Decider, Directive, Escalate, Resume}
+import akka.actor.{Actor, ActorContext, ActorInitializationException, ActorLogging, ActorRef, OneForOneStrategy, Props}
 import com.typesafe.config.{Config, ConfigFactory}
 import cromwell.core.Dispatcher.ServiceDispatcher
 import cromwell.core.WorkflowId
@@ -28,7 +29,19 @@ object MetadataServiceActor {
 
 case class MetadataServiceActor(serviceConfig: Config, globalConfig: Config)
   extends Actor with ActorLogging with MetadataDatabaseAccess with SingletonServicesStore {
-
+  
+  private val decider: Decider = {
+    case _: ActorInitializationException => Escalate
+    case _ => Resume
+  }
+  
+  override val supervisorStrategy = new OneForOneStrategy()(decider) {
+    override def logFailure(context: ActorContext, child: ActorRef, cause: Throwable, decision: Directive) = {
+      val childName = if (child == readActor) "Read" else "Write"
+      log.error(s"The $childName Metadata Actor died unexpectedly, metadata events might have been lost. Restarting it...", cause)
+    }
+  }
+  
   private val summaryActor: Option[ActorRef] = buildSummaryActor
 
   val readActor = context.actorOf(ReadMetadataActor.props(), "read-metadata-actor")
diff --git a/services/src/main/scala/cromwell/services/metadata/impl/WriteMetadataActor.scala b/services/src/main/scala/cromwell/services/metadata/impl/WriteMetadataActor.scala
index cb37d0bcb..b4db0320e 100644
--- a/services/src/main/scala/cromwell/services/metadata/impl/WriteMetadataActor.scala
+++ b/services/src/main/scala/cromwell/services/metadata/impl/WriteMetadataActor.scala
@@ -7,9 +7,10 @@ import cromwell.services.SingletonServicesStore
 import cromwell.services.metadata.MetadataEvent
 import cromwell.services.metadata.MetadataService.PutMetadataAction
 import cromwell.services.metadata.impl.WriteMetadataActor.{WriteMetadataActorData, WriteMetadataActorState}
+import org.slf4j.LoggerFactory
 
 import scala.concurrent.duration._
-import scala.util.{Failure, Success}
+import scala.util.{Failure, Success, Try}
 
 final case class WriteMetadataActor(batchRate: Int, flushRate: FiniteDuration)
   extends LoggingFSM[WriteMetadataActorState, WriteMetadataActorData] with ActorLogging with
@@ -64,6 +65,7 @@ final case class WriteMetadataActor(batchRate: Int, flushRate: FiniteDuration)
 
 object WriteMetadataActor {
   def props(batchRate: Int, flushRate: FiniteDuration) = Props(new WriteMetadataActor(batchRate, flushRate)).withDispatcher(ServiceDispatcher)
+  private lazy val logger = LoggerFactory.getLogger("WriteMetadataActor")
 
   sealed trait WriteMetadataActorMessage
   case object DbWriteComplete extends WriteMetadataActorMessage
@@ -76,14 +78,17 @@ object WriteMetadataActor {
 
   sealed trait WriteMetadataActorData {
     def addEvents(newEvents: Iterable[MetadataEvent]): WriteMetadataActorData = {
-      NonEmptyVector.fromVector(newEvents.toVector) match {
-        case Some(v) =>
+      Try(NonEmptyVector.fromVector(newEvents.toVector)) match {
+        case Success(Some(v)) =>
           val newEvents = this match {
             case NoEvents => v
             case HasEvents(e) => e.concatNev(v)
           }
           HasEvents(newEvents)
-        case None => this
+        case Success(None) => this
+        case Failure(f) =>
+          logger.error("Failed processing metadata events. Events will be dropped and not be sent to the database.", f)
+          this
       }
     }
   }

From c746ab7d8f468d3dccc1ac9842439ee880c3b684 Mon Sep 17 00:00:00 2001
From: Khalid Shakir <kshakir@broadinstitute.org>
Date: Wed, 22 Feb 2017 00:56:01 -0500
Subject: [PATCH 69/84] Better NPE handling during terminal JES parsing.

---
 .../main/scala/cromwell/backend/impl/jes/Run.scala |  42 +++---
 .../scala/cromwell/backend/impl/jes/RunSpec.scala  | 154 +++++++++++++++++++--
 2 files changed, 172 insertions(+), 24 deletions(-)

diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala
index 2de9ebc8e..70047bfe7 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala
@@ -8,8 +8,8 @@ import com.google.api.services.genomics.Genomics
 import com.google.api.services.genomics.model._
 import cromwell.backend.BackendJobDescriptor
 import cromwell.backend.impl.jes.RunStatus.{Failed, Initializing, Running, Success}
-import cromwell.core.labels.Labels
 import cromwell.core.ExecutionEvent
+import cromwell.core.labels.Labels
 import cromwell.core.logging.JobLogger
 import org.slf4j.LoggerFactory
 
@@ -117,22 +117,32 @@ object Run {
   }
 
   def interpretOperationStatus(op: Operation): RunStatus = {
-    if (op.getDone) {
-      lazy val eventList = getEventList(op)
-      lazy val ceInfo = op.getMetadata.get ("runtimeMetadata").asInstanceOf[GArrayMap[String,Object]].get("computeEngine").asInstanceOf[GArrayMap[String, String]]
-      lazy val machineType = Option(ceInfo.get("machineType"))
-      lazy val instanceName = Option(ceInfo.get("instanceName"))
-      lazy val zone = Option(ceInfo.get("zone"))
-
-      // If there's an error, generate a Failed status. Otherwise, we were successful!
-      Option(op.getError) match {
-        case None => Success(eventList, machineType, zone, instanceName)
-        case Some(error) => Failed(error.getCode, Option(error.getMessage), eventList, machineType, zone, instanceName)
+    require(op != null, "Operation must not be null.")
+    try {
+      if (op.getDone) {
+        lazy val eventList = getEventList(op)
+        lazy val computeEngineOption = for {
+          runtimeMetadata <- op.getMetadata.asScala.get("runtimeMetadata")
+          computeEngine <- runtimeMetadata.asInstanceOf[GArrayMap[String, Object]].asScala.get("computeEngine")
+        } yield computeEngine.asInstanceOf[GArrayMap[String, String]].asScala
+        lazy val machineType = computeEngineOption.flatMap(_.get("machineType"))
+        lazy val instanceName = computeEngineOption.flatMap(_.get("instanceName"))
+        lazy val zone = computeEngineOption.flatMap(_.get("zone"))
+
+        // If there's an error, generate a Failed status. Otherwise, we were successful!
+        Option(op.getError) match {
+          case None => Success(eventList, machineType, zone, instanceName)
+          case Some(error) =>
+            Failed(error.getCode, Option(error.getMessage), eventList, machineType, zone, instanceName)
+        }
+      } else if (op.hasStarted) {
+        Running
+      } else {
+        Initializing
       }
-    } else if (op.hasStarted) {
-      Running
-    } else {
-      Initializing
+    } catch {
+      case npe: NullPointerException =>
+        throw new RuntimeException(s"Caught NPE while processing operation ${op.getName}: $op", npe)
     }
   }
 
diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/RunSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/RunSpec.scala
index 5398ed66c..7deec7f84 100644
--- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/RunSpec.scala
+++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/RunSpec.scala
@@ -5,6 +5,7 @@ import java.util
 
 import com.google.api.client.util.ArrayMap
 import com.google.api.services.genomics.model.Operation
+import cromwell.backend.impl.jes.RunStatus.Success
 import cromwell.core.ExecutionEvent
 import org.scalatest.{FlatSpec, Matchers}
 import org.specs2.mock.{Mockito => MockitoTrait}
@@ -12,9 +13,149 @@ import org.specs2.mock.{Mockito => MockitoTrait}
 import scala.collection.JavaConverters._
 
 class RunSpec extends FlatSpec with Matchers with MockitoTrait {
-  "JES Run" should "parse events from Operation metadata" in {
+  behavior of "JES Run"
+
+  it should "parse events from Operation metadata" in {
     val op: Operation = new Operation()
+    op.setMetadata(eventsMetadata.asJava)
+
+    val list = Run.getEventList(op)
+    list should contain theSameElementsAs eventsExpected
+  }
+
+  it should "require operation be non-null" in {
+    val exception = intercept[RuntimeException](Run.interpretOperationStatus(null))
+    exception.getMessage should be("requirement failed: Operation must not be null.")
+  }
+
+  it should "catch and wrap null pointer exceptions in an empty operation" in {
+    val op = new Operation()
+
+    val exception = intercept[RuntimeException](Run.interpretOperationStatus(op))
+    exception.getMessage should be("Caught NPE while processing operation null: {}")
+  }
+
+  it should "catch and wrap null pointer exceptions in a name only operation" in {
+    val op = new Operation()
+    op.setName("my/customName")
+
+    val exception = intercept[RuntimeException](Run.interpretOperationStatus(op))
+    exception.getMessage should be("Caught NPE while processing operation my/customName: {name=my/customName}")
+  }
+
+  it should "parse an operation without machine information" in {
+    val op = new Operation()
+    op.setName("my/customName")
+    op.setDone(true)
+    op.setMetadata(eventsMetadata.asJava)
+
+    val runStatus = Run.interpretOperationStatus(op)
+
+    runStatus should be(a[Success])
+
+    val success = runStatus.asInstanceOf[Success]
+    success.instanceName should be(None)
+    success.machineType should be(None)
+    success.zone should be(None)
+    success.eventList should contain theSameElementsAs eventsExpected
+  }
+
+  it should "parse an operation with empty runtime metadata" in {
+    val op = new Operation()
+    op.setName("my/customName")
+    op.setDone(true)
+
+    val runtimeMetadata = ArrayMap.create[String, Object]()
+
+    val metadata = eventsMetadata ++ Map("runtimeMetadata" -> runtimeMetadata)
+    op.setMetadata(metadata.asJava)
+
+    val runStatus = Run.interpretOperationStatus(op)
+    runStatus should be(a[Success])
+
+    val success = runStatus.asInstanceOf[Success]
+    success.instanceName should be(None)
+    success.machineType should be(None)
+    success.zone should be(None)
+    success.eventList should contain theSameElementsAs eventsExpected
+  }
+
+  it should "parse an operation with empty compute engine information" in {
+    val op = new Operation()
+    op.setName("my/customName")
+    op.setDone(true)
+
+    val computeEngine = ArrayMap.create[String, String]()
+
+    val runtimeMetadata = ArrayMap.create[String, Object]()
+    runtimeMetadata.add("computeEngine", computeEngine)
+
+    val metadata = eventsMetadata ++ Map("runtimeMetadata" -> runtimeMetadata)
+    op.setMetadata(metadata.asJava)
+
+    val runStatus = Run.interpretOperationStatus(op)
+    runStatus should be(a[Success])
+
+    val success = runStatus.asInstanceOf[Success]
+    success.instanceName should be(None)
+    success.machineType should be(None)
+    success.zone should be(None)
+    success.eventList should contain theSameElementsAs eventsExpected
+  }
+
+  it should "parse an operation with partially filled compute engine" in {
+    val op = new Operation()
+    op.setName("my/customName")
+    op.setDone(true)
+
+    val computeEngine = ArrayMap.create[String, String]()
+    computeEngine.add("zone", "us-central1-b")
+
+    val runtimeMetadata = ArrayMap.create[String, Object]()
+    runtimeMetadata.add("computeEngine", computeEngine)
+
+    val metadata = eventsMetadata ++ Map("runtimeMetadata" -> runtimeMetadata)
+    op.setMetadata(metadata.asJava)
+
+    val runStatus = Run.interpretOperationStatus(op)
+
+    runStatus should be(a[Success])
+
+    val success = runStatus.asInstanceOf[Success]
+    success.instanceName should be(None)
+    success.machineType should be(None)
+    success.zone should be(Option("us-central1-b"))
+    success.eventList should contain theSameElementsAs eventsExpected
+  }
+
+  it should "parse an operation with filled compute engine" in {
+    val op = new Operation()
+    op.setName("my/customName")
+    op.setDone(true)
 
+    val computeEngine = ArrayMap.create[String, String]()
+    computeEngine.add("instanceName", "ggp-12345678901234567890")
+    computeEngine.add("machineType", "us-central1-b/g1-small")
+    computeEngine.add("zone", "us-central1-b")
+
+    val runtimeMetadata = ArrayMap.create[String, Object]()
+    runtimeMetadata.add("computeEngine", computeEngine)
+
+    val metadata = eventsMetadata ++ Map("runtimeMetadata" -> runtimeMetadata)
+    op.setMetadata(metadata.asJava)
+
+    val runStatus = Run.interpretOperationStatus(op)
+
+    runStatus should be(a[Success])
+
+    val success = runStatus.asInstanceOf[Success]
+    success.instanceName should be(Option("ggp-12345678901234567890"))
+    success.machineType should be(Option("us-central1-b/g1-small"))
+    success.zone should be(Option("us-central1-b"))
+    success.eventList should contain theSameElementsAs eventsExpected
+  }
+
+  private lazy val eventsMetadata: Map[String, AnyRef] = {
     val event1: ArrayMap[String, String] = ArrayMap.create(2)
     event1.add("description", "start")
     event1.add("startTime", "2015-12-05T00:00:01+00:00")
@@ -25,22 +166,19 @@ class RunSpec extends FlatSpec with Matchers with MockitoTrait {
 
     val events = new util.ArrayList(Seq(event1, event2).asJava)
 
-    val metadata: Map[String, AnyRef] = Map(
+    Map(
       "createTime" -> "2015-12-05T00:00:00+00:00",
       "startTime" -> "2015-12-05T00:00:01+00:00",
       "endTime" -> "2015-12-05T11:00:00+00:00",
       "events" -> events
     )
+  }
 
-    op.setMetadata(metadata.asJava)
-
-    val list = Run.getEventList(op)
-    list should contain theSameElementsAs List(
+  private lazy val eventsExpected =
+    List(
       ExecutionEvent("waiting for quota", OffsetDateTime.parse("2015-12-05T00:00:00+00:00")),
       ExecutionEvent("initializing VM", OffsetDateTime.parse("2015-12-05T00:00:01+00:00")),
       ExecutionEvent("start", OffsetDateTime.parse("2015-12-05T00:00:01+00:00")),
       ExecutionEvent("cromwell poll interval", OffsetDateTime.parse("2015-12-05T11:00:00+00:00"))
     )
-
-  }
 }

From 29d181adb176e318c8d2052d05044469bc498641 Mon Sep 17 00:00:00 2001
From: Khalid Shakir <kshakir@broadinstitute.org>
Date: Wed, 22 Feb 2017 12:44:22 -0500
Subject: [PATCH 70/84] Removed MakingABackend.MD

---
 MakingABackend.MD | 197 ------------------------------------------------------
 1 file changed, 197 deletions(-)
 delete mode 100644 MakingABackend.MD

diff --git a/MakingABackend.MD b/MakingABackend.MD
deleted file mode 100644
index 5106b3726..000000000
--- a/MakingABackend.MD
+++ /dev/null
@@ -1,197 +0,0 @@
-# Making a backend
-
-## Part 0: Introduction
-
-- These notes were added while making a new AWS backend for Amazon AWS.
-
-## Part 1 (October 13 2016): The skeleton:
-
-To start with, I just need to create a bunch of boilerplate which will eventually be filled in with all of the lovely AWS details!
-
-### Defining the awsBackend project:
-
-- Added entries to `project/Settings.scala`, `project/Dependencies.scala` and `build.sbt`
-- This was mainly just a copy/paste from existing backend projects. I made a few typos renaming everything and linking the dependencies properly though!
-- E.g. In my first commit I forgot to update the libraryDependencies name for my AWS backend project:
-```
-  val awsBackendSettings = List(
-    name := "cromwell-aws-backend",
-    libraryDependencies ++= awsBackendDependencies
-  ) ++ commonSettings
-```
-- I guessed that I'd need the AWS SDK so I included that immediately in Dependencies.scala:
-```
-  val awsBackendDependencies = List(
-    "com.amazonaws" % "aws-java-sdk" % "1.11.41"
-  )
-```
-- In build.scala I had to also edit the `lazy val root` to include a new `.aggregate(awsBackend)` and a new `.dependsOn(awsBackend)`
-
-### Directory structure:
-
-- This is probably going to be autogenerated for you in the directories specified in the above files. I'd already added my own directory structure and sbt managed to pick it up correctly in `supportedBackends/aws`.
-
-### AWS Job Execution Actor:
-- To run a job, Cromwell needs to instantiate a Job Execution actor. I'll fill in the details later but for now, I'll just add the constructor, props, and an unimplemented method definition for `execute`:
-```
-class AwsJobExecutionActor(override val jobDescriptor: BackendJobDescriptor,
-                           override val configurationDescriptor: BackendConfigurationDescriptor) extends BackendJobExecutionActor {
-
-  override def execute: Future[BackendJobExecutionResponse] = ???
-}
-
-object AwsJobExecutionActor {
-  def props(jobDescriptor: BackendJobDescriptor,
-            configurationDescriptor: BackendConfigurationDescriptor): Props = Props(new AwsJobExecutionActor(jobDescriptor, configurationDescriptor))
-}
-```
-
-### Actor factory:
-- This is the class which tells Cromwell which classes represent job execution actors, initialization actors and so on. I'm just adding a skeleton for now, with a constructor of the form the Cromwell expects:
-```
-case class AwsBackendActorFactory(name: String, configurationDescriptor: BackendConfigurationDescriptor) extends BackendLifecycleActorFactory {
-
-  override def jobExecutionActorProps(jobDescriptor: BackendJobDescriptor,
-                                      initializationData: Option[BackendInitializationData],
-                                      serviceRegistryActor: ActorRef,
-                                      backendSingletonActor: Option[ActorRef]): Props = AwsJobExecutionActor.props(jobDescriptor, configurationDescriptor)
-}
-```
-- There are a few other actor definitions that can be added to this file over time. But the only one that Cromwell *requires* to work is the job execution actor.
-
-### Reference conf:
-
-- Reference.conf is a set of reference options which shows people how to enable the backends that they want. So I'll add the initial config which people would add if they wanted the AWS backend (commented out in the reference so it's not enabled by default). This goes below all the other backend references:
-```
-    #AWS {
-    #  actor-factory = "cromwell.backend.impl.aws.AwsBackendActorFactory"
-    #  config {
-    #
-    #  }
-    #}
-```
-
-### Application.conf
-
-- OK so I've now told people how to add this backend... Now I actually add it to my own personal configuration file so I can try it out!
-```
-backend {
-  default = "AWS"
-  providers {
-    AWS {
-      actor-factory = "cromwell.backend.impl.aws.AwsBackendActorFactory"
-      config {
-      
-      }
-    }
-  }
-}
-```
-
-### Trying it out
-So we now have a backend skeleton! What happens when we run it? Well hopefully Cromwell will instantiate the backend far enough to reach the unimplemented execute method and then fall over. Let's give it a go!
-- I fire up cromwell in server mode with my modified application.conf.
-- I create a sample WDL that would sleep for 20 seconds if it actually worked:
-The input WDL:
-```
-task sleep {
-  command { sleep 20 }
-}
-workflow main {
-  call sleep
-}
-```
-- I submit the WDL to the swagger endpoint (http://localhost:8000/swagger/index.html?url=/swagger/cromwell.yaml) and watch the server logs...
-- And as expected:
-```
-2016-10-13 13:14:29,017 cromwell-system-akka.dispatchers.engine-dispatcher-39 INFO  - MaterializeWorkflowDescriptorActor [UUID(ddd827ba)]: Call-to-Backend assignments: main.sleep -> AWS
-2016-10-13 13:14:30,167 cromwell-system-akka.dispatchers.engine-dispatcher-39 INFO  - WorkflowExecutionActor-ddd827ba-091f-4c6f-b98f-cc9825717007 [UUID(ddd827ba)]: Starting calls: main.sleep:NA:1
-2016-10-13 13:14:30,983 cromwell-system-akka.actor.default-dispatcher-5 ERROR - guardian failed, shutting down system
-scala.NotImplementedError: an implementation is missing
-  at scala.Predef$.$qmark$qmark$qmark(Predef.scala:230)
-  at cromwell.backend.impl.aws.AwsJobExecutionActor.execute(AwsJobExecutionActor.scala:12)
-```
-- OK, so now I just need to implement `execute(): Future[JobExecutionResult]` and Cromwell can interface with AWS. How hard can it be!
-
-## Part 2 (October 13 2016): Using Amazon to sleep 20 seconds
-
-### Starting point
-- This was a learning experience after using the Google pipelines service to submit jobs! 
-- To get myself started, I've manually created an ECS cluster which I've called `ecs-t2micro-cluster` via the ECS web console.
-
-### Trial and Error
-
-- I see in the aws sdk docs that there's an AmazonECSAsyncClient class. That sounds promising! Luckily I already added the dependency on AWS SDK in Part 1 so I guess I can just write something basic in my AwsJobExecutionActor class and see what happens:
-
-- I ended up having to add some credentials options to the configuration file. The new `reference.conf` now looks like:
-```
-    #AWS {
-    #  actor-factory = "cromwell.backend.impl.aws.AwsBackendActorFactory"
-    #  config {
-    #    ## These two settings are required to authenticate with the ECS service:
-    #    accessKeyId = "..."
-    #    secretKey = "..."
-    #  }
-    #}
-``` 
-
-- After a little bit of experimentation with the ECS API, I was able to come up with a backend that works but is very limited... It is entirely synchronous in the `execute` method. That's certainly not a final answer but it works OK for running a single task. And we can now run that single `sleep` command successfully on the Amazon EC2 Container Service!
-  - The synchronous `execute` method:
-```
-class AwsJobExecutionActor(override val jobDescriptor: BackendJobDescriptor,
-                           override val configurationDescriptor: BackendConfigurationDescriptor) extends BackendJobExecutionActor {
-
-  val awsAccessKeyId = configurationDescriptor.backendConfig.as[String]("accessKeyId")
-  val awsSecretKey = configurationDescriptor.backendConfig.as[String]("secretKey")
-
-  val clusterName = "ecs-t2micro-cluster"
-
-  val credentials = new AWSCredentials {
-    override def getAWSAccessKeyId: String = awsAccessKeyId
-    override def getAWSSecretKey: String = awsSecretKey
-  }
-  val ecsAsyncClient = new AmazonECSAsyncClient(credentials)
-
-  override def execute: Future[BackendJobExecutionResponse] = {
-
-    val commandOverride = new ContainerOverride().withName("simple-app").withCommand(jobDescriptor.call.instantiateCommandLine(Map.empty, OnlyPureFunctions, identity).get)
-
-    val runRequest: RunTaskRequest = new RunTaskRequest()
-      .withCluster(clusterName)
-      .withCount(1)
-      .withTaskDefinition("ubuntuTask:1")
-      .withOverrides(new TaskOverride().withContainerOverrides(commandOverride))
-
-    val submitResultHandler = new AwsSdkAsyncHandler[RunTaskRequest, RunTaskResult]()
-    val _ = ecsAsyncClient.runTaskAsync(runRequest, submitResultHandler)
-
-    submitResultHandler.future map {
-      case AwsSdkAsyncResult(_, result) =>
-        log.info("AWS submission completed:\n{}", result.toString)
-        val taskArn= result.getTasks.asScala.head.getTaskArn
-        val taskDescription = waitUntilDone(taskArn)
-
-        log.info("AWS task completed!\n{}", taskDescription.toString)
-        SucceededResponse(jobDescriptor.key, Option(0), Map.empty, None, Seq.empty)
-    }
-  }
-
-  private def waitUntilDone(taskArn: String): Task = {
-    val describeTasksRequest = new DescribeTasksRequest().withCluster(clusterName).withTasks(List(taskArn).asJava)
-
-    val resultHandler = new AwsSdkAsyncHandler[DescribeTasksRequest, DescribeTasksResult]()
-    val _ = ecsAsyncClient.describeTasksAsync(describeTasksRequest, resultHandler)
-
-    val desribedTasks = Await.result(resultHandler.future, Duration.Inf)
-    val taskDescription = desribedTasks.result.getTasks.asScala.head
-    if (taskDescription.getLastStatus == DesiredStatus.STOPPED.toString) {
-      taskDescription
-    } else {
-      Thread.sleep(200)
-      waitUntilDone(taskArn)
-    }
-  }
-}
-```
-
-

From daf7dd00107a7328e4c297ce5c1312ba8b9b636e Mon Sep 17 00:00:00 2001
From: Khalid Shakir <kshakir@broadinstitute.org>
Date: Thu, 23 Feb 2017 09:31:05 -0500
Subject: [PATCH 71/84] Changed TES centaur log level from debug to info.

---
 src/bin/travis/resources/tes.conf | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/bin/travis/resources/tes.conf b/src/bin/travis/resources/tes.conf
index 33974e302..c32460328 100644
--- a/src/bin/travis/resources/tes.conf
+++ b/src/bin/travis/resources/tes.conf
@@ -1,4 +1,5 @@
 HttpPort: 9000
+LogLevel: info
 Storage:
   - Local:
       AllowedDirs:
@@ -10,4 +11,5 @@ Schedulers:
   Local:
     NumWorkers: 4
 Worker:
+  LogLevel: info
   Timeout: 1

From 422f9da40e74d377b839c38954085774cb105010 Mon Sep 17 00:00:00 2001
From: Thib <Horneth@users.noreply.github.com>
Date: Thu, 23 Feb 2017 13:00:44 -0500
Subject: [PATCH 72/84] Docker Hash (#1969)

* Docker Hash
---
 CHANGELOG.md                                       |   9 +
 README.md                                          |  36 +++
 .../backend/BackendLifecycleActorFactory.scala     |   6 +
 .../src/main/scala/cromwell/backend/backend.scala  |   4 +-
 .../standard/StandardCachingActorHelper.scala      |   3 +-
 .../test/scala/cromwell/backend/BackendSpec.scala  |   9 +-
 core/src/main/resources/reference.conf             |   9 +
 .../core/callcaching/CallCachingEligibility.scala  |  22 ++
 .../core/callcaching/docker/DockerFlow.scala       |  12 +
 .../core/callcaching/docker/DockerHashActor.scala  | 209 ++++++++++++++++
 .../callcaching/docker/DockerHashRequest.scala     |   3 +
 .../core/callcaching/docker/DockerHashResult.scala |  17 ++
 .../callcaching/docker/DockerImageIdentifier.scala |  83 ++++++
 .../registryv2/DockerRegistryV2AbstractFlow.scala  | 278 +++++++++++++++++++++
 .../docker/registryv2/flows/FlowUtils.scala        |  34 +++
 .../registryv2/flows/HttpFlowWithRetry.scala       | 120 +++++++++
 .../registryv2/flows/dockerhub/DockerHubFlow.scala |  49 ++++
 .../registryv2/flows/gcr/GcrAbstractFlow.scala     |  35 +++
 .../docker/registryv2/flows/gcr/GcrEuFlow.scala    |   8 +
 .../docker/registryv2/flows/gcr/GcrFlow.scala      |   8 +
 .../docker/registryv2/flows/gcr/GcrUsFlow.scala    |   8 +
 .../docker/registryv2/flows/gcr/GoogleFlow.scala   |  54 ++++
 .../callcaching/docker/DockerHashActorSpec.scala   | 117 +++++++++
 .../core/callcaching/docker/DockerHashMocks.scala  |  59 +++++
 .../docker/DockerImageIdentifierSpec.scala         |  61 +++++
 .../registryv2/flows/HttpFlowWithRetrySpec.scala   | 200 +++++++++++++++
 .../cromwell/engine/EngineWorkflowDescriptor.scala |   2 +-
 .../cromwell/engine/workflow/WorkflowActor.scala   |   5 +-
 .../engine/workflow/WorkflowManagerActor.scala     |   7 +-
 .../execution/EngineJobExecutionActor.scala        |  60 +++--
 .../lifecycle/execution/JobPreparationActor.scala  | 138 ----------
 .../execution/SubWorkflowExecutionActor.scala      |   8 +-
 .../execution/WorkflowExecutionActor.scala         |   8 +-
 .../execution/preparation/CallPreparation.scala    |  45 ++++
 .../preparation/JobPreparationActor.scala          | 215 ++++++++++++++++
 .../preparation/SubWorkflowPreparationActor.scala  |  60 +++++
 .../scala/cromwell/server/CromwellRootActor.scala  |  35 ++-
 .../test/scala/cromwell/CromwellTestKitSpec.scala  |  12 +
 .../scala/cromwell/SimpleWorkflowActorSpec.scala   |   1 +
 .../workflow/SingleWorkflowRunnerActorSpec.scala   |   5 +-
 .../engine/workflow/WorkflowActorSpec.scala        |   4 +-
 .../execution/SubWorkflowExecutionActorSpec.scala  |  12 +-
 .../execution/WorkflowExecutionActorSpec.scala     |   6 +-
 .../callcaching/EngineJobHashingActorSpec.scala    |   3 +-
 .../execution/ejea/EjeaCheckingJobStoreSpec.scala  |   4 +-
 .../execution/ejea/EjeaPreparingJobSpec.scala      |   2 +-
 .../ejea/EjeaRequestingExecutionTokenSpec.scala    |   4 +-
 .../lifecycle/execution/ejea/PerTestHelper.scala   |   9 +-
 .../preparation/JobPreparationActorSpec.scala      | 164 ++++++++++++
 .../preparation/JobPreparationTestHelper.scala     |  55 ++++
 .../filesystems/gcs/auth/GoogleAuthMode.scala      |   4 +
 .../gcs/auth/GoogleCredentialBundle.scala          |   6 +
 project/Dependencies.scala                         |   6 +-
 .../impl/jes/JesBackendInitializationData.scala    |   2 +
 .../impl/jes/JesBackendLifecycleActorFactory.scala |  12 +
 .../backend/impl/jes/JesInitializationActor.scala  |   7 +-
 .../jes/JesAsyncBackendJobExecutionActorSpec.scala |  25 +-
 .../SharedFileSystemJobExecutionActorSpec.scala    |   3 +-
 58 files changed, 2180 insertions(+), 202 deletions(-)
 create mode 100644 core/src/main/scala/cromwell/core/callcaching/CallCachingEligibility.scala
 create mode 100644 core/src/main/scala/cromwell/core/callcaching/docker/DockerFlow.scala
 create mode 100644 core/src/main/scala/cromwell/core/callcaching/docker/DockerHashActor.scala
 create mode 100644 core/src/main/scala/cromwell/core/callcaching/docker/DockerHashRequest.scala
 create mode 100644 core/src/main/scala/cromwell/core/callcaching/docker/DockerHashResult.scala
 create mode 100644 core/src/main/scala/cromwell/core/callcaching/docker/DockerImageIdentifier.scala
 create mode 100644 core/src/main/scala/cromwell/core/callcaching/docker/registryv2/DockerRegistryV2AbstractFlow.scala
 create mode 100644 core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/FlowUtils.scala
 create mode 100644 core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/HttpFlowWithRetry.scala
 create mode 100644 core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/dockerhub/DockerHubFlow.scala
 create mode 100644 core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/gcr/GcrAbstractFlow.scala
 create mode 100644 core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/gcr/GcrEuFlow.scala
 create mode 100644 core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/gcr/GcrFlow.scala
 create mode 100644 core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/gcr/GcrUsFlow.scala
 create mode 100644 core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/gcr/GoogleFlow.scala
 create mode 100644 core/src/test/scala/cromwell/core/callcaching/docker/DockerHashActorSpec.scala
 create mode 100644 core/src/test/scala/cromwell/core/callcaching/docker/DockerHashMocks.scala
 create mode 100644 core/src/test/scala/cromwell/core/callcaching/docker/DockerImageIdentifierSpec.scala
 create mode 100644 core/src/test/scala/cromwell/core/callcaching/docker/registryv2/flows/HttpFlowWithRetrySpec.scala
 delete mode 100644 engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/JobPreparationActor.scala
 create mode 100644 engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/preparation/CallPreparation.scala
 create mode 100644 engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/preparation/JobPreparationActor.scala
 create mode 100644 engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/preparation/SubWorkflowPreparationActor.scala
 create mode 100644 engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/preparation/JobPreparationActorSpec.scala
 create mode 100644 engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/preparation/JobPreparationTestHelper.scala
 create mode 100644 filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/auth/GoogleCredentialBundle.scala

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2285204fd..bb8f2b731 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -39,6 +39,15 @@
 * In some cases the SFS backend, used for Local, SGE, etc., coerced `WdlFile` to `WdlString` by using `.toUri`. This
 resulted in strings prepended with `file:///path/to/file`. Now absolute file paths will not contain the uri scheme.
 * Launch jobs on servers that support the GA4GH Task Execution Schema using the TES backend.
+* **Call caching: Cromwell will no longer try to use the cache for WDL tasks that contain a floating docker tag.** 
+  Call caching will still behave the same for tasks having a docker image with a specific hash.
+  See https://github.com/broadinstitute/cromwell#call-caching-docker-tags for more details. 
+* Added docker hash lookup. Cromwell will try to lookup the hash for a docker image with a floating tag, and use that hash when executing the job.
+  This will be reflected in the metadata where the docker runtime attribute will contains the hash that was used.
+  If Cromwell is unable to lookup the hash hash, the job will be run with the original user defined floating tag.
+  Cromwell is currently able to lookup public and private docker hashes for images on Docker Hub and Google Container Engine for job running on the JES backend.
+  For other backends, cromwell is able to lookup public docker hashes for Docker Hub and Google Container Engine.
+  See https://github.com/broadinstitute/cromwell#call-caching-docker-tags for more details. 
 
 ### Database schema changes
 * Added CUSTOM_LABELS as a field of WORKFLOW_STORE_ENTRY, to store workflow store entries.
diff --git a/README.md b/README.md
index 05be217ee..c4e7672bb 100644
--- a/README.md
+++ b/README.md
@@ -1844,6 +1844,42 @@ Cromwell also accepts two [workflow option](#workflow-options) related to call c
 
 > **Note:** If call caching is disabled, the to workflow options `read_from_cache` and `write_to_cache` will be ignored and the options will be treated as though they were 'false'.
 
+## Docker Tags
+
+Docker tags are a convenient way to point to a version of an image (ubuntu:14.04), or even the latest version (ubuntu:latest).
+For that purpose, tags are mutable, meaning that the image they point to can change, while the tag name stays the same.
+While this is very convenient in some cases, using mutable, or "floating" tags in WDL affects the reproducibility of the WDL file: the same WDL using "ubuntu:latest" run now, and a year, or even a month from now will actually run with different docker images.
+This has an even bigger impact when Call Caching is turned on in Cromwell, and could lead to unpredictable behaviors if a tag is updated in the middle of a workflow or even a scatter for example.
+Docker provides another way of identifying an image version, using the specific digest of the image. The digest is guaranteed to be different if 2 images have different byte content. For more information see https://docs.docker.com/registry/spec/api/#/content-digests
+A docker image with digest can be referenced as follows : **ubuntu@sha256:71cd81252a3563a03ad8daee81047b62ab5d892ebbfbf71cf53415f29c130950**
+The above image refers to a specific image of ubuntu, that does not depend on a floating tag.
+A WDL containing this Docker image run now and a year from now will run in the exact same container.
+
+In order to remove unpredictable behaviors, Cromwell takes the following approach regarding floating docker tags.
+
+When Cromwell finds a job ready to be run, it will first look at its docker runtime attribute, and apply the following logic:
+
+* The job doesn't specify a docker image: The job will be dispatched and all call caching settings (read/write) will apply normally.
+* The job does specify a docker runtime attribute:
+    * The docker image uses a hash: All call caching settings apply normally
+    * The docker image uses a floating tag:
+        Call caching `reading` will be disabled for this job. Specifically, Cromwell will *not* attempt to find an entry in the cache for this job.
+        Additionally, cromwell will attempt to look up the hash of the image. Upon success, it will replace the user's docker value with the hash.
+        This mechanism ensures that as long as Cromwell is able to lookup the hash, the job is guaranteed to have run on the container with that hash.
+        The docker value with the hash used for the job will be reported in the runtime attributes section of the metadata.
+        If Cromwell fails to lookup the hash (unsupported registry, wrong credentials, ...) it will run the job with the user provided floating tag.
+        If call caching writing is turned on, Cromwell will still write the job in the cache database, using:
+         * the hash if the lookup succeeded
+         * the floating tag otherwise.
+         
+Docker registry and access levels supported by Cromwell for docker hash lookup:
+
+|       |       DockerHub    ||       GCR       ||
+|:-----:|:---------:|:-------:|:------:|:-------:|
+|       |   Public  | Private | Public | Private |
+|  JES  |     X     |    X    |    X   |    X    |
+| Other |     X     |         |    X   |         |
+
 ## Local Filesystem Options
 When running a job on the Config (Shared Filesystem) backend, Cromwell provides some additional options in the backend's config section:
 
diff --git a/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala b/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala
index b6ae61917..e6aa055b1 100644
--- a/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala
+++ b/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala
@@ -59,6 +59,12 @@ trait BackendLifecycleActorFactory {
   lazy val fileHashingActorCount: Int = 50
 
   def fileHashingActorProps: Props = FileHashingActor.props(fileHashingFunction)
+  
+  /*
+   * Returns credentials that can be used to authenticate to a docker registry server
+   * in order to obtain a docker hash.
+   */
+  def dockerHashCredentials(initializationDataOption: Option[BackendInitializationData]): List[Any] = List.empty
 
   def jobExecutionTokenType: JobExecutionTokenType = JobExecutionTokenType("Default", None)
 }
diff --git a/backend/src/main/scala/cromwell/backend/backend.scala b/backend/src/main/scala/cromwell/backend/backend.scala
index b59b26b5d..7e5020d22 100644
--- a/backend/src/main/scala/cromwell/backend/backend.scala
+++ b/backend/src/main/scala/cromwell/backend/backend.scala
@@ -2,6 +2,7 @@ package cromwell.backend
 
 import com.typesafe.config.Config
 import cromwell.core.WorkflowOptions.WorkflowOption
+import cromwell.core.callcaching.CallCachingEligibility
 import cromwell.core.labels.Labels
 import cromwell.core.{CallKey, WorkflowId, WorkflowOptions}
 import wdl4s._
@@ -25,7 +26,8 @@ case class BackendJobDescriptorKey(call: TaskCall, index: Option[Int], attempt:
 case class BackendJobDescriptor(workflowDescriptor: BackendWorkflowDescriptor,
                                 key: BackendJobDescriptorKey,
                                 runtimeAttributes: Map[LocallyQualifiedName, WdlValue],
-                                inputDeclarations: EvaluatedTaskInputs) {
+                                inputDeclarations: EvaluatedTaskInputs,
+                                callCachingEligibility: CallCachingEligibility) {
   val fullyQualifiedInputs = inputDeclarations map { case (declaration, value) => declaration.fullyQualifiedName -> value }
   val call = key.call
   override val toString = s"${key.mkTag(workflowDescriptor.id)}"
diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardCachingActorHelper.scala b/backend/src/main/scala/cromwell/backend/standard/StandardCachingActorHelper.scala
index 7c72d6423..9dc801512 100644
--- a/backend/src/main/scala/cromwell/backend/standard/StandardCachingActorHelper.scala
+++ b/backend/src/main/scala/cromwell/backend/standard/StandardCachingActorHelper.scala
@@ -7,6 +7,7 @@ import cromwell.backend.io.{JobPaths, WorkflowPaths}
 import cromwell.backend.validation.{RuntimeAttributesValidation, ValidatedRuntimeAttributes}
 import cromwell.core.logging.JobLogging
 import cromwell.core.path.Path
+import cromwell.services.metadata.CallMetadataKeys
 import wdl4s.TaskCall
 
 import scala.util.Try
@@ -69,7 +70,7 @@ trait StandardCachingActorHelper extends JobCachingActorHelper {
     */
   def startMetadataKeyValues: Map[String, Any] = {
     val runtimeAttributesMetadata = RuntimeAttributesValidation.toMetadataStrings(validatedRuntimeAttributes) map {
-      case (key, value) => (s"runtimeAttributes:$key", value)
+      case (key, value) => (s"${CallMetadataKeys.RuntimeAttributes}:$key", value)
     }
 
     val fileMetadata = jobPaths.metadataPaths
diff --git a/backend/src/test/scala/cromwell/backend/BackendSpec.scala b/backend/src/test/scala/cromwell/backend/BackendSpec.scala
index a80a9496d..549f79f86 100644
--- a/backend/src/test/scala/cromwell/backend/BackendSpec.scala
+++ b/backend/src/test/scala/cromwell/backend/BackendSpec.scala
@@ -3,6 +3,7 @@ package cromwell.backend
 import com.typesafe.config.ConfigFactory
 import cromwell.backend.BackendJobExecutionActor.{BackendJobExecutionResponse, JobFailedNonRetryableResponse, JobFailedRetryableResponse, JobSucceededResponse}
 import cromwell.backend.io.TestWorkflows._
+import cromwell.core.callcaching.CallCachingEligible
 import cromwell.core.labels.Labels
 import cromwell.core.{WorkflowId, WorkflowOptions}
 import lenthall.exception.AggregatedException
@@ -55,7 +56,7 @@ trait BackendSpec extends ScalaFutures with Matchers with Mockito {
     val inputDeclarations = call.evaluateTaskInputs(inputs, NoFunctions).get // .get is ok because this is a test
     val evaluatedAttributes = RuntimeAttributeDefinition.evaluateRuntimeAttributes(call.task.runtimeAttributes, NoFunctions, inputDeclarations).get // .get is OK here because this is a test
     val runtimeAttributes = RuntimeAttributeDefinition.addDefaultsToAttributes(runtimeAttributeDefinitions, options)(evaluatedAttributes)
-    BackendJobDescriptor(workflowDescriptor, jobKey, runtimeAttributes, inputDeclarations)
+    BackendJobDescriptor(workflowDescriptor, jobKey, runtimeAttributes, inputDeclarations, CallCachingEligible)
   }
 
   def jobDescriptorFromSingleCallWorkflow(wdl: WdlSource,
@@ -67,7 +68,7 @@ trait BackendSpec extends ScalaFutures with Matchers with Mockito {
     val inputDeclarations = fqnMapToDeclarationMap(workflowDescriptor.knownValues)
     val evaluatedAttributes = RuntimeAttributeDefinition.evaluateRuntimeAttributes(call.task.runtimeAttributes, NoFunctions, inputDeclarations).get // .get is OK here because this is a test
     val runtimeAttributes = RuntimeAttributeDefinition.addDefaultsToAttributes(runtimeAttributeDefinitions, options)(evaluatedAttributes)
-    BackendJobDescriptor(workflowDescriptor, jobKey, runtimeAttributes, inputDeclarations)
+    BackendJobDescriptor(workflowDescriptor, jobKey, runtimeAttributes, inputDeclarations, CallCachingEligible)
   }
 
   def jobDescriptorFromSingleCallWorkflow(wdl: WdlSource,
@@ -81,7 +82,7 @@ trait BackendSpec extends ScalaFutures with Matchers with Mockito {
     val inputDeclarations = fqnMapToDeclarationMap(workflowDescriptor.knownValues)
     val evaluatedAttributes = RuntimeAttributeDefinition.evaluateRuntimeAttributes(call.task.runtimeAttributes, NoFunctions, inputDeclarations).get // .get is OK here because this is a test
     val runtimeAttributes = RuntimeAttributeDefinition.addDefaultsToAttributes(runtimeAttributeDefinitions, options)(evaluatedAttributes)
-    BackendJobDescriptor(workflowDescriptor, jobKey, runtimeAttributes, inputDeclarations)
+    BackendJobDescriptor(workflowDescriptor, jobKey, runtimeAttributes, inputDeclarations, CallCachingEligible)
   }
 
   def assertResponse(executionResponse: BackendJobExecutionResponse, expectedResponse: BackendJobExecutionResponse) = {
@@ -127,7 +128,7 @@ trait BackendSpec extends ScalaFutures with Matchers with Mockito {
 
   def firstJobDescriptor(workflowDescriptor: BackendWorkflowDescriptor,
                          inputs: Map[String, WdlValue] = Map.empty) = {
-    BackendJobDescriptor(workflowDescriptor, firstJobDescriptorKey(workflowDescriptor), Map.empty, fqnMapToDeclarationMap(inputs))
+    BackendJobDescriptor(workflowDescriptor, firstJobDescriptorKey(workflowDescriptor), Map.empty, fqnMapToDeclarationMap(inputs), CallCachingEligible)
   }
 }
 
diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf
index 33f995266..3aab8d3a4 100644
--- a/core/src/main/resources/reference.conf
+++ b/core/src/main/resources/reference.conf
@@ -138,6 +138,15 @@ google {
   ]
 }
 
+docker {
+  // Set this to match your available quota against the Google Container Engine API
+  gcr-api-queries-per-100-seconds = 1000
+  // Time in minutes before an entry expires from the docker hashes cache and needs to be fetched again
+  cache-entry-ttl = "20 minutes"
+  // Maximum number of elements to be kept in the cache. If the limit is reached, old elements will be removed from the cache
+  cache-size = 200
+}
+
 engine {
   # This instructs the engine which filesystems are at its disposal to perform any IO operation that it might need.
   # For instance, WDL variables declared at the Workflow level will be evaluated using the filesystems declared here.
diff --git a/core/src/main/scala/cromwell/core/callcaching/CallCachingEligibility.scala b/core/src/main/scala/cromwell/core/callcaching/CallCachingEligibility.scala
new file mode 100644
index 000000000..f39d82e19
--- /dev/null
+++ b/core/src/main/scala/cromwell/core/callcaching/CallCachingEligibility.scala
@@ -0,0 +1,22 @@
+package cromwell.core.callcaching
+
+sealed trait CallCachingEligibility
+case object CallCachingEligible extends CallCachingEligibility
+sealed trait CallCachingIneligible extends CallCachingEligibility {
+  def message: String
+}
+  
+case class FloatingDockerTagWithHash(hash: String) extends CallCachingIneligible {
+  override val message = s"""You are using a floating docker tag in this task. Cromwell does not consider tasks with floating tags to be eligible for call caching.
+        |If you want this task to be eligible for call caching in the future, use a docker runtime attribute with a digest instead.
+        |This is the exact docker image that was used for this job: $hash
+        |You can replace the docker runtime attribute in your task with the above value to make this task eligible for call caching.""".stripMargin
+}
+
+case object FloatingDockerTagWithoutHash extends CallCachingIneligible {
+  override val message = s"""You are using a floating docker tag in this task. Cromwell does not consider tasks with floating tags to be eligible for call caching.
+         |If you want this task to be eligible for call caching in the future, use a docker runtime attribute with a digest instead.
+         |Cromwell attempted to retrieve the current hash for this docker image but failed.
+         |This is not necessarily a cause for concern as Cromwell is currently only able to retrieve hashes for Dockerhub and GCR images.
+         |The job will be dispatched to the appropriate backend that will attempt to run it.""".stripMargin
+}
diff --git a/core/src/main/scala/cromwell/core/callcaching/docker/DockerFlow.scala b/core/src/main/scala/cromwell/core/callcaching/docker/DockerFlow.scala
new file mode 100644
index 000000000..cc5dc8bf0
--- /dev/null
+++ b/core/src/main/scala/cromwell/core/callcaching/docker/DockerFlow.scala
@@ -0,0 +1,12 @@
+package cromwell.core.callcaching.docker
+
+import akka.NotUsed
+import akka.stream.{FlowShape, Graph}
+
+/**
+  * Interface used by the docker hash actor to build a flow and validate whether or not it can accept an image.
+  */
+trait DockerFlow {
+  def buildFlow(): Graph[FlowShape[DockerHashActor.DockerHashContext, (DockerHashActor.DockerHashResponse, DockerHashActor.DockerHashContext)], NotUsed]
+  def accepts(dockerImageIdentifierWithoutHash: DockerImageIdentifierWithoutHash): Boolean
+}
diff --git a/core/src/main/scala/cromwell/core/callcaching/docker/DockerHashActor.scala b/core/src/main/scala/cromwell/core/callcaching/docker/DockerHashActor.scala
new file mode 100644
index 000000000..5e816de15
--- /dev/null
+++ b/core/src/main/scala/cromwell/core/callcaching/docker/DockerHashActor.scala
@@ -0,0 +1,209 @@
+package cromwell.core.callcaching.docker
+
+import akka.actor.{Actor, ActorLogging, ActorRef, Props}
+import akka.pattern.pipe
+import akka.stream.QueueOfferResult.{Dropped, Enqueued, QueueClosed}
+import akka.stream._
+import akka.stream.scaladsl.{GraphDSL, Merge, Partition, Sink, Source}
+import com.google.common.cache.CacheBuilder
+import cromwell.core.callcaching.docker.DockerHashActor._
+import org.slf4j.LoggerFactory
+
+import scala.concurrent.Future
+import scala.concurrent.duration.FiniteDuration
+
+final class DockerHashActor(
+                           dockerRegistryFlows: Seq[DockerFlow],
+                           queueBufferSize: Int,
+                           cacheEntryTTL: FiniteDuration,
+                           cacheSize: Long
+                          )(implicit val materializer: ActorMaterializer) extends Actor with ActorLogging {
+
+  implicit val system = context.system
+  implicit val ec = context.dispatcher
+  
+  /* Use the guava CacheBuilder class that implements a thread safe map with built in cache features.
+   * https://google.github.io/guava/releases/20.0/api/docs/com/google/common/cache/CacheBuilder.html
+   * Elements will be:
+   *  - Added to the cache by the updateCacheSink, running on a thread from the stream thread pool
+   *  - Accessed by this actor on its receive method thread, to check if an element is in the cache and use it
+   *  - Automatically removed from the cache by the cache itself
+   *  
+   *  + Concurrency level is the number of expected threads to modify the cache.
+   *      Set to "2" because the stream will add elements, and the cache itself remove them.
+   *      This value has not a critical impact: 
+   *        https://google.github.io/guava/releases/20.0/api/docs/com/google/common/cache/CacheBuilder.html#concurrencyLevel-int-
+   *        
+   *  + expireAfterWrite sets the time after which cache entries must expire. 
+   *      We use expireAfterWrite (as opposed to expireAfterAccess because we want to the entry to expire
+   *      even if it's accessed. The goal here is to force the actor to ask again for the hash after a certain 
+   *      amount of time to guarantee its relative accuracy.
+   *      
+   *  + maximumSize sets the maximum amount of entries the cache can contain. 
+   *    If/when this size is reached, least used entries will be expired
+   */  
+  private val cache = CacheBuilder.newBuilder()
+    .concurrencyLevel(2)
+    .expireAfterWrite(cacheEntryTTL._1, cacheEntryTTL._2)
+    .maximumSize(cacheSize)
+    .build[DockerImageIdentifierWithoutHash, DockerHashResult]()
+  
+  /*
+   * Never fail the stream, if an exception occurs just drop the element
+   * The sender is responsible for sending a new message if it didn't get a response
+   * within a reasonable amount of time
+   */
+  private val decider: Supervision.Decider = _ => Supervision.Resume
+
+  /*
+   * Intermediate sink responsible for updating the cache as soon as a successful hash is retrieved
+   */
+  private val updateCacheSink = Sink.foreach[(DockerHashResponse, DockerHashContext)] {
+    case (response: DockerHashResponseSuccess, dockerHashContext) => 
+      cache.put(dockerHashContext.dockerImageID, response.dockerHash)
+    case _ => // Only put successful hashes in the cache
+  }
+  
+  /*
+   * Final sink of the stream. Sends the response to the caller.
+   */
+  private val replySink = Sink.foreach[(DockerHashResponse, DockerHashContext)] {
+    case (response, dockerHashContext) => dockerHashContext.replyTo ! response
+  }
+
+  /*
+    Send each message to the corresponding docker registry flow if it exists,
+    and then merge the results back together.
+    If no flow is able to handle the request, create a DockerHashUnknownRegistry response
+                  _______                       ______
+                  |      |                      |     |
+                  |      | -> dockerhub flow -> |     | 
+      hash        |      | -> gcr flow  ->      |     | -> Response
+      context ->  |      | -> ...       ->      |     |
+                  |______|                      |_____|
+   */
+  private val dockerFlow = GraphDSL.create() { implicit builder =>
+    import GraphDSL.Implicits._
+    
+    def mapContextToFlow(dockerHashContext: DockerHashContext) = {
+      dockerRegistryFlows.indexWhere(_.accepts(dockerHashContext.dockerImageID)) match {
+          // requests with no matching host are sent to the last port
+        case -1 => dockerRegistryFlows.length
+        case found => found
+      }
+    }
+
+    val partitionFlows = builder.add(Partition[DockerHashContext](dockerRegistryFlows.size + 1, mapContextToFlow))
+    val mergeFlows = builder.add(Merge[(DockerHashResponse, DockerHashContext)](dockerRegistryFlows.size + 1))
+    
+    // Build a partial flow for each dockerRegistryFlow available
+    val flows = dockerRegistryFlows map { _.buildFlow() }
+
+    // Connect each flow to one output port of the broadcast, and merge them all back together
+    dockerRegistryFlows.indices foreach { i =>
+      partitionFlows.out(i) ~> flows(i) ~> mergeFlows.in(i)
+    }
+
+    val noMatch = partitionFlows.out(dockerRegistryFlows.size).map(dockerContext => (DockerHashUnknownRegistry(dockerContext.dockerImageID), dockerContext)).outlet
+    
+    noMatch ~> mergeFlows.in(dockerRegistryFlows.size)
+    
+    FlowShape(partitionFlows.in, mergeFlows.out)
+  }
+
+  private val queue = Source.queue[DockerHashContext](queueBufferSize, OverflowStrategy.dropNew)
+    .via(dockerFlow)
+    .alsoTo(updateCacheSink)
+    .to(replySink)
+    .withAttributes(ActorAttributes.supervisionStrategy(decider))
+    .run()
+  
+  override def receive: Receive = {
+    // When receiving a request, just enqueue it and pipe the result back to self
+    // Note: could be worth having another actor to send it to, to prevent overflowing this mailbox
+    case request: DockerHashRequest =>
+      val replyTo = sender()
+      
+      checkCache(request.dockerImageID) match {
+        case Some(cacheHit) => replyTo ! cacheHit
+        case None => sendToStream(DockerHashContext(request, replyTo))
+      }
+    case EnqueueResponse(Enqueued, dockerHashContext) => // All good !
+    case EnqueueResponse(Dropped, dockerHashContext) => dockerHashContext.replyTo ! DockerHashBackPressure(dockerHashContext.request)
+      // In any of the below case, the stream is in a state where it will not be able to receive new elements.
+      // This means something blew off so we can just restart the actor to re-instantiate a new stream
+    case EnqueueResponse(QueueClosed, dockerHashContext) =>
+      val exception = new RuntimeException(s"Failed to enqueue docker hash request ${dockerHashContext.request}. Queue was closed")
+      logAndRestart(exception)
+    case EnqueueResponse(QueueOfferResult.Failure(failure), dockerHashContext) => 
+      logAndRestart(failure)
+    case FailedToEnqueue(throwable, dockerHashContext) =>
+      logAndRestart(throwable)
+  }
+  
+  private def logAndRestart(throwable: Throwable) = {
+    log.warning("Failed to process docker hash request", throwable)
+    // Throw the exception that will be caught by supervisor and restart the actor
+    throw DockerHashActorException(throwable)
+  }
+  
+  private def checkCache(dockerImageIdentifierWithoutHash: DockerImageIdentifierWithoutHash) = {
+    Option(cache.getIfPresent(dockerImageIdentifierWithoutHash)) map { hashResult => 
+      DockerHashResponseSuccess(hashResult) 
+    }
+  }
+  
+  private def sendToStream(dockerHashContext: DockerHashContext) = {
+    val enqueue = queue offer dockerHashContext map { result =>
+      EnqueueResponse(result, dockerHashContext)
+    } recoverWith {
+      case t => Future.successful(FailedToEnqueue(t, dockerHashContext))
+    }
+
+    pipe(enqueue) to self
+    ()
+  }
+}
+
+object DockerHashActor {
+
+  val logger = LoggerFactory.getLogger("DockerRegistry")
+
+  /* Response Messages */
+  sealed trait DockerHashResponse
+  
+  case class DockerHashResponseSuccess(dockerHash: DockerHashResult) extends DockerHashResponse
+  
+  sealed trait DockerHashFailureResponse extends DockerHashResponse {
+    def reason: String
+  }
+  case class DockerHashFailedResponse(failure: Throwable, dockerIdentifier: DockerImageIdentifierWithoutHash) extends DockerHashFailureResponse {
+    override val reason = s"Failed to get docker hash for ${dockerIdentifier.fullName} ${failure.getMessage}"
+  }
+  case class DockerHashUnknownRegistry(dockerImageId: DockerImageIdentifierWithoutHash) extends DockerHashFailureResponse {
+    override val reason = s"Registry ${dockerImageId.host} is not supported"
+  }
+  case class DockerHashNotFound(dockerImageId: DockerImageIdentifierWithoutHash) extends DockerHashFailureResponse {
+    override val reason = s"Docker image ${dockerImageId.fullName} not found"
+  }
+  case class DockerHashUnauthorized(dockerImageId: DockerImageIdentifierWithoutHash) extends DockerHashFailureResponse {
+    override val reason = s"Unauthorized to get docker hash ${dockerImageId.fullName}"
+  }
+
+  case class DockerHashBackPressure(originalRequest: DockerHashRequest) extends DockerHashResponse
+  case class DockerHashActorException(failure: Throwable) extends RuntimeException(failure)
+  
+  /* Internal ADTs */
+  case class DockerHashContext(request: DockerHashRequest, replyTo: ActorRef) {
+    val dockerImageID = request.dockerImageID
+    val credentials = request.credentials
+  }
+
+  private case class EnqueueResponse(result: QueueOfferResult, dockerHashContext: DockerHashContext)
+  private case class FailedToEnqueue(failure: Throwable, dockerHashContext: DockerHashContext)
+
+  def props(dockerRegistryFlows: Seq[DockerFlow],
+            queueBufferSize: Int = 100,
+            cacheEntryTTL: FiniteDuration,
+            cacheSize: Long)(materializer: ActorMaterializer) = Props(new DockerHashActor(dockerRegistryFlows, queueBufferSize, cacheEntryTTL, cacheSize)(materializer))
+}
diff --git a/core/src/main/scala/cromwell/core/callcaching/docker/DockerHashRequest.scala b/core/src/main/scala/cromwell/core/callcaching/docker/DockerHashRequest.scala
new file mode 100644
index 000000000..6c7206d25
--- /dev/null
+++ b/core/src/main/scala/cromwell/core/callcaching/docker/DockerHashRequest.scala
@@ -0,0 +1,3 @@
+package cromwell.core.callcaching.docker
+
+case class DockerHashRequest(dockerImageID: DockerImageIdentifierWithoutHash, credentials: List[Any] = List.empty)
diff --git a/core/src/main/scala/cromwell/core/callcaching/docker/DockerHashResult.scala b/core/src/main/scala/cromwell/core/callcaching/docker/DockerHashResult.scala
new file mode 100644
index 000000000..fbfd18de9
--- /dev/null
+++ b/core/src/main/scala/cromwell/core/callcaching/docker/DockerHashResult.scala
@@ -0,0 +1,17 @@
+package cromwell.core.callcaching.docker
+
+object DockerHashResult {
+  // See https://docs.docker.com/registry/spec/api/#/content-digests
+  val DigestRegex = """([a-zA-Z0-9_+.-]+):([a-zA-Z0-9]+)""".r
+  
+  def apply(str: String): DockerHashResult = {
+    str match {
+      case DigestRegex(alg, hash) => new DockerHashResult(alg, hash)
+      case _ => throw new IllegalArgumentException(s"Hash value $str does not have the expected 'algorithm:hash' syntax")
+    }
+  }
+}
+
+case class DockerHashResult(hashAlgorithm: String, hashValue: String) {
+  val algorithmAndHash = s"$hashAlgorithm:$hashValue"
+}
diff --git a/core/src/main/scala/cromwell/core/callcaching/docker/DockerImageIdentifier.scala b/core/src/main/scala/cromwell/core/callcaching/docker/DockerImageIdentifier.scala
new file mode 100644
index 000000000..9e40ab15d
--- /dev/null
+++ b/core/src/main/scala/cromwell/core/callcaching/docker/DockerImageIdentifier.scala
@@ -0,0 +1,83 @@
+package cromwell.core.callcaching.docker
+
+import scala.util.{Failure, Success, Try}
+
+sealed trait DockerImageIdentifier {
+  def host: Option[String]
+  def repository: String
+  def image: String
+  def reference: String
+
+  lazy val name = s"$repository/$image"
+  lazy val hostAsString = host map { h => s"$h/" } getOrElse ""
+  lazy val fullName = s"$hostAsString$repository/$image:$reference"
+}
+
+case class DockerImageIdentifierWithoutHash(host: Option[String], repository: String, image: String, reference: String) extends DockerImageIdentifier {
+  def withHash(hash: DockerHashResult) =  DockerImageIdentifierWithHash(host, repository, image, reference, hash)
+}
+
+case class DockerImageIdentifierWithHash(host: Option[String], repository: String, image: String, reference: String, hash: DockerHashResult) extends DockerImageIdentifier {
+  override lazy val fullName: String = s"$hostAsString$repository/$image@${hash.algorithmAndHash}"
+}
+
+object DockerImageIdentifier {
+  // See https://github.com/docker-library/official-images/tree/master/library
+  private val DefaultDockerRepo = "library"
+  private val DefaultDockerTag = "latest"
+  private val TagSeparator = ":"
+  private val DigestSeparator = "@"
+  
+  private val DockerStringRegex =
+    s"""
+       (?x)                                     # Turn on comments and whitespace insensitivity
+       
+       (                                        # Begin capturing group for name
+          [a-z0-9]+(?:[._-][a-z0-9]+)*          # API v2 name component regex - see https://docs.docker.com/registry/spec/api/#/overview
+          (?:/[a-z0-9]+(?:[._-][a-z0-9]+)*)*    # Optional additional name components separated by /
+       )                                        # End capturing group for name
+       
+       (?:   
+          (                                     # Begin capturing group for tag separator
+            :|@                                 # Tag separator. ':' is followed by a tag, '@' is followed by a digest
+          )                                     # End capturing group for tag separator
+       
+          (                                     # Begin capturing group for reference 
+            [A-Za-z0-9]+(?:[.:_-][A-Za-z0-9]+)* # Reference
+          )                                     # End capturing group for reference  
+       )?                                      
+       """.trim.r
+  
+  def fromString(dockerString: String): Try[DockerImageIdentifier] = {
+    dockerString match {
+        // Just a name means latest tag implicitly
+      case DockerStringRegex(name, null, null) => buildId(name, TagSeparator,  DefaultDockerTag)
+      case DockerStringRegex(name, tagSeparator, reference) => buildId(name, tagSeparator, reference)
+      case _ => Failure(new IllegalArgumentException(s"Docker image $dockerString has an invalid syntax."))
+    }
+  }
+  
+  private def isRegistryHostName(str: String) = str.contains('.')
+  
+  private def buildId(name: String, tagSeparator: String, reference: String) = {
+    val (dockerHost, dockerRepo, dockerImage) = name.split('/').toList match {
+      // If just one component (e.g ubuntu), assume default repo
+      case image :: Nil => (None, DefaultDockerRepo, image)
+      // If repo/image (e.g broadinstitute/cromwell) without host
+      case repo :: image :: Nil if !isRegistryHostName(repo) => (None, repo, image)
+      // If host/image (e.g index.docker.io/ubuntu), assume default repo
+      case host :: image :: Nil if isRegistryHostName(host) => (Option(host), DefaultDockerRepo, image)
+      // Not a host followed more than one components
+      case nothost :: rest if !isRegistryHostName(nothost) => (None, s"$nothost/${rest.init.mkString("/")}", rest.last)
+      // A host followed more than one components (e.g gcr.io/google-containers/alpine-with-bash)
+      case host :: rest if isRegistryHostName(host) => (Option(host), rest.init.mkString("/"), rest.last)
+    }
+    
+    tagSeparator match {
+      case DigestSeparator => Try(DockerHashResult(reference)) map { hash => DockerImageIdentifierWithHash(dockerHost, dockerRepo, dockerImage, reference, hash) }
+      case TagSeparator => Success(DockerImageIdentifierWithoutHash(dockerHost, dockerRepo, dockerImage, reference))
+        // Should have been caught by the regex, but in case..
+      case other => Failure(new IllegalArgumentException(s"Invalid separator between image and tag in docker attribute: $other"))
+    }
+  }
+}
diff --git a/core/src/main/scala/cromwell/core/callcaching/docker/registryv2/DockerRegistryV2AbstractFlow.scala b/core/src/main/scala/cromwell/core/callcaching/docker/registryv2/DockerRegistryV2AbstractFlow.scala
new file mode 100644
index 000000000..0514e0c59
--- /dev/null
+++ b/core/src/main/scala/cromwell/core/callcaching/docker/registryv2/DockerRegistryV2AbstractFlow.scala
@@ -0,0 +1,278 @@
+package cromwell.core.callcaching.docker.registryv2
+
+import akka.NotUsed
+import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._
+import akka.http.scaladsl.model.HttpHeader.ParsingResult
+import akka.http.scaladsl.model._
+import akka.http.scaladsl.model.headers.{Authorization, OAuth2BearerToken}
+import akka.http.scaladsl.unmarshalling.Unmarshal
+import akka.stream._
+import akka.stream.scaladsl.{Flow, GraphDSL, Merge}
+import cromwell.core.callcaching.docker.DockerHashActor._
+import cromwell.core.callcaching.docker.registryv2.DockerRegistryV2AbstractFlow._
+import cromwell.core.callcaching.docker.registryv2.flows.{FlowUtils, HttpFlowWithRetry}
+import cromwell.core.callcaching.docker.{DockerFlow, DockerHashResult, DockerImageIdentifierWithoutHash}
+import spray.json._
+
+import scala.concurrent.{ExecutionContext, Future}
+import scala.util.{Failure, Success, Try}
+
+object DockerRegistryV2AbstractFlow {
+  type HttpDockerFlow = Flow[(HttpRequest, (DockerHashContext, HttpRequest)), (Try[HttpResponse], (DockerHashContext, HttpRequest)), NotUsed]
+  
+  val DigestHeaderName = "Docker-Content-Digest".toLowerCase
+  val AcceptHeader = HttpHeader.parse("Accept", "application/vnd.docker.distribution.manifest.v2+json") match {
+    case ParsingResult.Ok(header, errors) if errors.isEmpty => header
+    case ParsingResult.Ok(header, errors) =>
+      errors foreach { err => logger.warn(err.formatPretty) }
+      header
+    case ParsingResult.Error(error) => throw new RuntimeException(error.formatPretty)
+  }
+}
+
+/**
+  * Implements logic to build a flow that can retrieve a docker hash 
+  * from a registry server conforming to the docker registry API v2 Specification:
+  * https://docs.docker.com/registry/spec/api/
+  */
+abstract class DockerRegistryV2AbstractFlow(httpClientFlow: HttpDockerFlow)(implicit ec: ExecutionContext, materializer: ActorMaterializer) extends DockerFlow {
+  // Wraps the Http flow in a retryable flow to enable auto retries
+  final private val httpFlowWithRetry = new HttpFlowWithRetry[DockerHashContext](httpClientFlow, isRetryable = isRetryable).flow
+
+  final private val tokenFlow = {
+    val responseHandlerFlow = Flow[(HttpResponse, DockerHashContext)].mapAsync(1)(Function.tupled(tokenResponseHandler))
+    requestTransformFlow(buildTokenRequest, responseHandlerFlow)
+  }
+
+  final private val manifestFlow = {
+    val responseHandlerFlow = Flow[(HttpResponse, DockerHashContext)].map(Function.tupled(manifestResponseHandler))
+    requestTransformFlow(Function.tupled(buildManifestRequest _), responseHandlerFlow)
+  }
+  
+  /**
+    * Builds a flow for this docker registry
+    */
+  def buildFlow() = GraphDSL.create() { implicit builder =>
+    import GraphDSL.Implicits._
+    
+    // Decouple the token flow from the manifest flow with ".async" 
+    // this way they can run in parallel from each other 
+    // while still maintaining the final ordering
+    val token = builder.add(tokenFlow.async)
+    val manifest = builder.add(manifestFlow.async)
+    val mergeResponses = builder.add(Merge[(DockerHashResponse, DockerHashContext)](3))
+    
+    token.out0 ~> manifest.in
+                  manifest.out0 ~> mergeResponses.in(0)
+                  manifest.out1 ~> mergeResponses.in(1)
+    token.out1       ~>            mergeResponses.in(2)
+    
+    FlowShape(token.in, mergeResponses.out)
+  }
+
+  /**
+    * Returns true if this flow is able to process this docker image,
+    * false otherwise
+    */
+  def accepts(dockerImageIdentifierWithoutHash: DockerImageIdentifierWithoutHash) = dockerImageIdentifierWithoutHash.host.contains(registryHostName)
+  
+  /* Methods that must to be implemented by a subclass */
+
+  /**
+    * (e.g registry-1.docker.io)
+    */
+  protected def registryHostName: String
+
+  /**
+    * (e.g auth.docker.io)
+    */
+  protected def authorizationServerHostName: String
+
+  /**
+    * Builds the list of headers for the token request
+    */
+  protected def buildTokenRequestHeaders(dockerHashContext: DockerHashContext): List[HttpHeader]
+  
+  /* Methods that may be overridden by a subclass */
+
+  /**
+    * Returns true if an http response failed in such a way that it can be retried,
+    * false otherwise.
+    */
+  protected def isRetryable(httpResponse: HttpResponse) = {
+    httpResponse.status match {
+      case StatusCodes.TooManyRequests => true
+      case StatusCodes.InternalServerError => true
+      case _ => false
+    }
+  }
+
+  /**
+    * service parameter that can be used in the token request
+    * (e.g https://auth.docker.io/token?service=registry.docker.io&scope=repository:library/ubuntu:pull)
+    */
+  protected def serviceName: Option[String] = None
+
+  /**
+    * Builds the token URI to be queried based on a DockerImageID
+    */
+  protected def buildTokenRequestUri(dockerImageID: DockerImageIdentifierWithoutHash): String = {
+    val service = serviceName map { name => s"service=$name&" } getOrElse ""
+    s"https://$authorizationServerHostName/token?${service}scope=repository:${dockerImageID.name}:pull"
+  }
+
+  /**
+    * Http method used for the manifest request
+    */
+  protected def manifestRequestHttpMethod: HttpMethod = HttpMethods.GET
+
+  /**
+    * Maps a failed http response to a DockerHashFailureResponse
+    */
+  protected def httpFailureMapper(response: HttpResponse, dockerHashContext: DockerHashContext) = {
+    response.status match {
+      case StatusCodes.NotFound => (DockerHashNotFound(dockerHashContext.dockerImageID), dockerHashContext)
+      case StatusCodes.Unauthorized => (DockerHashUnauthorized(dockerHashContext.dockerImageID), dockerHashContext)
+      case other => (
+        DockerHashFailedResponse(
+          new Exception(s"Docker hash lookup failed with code ${other.intValue()}. ${other.defaultMessage()}"), dockerHashContext.dockerImageID
+        ),
+        dockerHashContext
+        )
+    }
+  }
+
+  /**
+    * Generic method to build a flow that creates a request, sends it,
+    * gets the result and performs some transformation on the result
+    * @param requestBuilderFunction Function that builds a request from an input object of type A
+    * @param responseHandlerFlow Function that transforms a response in an object of type Try[B]
+    * @tparam A type of the input value of the flow
+    * @tparam B type of the output value of the flow, if successful
+    * @return flow
+    */
+  private def requestTransformFlow[A, B](
+                                          requestBuilderFunction: A => (HttpRequest, DockerHashContext),
+                                          responseHandlerFlow: Flow[(HttpResponse, DockerHashContext), (Try[B], DockerHashContext), NotUsed]
+                                        ) = GraphDSL.create() { implicit builder =>
+    import GraphDSL.Implicits._
+
+    // Build a request from an A
+    val request = builder.add(Flow.fromFunction(requestBuilderFunction))
+
+    // Perform http requests
+    val http = builder.add(httpFlowWithRetry)
+    val successfulResponse = http.out0.via(responseHandlerFlow)
+    val failedResponse = http.out1.map(Function.tupled(httpFailureMapper))
+
+    // Partition parsing results
+    val parser = builder.add(FlowUtils.fanOutTry[B, DockerHashContext])
+    val successfulParsing: Outlet[(B, DockerHashContext)] = parser.out0
+    val failedParsing = parser.out1.map(Function.tupled(throwableToFailureMessage))
+
+    // Merge failures from failed response and failed parsing
+    val mergeFailures = builder.add(Merge[(DockerHashResponse, DockerHashContext)](2))
+
+    request ~> http.in
+    successfulResponse ~> parser.in
+
+    failedResponse ~> mergeFailures.in(0)
+    failedParsing ~> mergeFailures.in(1)
+
+    new FanOutShape2(request.in, successfulParsing, mergeFailures.out)
+  }
+
+  /**
+    * Builds the token request
+    */
+  private def buildTokenRequest(dockerHashContext: DockerHashContext) = {
+    val authorizationHeaders = buildTokenRequestHeaders(dockerHashContext)
+
+    val tokenHttpRequest = HttpRequest(
+      method = HttpMethods.GET,
+      uri = buildTokenRequestUri(dockerHashContext.dockerImageID),
+      headers = authorizationHeaders
+    )
+
+    (tokenHttpRequest, dockerHashContext)
+  }
+
+  /**
+    * Parse the http response coming back from the token request and returns the body as JsObject
+    */
+  private def tokenResponseHandler(response: HttpResponse, dockerHashContext: DockerHashContext) = response match {
+    case httpResponse if httpResponse.status.isSuccess() =>
+      Unmarshal(httpResponse.entity).to[String] map { body =>
+        (
+          Try(body.parseJson.asJsObject) flatMap extractToken,
+          dockerHashContext
+        )
+      } recoverWith {
+        case failure => Future.successful((Failure(failure), dockerHashContext))
+      }
+    case httpResponse =>
+      Future.successful(
+        (
+        Failure(new Exception(s"Request to obtain docker registry access token returned with a failure code: ${httpResponse.status.value}")),
+        dockerHashContext
+        )
+      )
+  }
+
+  /**
+    * Extract the access token from the json body of the http response
+    */
+  private def extractToken(jsObject: JsObject) = {
+    jsObject.fields.get("token") match {
+      case Some(token: JsString) => Success(token.value)
+      case Some(other) => Failure(new Exception("Token response contains a non-string token field"))
+      case None => Failure(new Exception("Token response did not contain a token field"))
+    }
+  }
+
+  /**
+    * Builds the manifest URI to be queried based on a DockerImageID
+    */
+  private def buildManifestUri(dockerImageID: DockerImageIdentifierWithoutHash): String = {
+    s"https://$registryHostName/v2/${dockerImageID.name}/manifests/${dockerImageID.reference}"
+  }
+
+  /**
+    * Builds the manifest http request
+    */
+  private def buildManifestRequest(token: String, dockerHashContext: DockerHashContext) = {
+    val authorizationHeader = Authorization(OAuth2BearerToken(token))
+
+    val manifestRequest = HttpRequest(
+      method = manifestRequestHttpMethod,
+      uri = buildManifestUri(dockerHashContext.dockerImageID),
+      headers = scala.collection.immutable.Seq(AcceptHeader, authorizationHeader)
+    )
+
+    (manifestRequest, dockerHashContext)
+  }
+
+  /**
+    * Handles the http response from the manifest request
+    */
+  private def manifestResponseHandler(response: HttpResponse, dockerHashContext: DockerHashContext) = {
+    (extractContentDigest(response.headers), dockerHashContext) 
+  }
+  
+  /**
+    * Extracts the digest from the response headers
+    */
+  private def extractContentDigest(headers: Seq[HttpHeader]) = {
+    headers find { _.is(DigestHeaderName) } match {
+      case Some(digestHeader) => Success(DockerHashResponseSuccess(DockerHashResult(digestHeader.value())))
+      case None => Failure(new Exception("Cannot find digest header"))
+    }
+  }
+
+  /**
+    * Wraps a throwable into a DockerHashFailedResponse
+    */
+  private def throwableToFailureMessage(throwable: Throwable, dockerHashContext: DockerHashContext): (DockerHashResponse, DockerHashContext) = {
+    (DockerHashFailedResponse(throwable, dockerHashContext.dockerImageID), dockerHashContext)
+  }
+}
diff --git a/core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/FlowUtils.scala b/core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/FlowUtils.scala
new file mode 100644
index 000000000..715d7a724
--- /dev/null
+++ b/core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/FlowUtils.scala
@@ -0,0 +1,34 @@
+package cromwell.core.callcaching.docker.registryv2.flows
+
+import akka.stream.FanOutShape2
+import akka.stream.scaladsl.{GraphDSL, Partition}
+
+import scala.util.{Failure, Success, Try}
+
+object FlowUtils {
+
+  /**
+    * Takes an input of the form (Try[T], U) and exposes 2 output ports.
+    * U is the type of the context value to be passed along.
+    * The first one will emit a pair of the form (value, context) if the try is a success.
+    * The second one will emit a pair of the form (throwable, context) if the try is a failure.
+    */
+  def fanOutTry[T, U] = GraphDSL.create() { implicit builder =>
+    import GraphDSL.Implicits._
+
+    val partition = builder.add(Partition[(Try[T], U)](2, {
+      case (Success(v), _) => 0
+      case (Failure(f), _) => 1
+    }))
+
+    val successOut: PortOps[(T, U)] = partition.out(0) collect {
+      case (Success(value), flowContext) => (value, flowContext)
+    }
+
+    val failureOut: PortOps[(Throwable, U)] = partition.out(1) collect {
+      case (Failure(failure), flowContext) => (failure, flowContext)
+    }
+
+    new FanOutShape2[(Try[T], U), (T, U), (Throwable, U)](partition.in, successOut.outlet, failureOut.outlet)
+  }
+}
diff --git a/core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/HttpFlowWithRetry.scala b/core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/HttpFlowWithRetry.scala
new file mode 100644
index 000000000..6509d1027
--- /dev/null
+++ b/core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/HttpFlowWithRetry.scala
@@ -0,0 +1,120 @@
+package cromwell.core.callcaching.docker.registryv2.flows
+
+import akka.NotUsed
+import akka.http.scaladsl.model.{HttpRequest, HttpResponse, StatusCodes}
+import akka.stream.scaladsl.{Flow, GraphDSL, Merge, Partition}
+import akka.stream.{FanOutShape2, OverflowStrategy}
+import cromwell.core.callcaching.docker.registryv2.flows.FlowUtils._
+import cromwell.core.callcaching.docker.registryv2.flows.HttpFlowWithRetry._
+
+import scala.util.Try
+
+object HttpFlowWithRetry {
+  def defaultIsRetryable(response: HttpResponse) = {
+    response.status match {
+      case StatusCodes.TooManyRequests => true
+      case StatusCodes.InternalServerError => true
+      case _ => false
+    }
+  }
+
+  /**
+    * In order to allow for retries, the http context object needs to encapsulate the original request,
+    * so that it can be re-submitted if necessary.
+    * This type provides this encapsulation by creating a pair of (T, HttpRequest) where T is the custom type that
+    * the user wishes to use as a context for their specific use case.
+    */
+  type ContextWithRequest[T] = (T, HttpRequest)
+
+  /**
+    * Type for a http flow using ContextWithRequest as context type.
+    */
+  type RetryableHttpFlow[T] = Flow[(HttpRequest, ContextWithRequest[T]), (Try[HttpResponse], ContextWithRequest[T]), NotUsed]
+}
+
+/**
+  * Adds additional retry logic around an existing akka http flow.
+  * @param httpClientFlow The akka http flow to use underneath
+  * @param retryBufferSize size of the buffer for requests to be retried. When the buffer is full, requests WILL BE DROPPED
+  *                        Act accordingly by resubmitting requests after a certain amount of time with no response for example.
+  * @param isRetryable function to determine weather or not an http request should be retried based on the http response                        
+  * @tparam T Type of the context data to be passed along with the request
+  * @return A http flow that will retry on request failures and on failed responses as seen fit by the isRetryable method
+  */
+case class HttpFlowWithRetry[T](
+                            httpClientFlow: RetryableHttpFlow[T],
+                            retryBufferSize: Int = 1000,
+                            isRetryable: HttpResponse => Boolean = defaultIsRetryable
+                          ) {
+
+  lazy val flow = GraphDSL.create() { implicit builder =>
+    import GraphDSL.Implicits._
+
+    // Http shape
+    val http = builder.add(httpClientFlow)
+    
+    // Wrap the user context into a pair of (flowContext, httpRequest) to allow for retries
+    val source = builder.add(Flow[(HttpRequest, T)] map {
+      case (httpRequest, flowContext) => (httpRequest, (flowContext, httpRequest))
+    })
+    
+    // Partition Try[HttpResponse] into Success and Failure
+    val partitionResponseTry = builder.add(fanOutTry[HttpResponse, (T, HttpRequest)])
+    
+    val requestSuccessful = partitionResponseTry.out0
+    // Extract the request from the failed try so it can be resubmitted
+    val requestFailed = partitionResponseTry.out1.log("request failed") map toRetryableRequest
+    
+    // Merges requests coming from 3 input ports:
+    // - in(0) takes requests from the outside
+    // - in(1) takes retryable failed requests
+    // - in(2) takes retryable failed responses (with non successful and retryable status code)
+    val mergeRequests = builder.add(Merge[(HttpRequest, (T, HttpRequest))](3))
+    
+    // Splits http response into 3 categories:
+    // - out(0) emits successful responses
+    // - out(1) emits failed but retryable responses
+    // - out(2) emits failed, non-retryable responses
+    val partitionHttpResponse = builder.add(Partition[(HttpResponse, (T, HttpRequest))](3, {
+      // Successful return code
+      case (httpResponse, flowContext) if httpResponse.status.isSuccess() => 0
+      // Failed return code but retryable
+      case (httpResponse, flowContext) if isRetryable(httpResponse) => 1
+      // Failed return code and non retryable
+      case (httpResponse, flowContext) => 2
+    }))
+
+    val responseSuccessful = partitionHttpResponse.out(0) map toTerminalResponse
+    val responseRetryable = partitionHttpResponse.out(1) map toRetryableRequest
+    val responseFailed = partitionHttpResponse.out(2) map toTerminalResponse
+    
+    // Buffer for retryable requests. Will DROP requests when full
+    // Will also delay re submission
+    val retryBuffer = Flow[(HttpRequest, (T, HttpRequest))]
+      .buffer(retryBufferSize, OverflowStrategy.dropHead)
+
+    // Submit outside requests to the first port of the mergeRequests merger 
+    source ~> mergeRequests.in(0)
+    
+    // Submit request to underlying http flow  -  Partition responses: Failure | Success
+    mergeRequests     ~>     http      ~>         partitionResponseTry.in
+                                                  // Success -> analyze response and partition into 3 (see above)
+                                                  requestSuccessful ~> partitionHttpResponse
+    // Retry failed requests (Try[HttpResponse] was a Failure)
+    mergeRequests.in(1)     <~ retryBuffer <~     requestFailed.outlet
+    // Retry retryable failed responses (the HttpResponse had a non successful return code and was deemed retryable)
+    mergeRequests.in(2)        <~        retryBuffer         <~        responseRetryable.outlet
+
+    new FanOutShape2(source.in, responseSuccessful.outlet, responseFailed.outlet)
+  }
+
+  // create a re-submittable request from a failed retryable
+  private def toRetryableRequest(value: (Any, (T, HttpRequest))) = value match {
+    case (_, (flowContext, request)) => (request, (flowContext, request))
+  }
+
+  // extract only the response and the context so it can be emitted through the output port
+  private def toTerminalResponse(value: (HttpResponse, (T, HttpRequest))) = value match {
+    case (response, (flowContext, _)) => (response, flowContext)
+  }
+}
diff --git a/core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/dockerhub/DockerHubFlow.scala b/core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/dockerhub/DockerHubFlow.scala
new file mode 100644
index 000000000..3ac567f64
--- /dev/null
+++ b/core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/dockerhub/DockerHubFlow.scala
@@ -0,0 +1,49 @@
+package cromwell.core.callcaching.docker.registryv2.flows.dockerhub
+
+import akka.http.scaladsl.model.headers.{Authorization, BasicHttpCredentials}
+import akka.http.scaladsl.model.{HttpMethod, HttpMethods}
+import akka.stream.ActorMaterializer
+import cromwell.core.DockerCredentials
+import cromwell.core.callcaching.docker.DockerHashActor.DockerHashContext
+import cromwell.core.callcaching.docker.DockerImageIdentifierWithoutHash
+import cromwell.core.callcaching.docker.registryv2.flows.dockerhub.DockerHubFlow._
+import cromwell.core.callcaching.docker.registryv2.DockerRegistryV2AbstractFlow
+import cromwell.core.callcaching.docker.registryv2.DockerRegistryV2AbstractFlow.HttpDockerFlow
+
+import scala.concurrent.ExecutionContext
+
+object DockerHubFlow {
+  private val DockerHubDefaultRegistry = "index.docker.io"
+  private val RegistryHostName = "registry-1.docker.io"
+  private val AuthorizationServerHostName = "auth.docker.io"
+  private val ServiceName = Option("registry.docker.io")
+}
+
+class DockerHubFlow(httpClientFlow: HttpDockerFlow)(implicit ec: ExecutionContext, materializer: ActorMaterializer) 
+  extends DockerRegistryV2AbstractFlow(httpClientFlow)(ec, materializer) {
+
+  override val registryHostName = RegistryHostName
+  override val authorizationServerHostName = AuthorizationServerHostName
+  override val serviceName = ServiceName
+
+  /**
+    * Builds the list of headers for the token request
+    */
+   def buildTokenRequestHeaders(dockerHashContext: DockerHashContext) = {
+    dockerHashContext.credentials collect {
+      case DockerCredentials(_, token) => 
+        Authorization(BasicHttpCredentials(token))
+    }
+  }
+  
+  override def accepts(dockerImageIdentifierWithoutHash: DockerImageIdentifierWithoutHash) = {
+    dockerImageIdentifierWithoutHash.host match {
+        // If no host is provided, assume dockerhub
+      case None => true
+      case Some(host) if host == registryHostName || host == DockerHubDefaultRegistry => true
+      case _ => false
+    }
+  }
+
+   override def manifestRequestHttpMethod: HttpMethod = HttpMethods.HEAD
+}
diff --git a/core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/gcr/GcrAbstractFlow.scala b/core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/gcr/GcrAbstractFlow.scala
new file mode 100644
index 000000000..09e946412
--- /dev/null
+++ b/core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/gcr/GcrAbstractFlow.scala
@@ -0,0 +1,35 @@
+package cromwell.core.callcaching.docker.registryv2.flows.gcr
+
+import akka.http.scaladsl.model.headers.{Authorization, OAuth2BearerToken}
+import akka.stream.ActorMaterializer
+import com.google.api.client.auth.oauth2.Credential
+import cromwell.core.callcaching.docker.DockerHashActor.DockerHashContext
+import cromwell.core.callcaching.docker.registryv2.DockerRegistryV2AbstractFlow
+import cromwell.core.callcaching.docker.registryv2.DockerRegistryV2AbstractFlow.HttpDockerFlow
+
+import scala.concurrent.ExecutionContext
+import scala.concurrent.duration._
+
+abstract class GcrAbstractFlow(httpClientFlow: HttpDockerFlow, host: String)(implicit ec: ExecutionContext, materializer: ActorMaterializer) extends DockerRegistryV2AbstractFlow(httpClientFlow)(ec, materializer) {
+  
+  private val AccessTokenAcceptableTTL = 1.minute.toSeconds
+  
+  override val registryHostName = host
+  override val authorizationServerHostName = s"$host/v2"
+  
+  /**
+    * Builds the list of headers for the token request
+    */
+   def buildTokenRequestHeaders(dockerHashContext: DockerHashContext) = {
+    dockerHashContext.credentials collect {
+      case credential: Credential => Authorization(OAuth2BearerToken(freshAccessToken(credential)))
+    }
+  }
+  
+  private def freshAccessToken(credential: Credential) = {
+    if (credential.getExpiresInSeconds < AccessTokenAcceptableTTL) {
+      credential.refreshToken()
+    }
+    credential.getAccessToken
+  }
+}
diff --git a/core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/gcr/GcrEuFlow.scala b/core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/gcr/GcrEuFlow.scala
new file mode 100644
index 000000000..de7856c60
--- /dev/null
+++ b/core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/gcr/GcrEuFlow.scala
@@ -0,0 +1,8 @@
+package cromwell.core.callcaching.docker.registryv2.flows.gcr
+
+import akka.stream.ActorMaterializer
+import cromwell.core.callcaching.docker.registryv2.DockerRegistryV2AbstractFlow.HttpDockerFlow
+
+import scala.concurrent.ExecutionContext
+
+private class GcrEuFlow(httpClientFlow: HttpDockerFlow)(implicit ec: ExecutionContext, materializer: ActorMaterializer) extends GcrAbstractFlow(httpClientFlow, "eu.gcr.io")
diff --git a/core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/gcr/GcrFlow.scala b/core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/gcr/GcrFlow.scala
new file mode 100644
index 000000000..4bc27a4d0
--- /dev/null
+++ b/core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/gcr/GcrFlow.scala
@@ -0,0 +1,8 @@
+package cromwell.core.callcaching.docker.registryv2.flows.gcr
+
+import akka.stream.ActorMaterializer
+import cromwell.core.callcaching.docker.registryv2.DockerRegistryV2AbstractFlow.HttpDockerFlow
+
+import scala.concurrent.ExecutionContext
+
+private class GcrFlow(httpClientFlow: HttpDockerFlow)(implicit ec: ExecutionContext, materializer: ActorMaterializer) extends GcrAbstractFlow(httpClientFlow, "gcr.io")
diff --git a/core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/gcr/GcrUsFlow.scala b/core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/gcr/GcrUsFlow.scala
new file mode 100644
index 000000000..57a93f753
--- /dev/null
+++ b/core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/gcr/GcrUsFlow.scala
@@ -0,0 +1,8 @@
+package cromwell.core.callcaching.docker.registryv2.flows.gcr
+
+import akka.stream.ActorMaterializer
+import cromwell.core.callcaching.docker.registryv2.DockerRegistryV2AbstractFlow.HttpDockerFlow
+
+import scala.concurrent.ExecutionContext
+
+private class GcrUsFlow(httpClientFlow: HttpDockerFlow)(implicit ec: ExecutionContext, materializer: ActorMaterializer) extends GcrAbstractFlow(httpClientFlow, "us.gcr.io")
diff --git a/core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/gcr/GoogleFlow.scala b/core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/gcr/GoogleFlow.scala
new file mode 100644
index 000000000..d66bf8eba
--- /dev/null
+++ b/core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/gcr/GoogleFlow.scala
@@ -0,0 +1,54 @@
+package cromwell.core.callcaching.docker.registryv2.flows.gcr
+
+import akka.stream.scaladsl.{Flow, GraphDSL, Merge, Partition}
+import akka.stream.{ActorMaterializer, FlowShape, ThrottleMode}
+import cromwell.core.callcaching.docker.DockerHashActor.{DockerHashContext, DockerHashResponse, DockerHashUnknownRegistry}
+import cromwell.core.callcaching.docker.registryv2.DockerRegistryV2AbstractFlow.HttpDockerFlow
+import cromwell.core.callcaching.docker.{DockerFlow, DockerImageIdentifierWithoutHash}
+
+import scala.concurrent.ExecutionContext
+import scala.concurrent.duration._
+import scala.language.postfixOps
+
+class GoogleFlow(httpClientFlow: HttpDockerFlow, queriesPer100Sec: Int)(implicit ec: ExecutionContext, materializer: ActorMaterializer) extends DockerFlow {
+  
+  private val gcrFlow = new GcrFlow(httpClientFlow)
+  private val usGcrFlow = new GcrUsFlow(httpClientFlow)
+  private val euGcrFlow = new GcrEuFlow(httpClientFlow)
+  
+  private val googleFlows = List(gcrFlow, usGcrFlow, euGcrFlow)
+  
+  // Throttle all requests to GCR
+  private val throttler = Flow[DockerHashContext].throttle(queriesPer100Sec, 100 seconds, queriesPer100Sec, ThrottleMode.Shaping)
+
+  override def buildFlow() = GraphDSL.create() { implicit builder =>
+    import GraphDSL.Implicits._
+
+    // This flow should only receive images that have been vetted by accepts.
+    def mapContextToFlow(dockerHashContext: DockerHashContext) = {
+      googleFlows.indexWhere(_.accepts(dockerHashContext.dockerImageID)) match {
+        case -1 => googleFlows.length
+        case found => found
+      }
+    }
+    
+    // If we ever get an image that none of the above flow can process,
+    // which shouldn't happen because the caller should have called "accepts" before
+    // to make sure this flow can handle the image
+    val failFlow = builder.add(Flow[DockerHashContext].map(dockerContext => (DockerHashUnknownRegistry(dockerContext.dockerImageID), dockerContext)))
+    
+    val partition = builder.add(Partition[DockerHashContext](googleFlows.length + 1, mapContextToFlow))
+    val merge = builder.add(Merge[(DockerHashResponse, DockerHashContext)](googleFlows.length + 1))
+    val throttle = builder.add(throttler)
+    
+    throttle ~> partition
+                googleFlows.indices foreach { i => partition.out(i) ~> googleFlows(i).buildFlow() ~> merge.in(i) }
+                partition.out(googleFlows.length) ~> failFlow ~> merge.in(googleFlows.length)
+    
+    FlowShape(throttle.in, merge.out)
+  }
+  
+  override def accepts(dockerImageIdentifierWithoutHash: DockerImageIdentifierWithoutHash) = {
+    googleFlows.exists(_.accepts(dockerImageIdentifierWithoutHash))
+  }
+}
diff --git a/core/src/test/scala/cromwell/core/callcaching/docker/DockerHashActorSpec.scala b/core/src/test/scala/cromwell/core/callcaching/docker/DockerHashActorSpec.scala
new file mode 100644
index 000000000..a7676e18d
--- /dev/null
+++ b/core/src/test/scala/cromwell/core/callcaching/docker/DockerHashActorSpec.scala
@@ -0,0 +1,117 @@
+package cromwell.core.callcaching.docker
+
+import akka.http.scaladsl.Http
+import akka.http.scaladsl.model.HttpRequest
+import akka.stream.ActorMaterializer
+import akka.testkit.ImplicitSender
+import cromwell.core.Tags.IntegrationTest
+import cromwell.core.TestKitSuite
+import cromwell.core.callcaching.docker.DockerHashActor._
+import cromwell.core.callcaching.docker.registryv2.flows.dockerhub.DockerHubFlow
+import cromwell.core.callcaching.docker.registryv2.flows.gcr.GoogleFlow
+import org.scalatest.{FlatSpecLike, Matchers}
+
+import scala.concurrent.duration._
+import scala.language.postfixOps
+
+class DockerHashActorSpec extends TestKitSuite with FlatSpecLike with Matchers with ImplicitSender {
+
+  behavior of "DockerRegistryActor"
+  
+  implicit val materializer = ActorMaterializer()
+  implicit val ex = system.dispatcher
+  
+  val httpPool = Http().superPool[(DockerHashContext, HttpRequest)]()
+  val registryFlows = Seq(new DockerHubFlow(httpPool), new GoogleFlow(httpPool, 50000))
+  // Disable cache by setting a cache size of 0 - A separate test tests the cache
+  val dockerActor = system.actorOf(DockerHashActor.props(registryFlows, 1000, 20 minutes, 0)(materializer))
+  
+  private def dockerImage(string: String) = DockerImageIdentifier.fromString(string).get.asInstanceOf[DockerImageIdentifierWithoutHash]
+  
+  private def makeRequest(string: String) = {
+    DockerHashRequest(dockerImage(string))
+  }
+
+  override protected def afterAll() = {
+    system.stop(dockerActor)
+    Http().shutdownAllConnectionPools()
+    materializer.shutdown()
+    super.afterAll()
+  }
+  
+  it should "retrieve a public docker hash" taggedAs IntegrationTest in {
+    dockerActor ! makeRequest("ubuntu:latest")
+    
+    expectMsgPF(5 second) {
+      case DockerHashResponseSuccess(DockerHashResult(alg, hash)) => 
+        alg shouldBe "sha256"
+        hash should not be empty
+    }
+  }
+
+  it should "retrieve a public docker hash on gcr" taggedAs IntegrationTest in {
+    dockerActor ! makeRequest("gcr.io/google-containers/alpine-with-bash:1.0")
+
+    expectMsgPF(5 second) {
+      case DockerHashResponseSuccess(DockerHashResult(alg, hash)) =>
+        alg shouldBe "sha256"
+        hash should not be empty
+    }
+  }
+  
+  it should "send image not found message back if the image does not exist" taggedAs IntegrationTest in {
+    val notFound = makeRequest("ubuntu:nonexistingtag")
+    dockerActor ! notFound
+
+    expectMsgClass(5 seconds, classOf[DockerHashNotFound])
+  }
+
+  it should "send unauthorized message back if the user doesn't have permission on this image" taggedAs IntegrationTest in {
+    val unauthorized = makeRequest("tjeandet/sinatra:v1")
+    dockerActor ! unauthorized
+
+    expectMsgClass(5 seconds, classOf[DockerHashUnauthorized])
+  }
+  
+  it should "send an unrecognized host message if no flow can process the docker string" taggedAs IntegrationTest in {
+    val unauthorized = makeRequest("unknown.io/image:v1")
+    dockerActor ! unauthorized
+
+    expectMsgClass(5 seconds, classOf[DockerHashUnknownRegistry])
+  }
+  
+  it should "cache results" in {
+    val hashSuccess = DockerHashResult("sha256", "hashvalue")
+    val responseSuccess = DockerHashResponseSuccess(hashSuccess)
+    val mockResponseSuccess = MockHashResponse(responseSuccess, 1)
+    
+    val image1 = dockerImage("ubuntu:latest")
+    val request = DockerHashRequest(image1)
+    
+    val responseFailure = DockerHashFailedResponse(new Exception("Docker hash failed - part of test flow"), image1)
+    val mockResponseFailure = MockHashResponse(responseFailure, 1)
+    
+    // Send back success, failure, success, failure, ...
+    val mockHttpFlow = new DockerFlowMock(mockResponseSuccess, mockResponseFailure)
+    val dockerActorWithCache = system.actorOf(DockerHashActor.props(Seq(mockHttpFlow), 1000, 3 seconds, 10)(materializer))
+    
+    dockerActorWithCache ! request
+    expectMsg(DockerHashResponseSuccess(hashSuccess))
+    // Necessary to give some time to the cache to be updated - as it's decoupled from sending back the response
+    Thread.sleep(1000)
+    
+    dockerActorWithCache ! request
+    // Without caching, the second request would have yielded a Failure since the mock flow alternates between a success and a failure
+    // Getting a success here means the request didn't make it to the stream
+    expectMsg(responseSuccess)
+    // Second check to make sure only one of the requests made it to the stream
+    mockHttpFlow.count() shouldBe 1
+    // Wait 3 seconds to give the cache time to expire the value
+    Thread.sleep(3000)
+    dockerActorWithCache ! request
+    // Expect a failure this time, as the cache should be empty
+    expectMsg(responseFailure)
+    mockHttpFlow.count() shouldBe 2
+  }
+  
+}
diff --git a/core/src/test/scala/cromwell/core/callcaching/docker/DockerHashMocks.scala b/core/src/test/scala/cromwell/core/callcaching/docker/DockerHashMocks.scala
new file mode 100644
index 000000000..ee86cc49e
--- /dev/null
+++ b/core/src/test/scala/cromwell/core/callcaching/docker/DockerHashMocks.scala
@@ -0,0 +1,59 @@
+package cromwell.core.callcaching.docker
+
+import akka.http.scaladsl.model._
+import akka.stream.scaladsl.Flow
+import cromwell.core.callcaching.docker.DockerHashActor.{DockerHashContext, DockerHashResponse}
+
+import scala.util.Try
+
+case class MockHttpResponse(httpResponse: Try[HttpResponse], nb: Int)
+
+class HttpMock[T](responses: MockHttpResponse*) {
+  private var responsesLeft = responses.toBuffer
+  
+  private def nextResponse(value: (HttpRequest, (T, HttpRequest))): (Try[HttpResponse], (T, HttpRequest)) = value match {
+    case (request, context) =>
+      responsesLeft.headOption match {
+        case Some(mockResponse) =>
+          if (mockResponse.nb > 1)
+            responsesLeft.update(0, mockResponse.copy(nb = mockResponse.nb - 1))
+          else
+            responsesLeft.remove(0)
+          (mockResponse.httpResponse, context)
+        // When we hit the end, loop
+        case None =>
+          responsesLeft = responses.toBuffer
+          nextResponse(value)
+      }
+  }
+  
+  def httpMock() = Flow[(HttpRequest, (T, HttpRequest))] map nextResponse
+}
+
+case class MockHashResponse(hashResponse: DockerHashResponse, nb: Int)
+
+class DockerFlowMock(responses: MockHashResponse*) extends DockerFlow {
+  private var responsesLeft = responses.toBuffer
+  // Counts the number of elements going through this "flow"
+  private var _count: Int = 0
+
+  private def nextResponse(context: DockerHashContext): (DockerHashResponse, DockerHashContext) = 
+      responsesLeft.headOption match {
+        case Some(mockResponse) =>
+          _count += 1
+          if (mockResponse.nb > 1)
+            responsesLeft.update(0, mockResponse.copy(nb = mockResponse.nb - 1))
+          else
+            responsesLeft.remove(0)
+          (mockResponse.hashResponse, context)
+        // When we hit the end, loop
+        case None =>
+          responsesLeft = responses.toBuffer
+          nextResponse(context)
+  }
+  
+  def count() = _count
+
+  override def buildFlow() = Flow[DockerHashContext] map nextResponse
+  override def accepts(dockerImageIdentifierWithoutHash: DockerImageIdentifierWithoutHash): Boolean = true
+}
diff --git a/core/src/test/scala/cromwell/core/callcaching/docker/DockerImageIdentifierSpec.scala b/core/src/test/scala/cromwell/core/callcaching/docker/DockerImageIdentifierSpec.scala
new file mode 100644
index 000000000..9d0fb6218
--- /dev/null
+++ b/core/src/test/scala/cromwell/core/callcaching/docker/DockerImageIdentifierSpec.scala
@@ -0,0 +1,61 @@
+package cromwell.core.callcaching.docker
+
+import org.scalatest.prop.TableDrivenPropertyChecks
+import org.scalatest.{FlatSpec, Matchers}
+
+class DockerImageIdentifierSpec extends FlatSpec with Matchers with TableDrivenPropertyChecks {
+  behavior of "DockerImageID"
+  
+  it should "parse valid docker images" in {
+    val valid = Table(
+      ("sourceString",                            "host",              "repo",          "image",     "reference"),
+      
+      // Without tags -> latest
+      ("ubuntu",                                  None,               "library",        "ubuntu",     "latest"),
+      ("broad/cromwell",                          None,               "broad",          "cromwell",   "latest"),
+      ("index.docker.io/ubuntu",         Option("index.docker.io"),   "library",        "ubuntu",     "latest"),
+      ("broad/cromwell/submarine",                None,               "broad/cromwell", "submarine",  "latest"),
+      ("gcr.io/google/alpine",              Option("gcr.io"),         "google",         "alpine",     "latest"),
+
+      // With tags
+      ("ubuntu:latest",                           None,               "library",        "ubuntu",     "latest"),
+      ("ubuntu:1235-SNAP",                        None,               "library",        "ubuntu",     "1235-SNAP"),
+      ("ubuntu:V3.8-5_1",                         None,               "library",        "ubuntu",     "V3.8-5_1")
+    )
+    
+    forAll(valid) { (dockerString, host, repo, image, reference) =>
+      val imageId = DockerImageIdentifier.fromString(dockerString)
+      imageId.isSuccess shouldBe true
+      val successfulId = imageId.get
+      successfulId.host shouldBe host
+      successfulId.repository shouldBe repo
+      successfulId.image shouldBe image
+      successfulId.reference shouldBe reference
+    }
+
+    // With digest
+    val withDigest = DockerImageIdentifier.fromString("ubuntu@sha256:45168651")
+    withDigest.isSuccess shouldBe true
+    val successfulDigest = withDigest.get
+    successfulDigest.host shouldBe None
+    successfulDigest.repository shouldBe "library"
+    successfulDigest.image shouldBe "ubuntu"
+    successfulDigest.reference shouldBe "sha256:45168651"
+    successfulDigest.isInstanceOf[DockerImageIdentifierWithHash] shouldBe true
+    successfulDigest.asInstanceOf[DockerImageIdentifierWithHash].hash shouldBe DockerHashResult("sha256", "45168651")
+  }
+
+  it should "not parse invalid docker images" in {
+    val invalid = List(
+      "_notvalid:latest",
+      "NotValid:latest",
+      "not:_valid",
+      "/not:valid",
+      "not%valid"
+    )
+    
+    invalid foreach { image =>
+      DockerImageIdentifier.fromString(image).isSuccess shouldBe false
+    }
+  }
+}
diff --git a/core/src/test/scala/cromwell/core/callcaching/docker/registryv2/flows/HttpFlowWithRetrySpec.scala b/core/src/test/scala/cromwell/core/callcaching/docker/registryv2/flows/HttpFlowWithRetrySpec.scala
new file mode 100644
index 000000000..67d69bce7
--- /dev/null
+++ b/core/src/test/scala/cromwell/core/callcaching/docker/registryv2/flows/HttpFlowWithRetrySpec.scala
@@ -0,0 +1,200 @@
+package cromwell.core.callcaching.docker.registryv2.flows
+
+import akka.NotUsed
+import akka.http.scaladsl.model.{HttpRequest, HttpResponse, StatusCodes}
+import akka.stream._
+import akka.stream.scaladsl.{GraphDSL, RunnableGraph, Sink, Source}
+import akka.testkit.{ImplicitSender, TestProbe}
+import cromwell.core.TestKitSuite
+import cromwell.core.callcaching.docker.{HttpMock, MockHttpResponse}
+import org.scalatest.{FlatSpecLike, Matchers}
+
+import scala.concurrent.duration._
+import scala.language.postfixOps
+import scala.util.{Failure, Success}
+
+class HttpFlowWithRetrySpec extends TestKitSuite with FlatSpecLike with Matchers with ImplicitSender {
+  behavior of "HttpFlowWithRetry"
+
+  implicit val materializer = ActorMaterializer()
+
+  override protected def afterAll() = {
+    materializer.shutdown()
+    super.afterAll()
+  }
+  
+  def stopProbe(probe: TestProbe) = system stop probe.ref
+
+  def makeRunnableGraph(httpFlowWithRetry: HttpFlowWithRetry[Any],
+                        source: Source[(HttpRequest, Unit), NotUsed] = Source.single((HttpRequest(), ()))
+                       ) = {
+    
+    val probeSuccess = TestProbe()
+    val probeFailure = TestProbe()
+    val sinkSuccess = Sink.actorRef(probeSuccess.ref, null)
+    val sinkFailure = Sink.actorRef(probeFailure.ref, null)
+    
+    val graph = RunnableGraph.fromGraph(GraphDSL.create() { implicit b =>
+      import akka.stream.scaladsl.GraphDSL.Implicits._
+      val httpRetryMockPorts = b.add(httpFlowWithRetry.flow)
+
+      source ~> httpRetryMockPorts.in
+      httpRetryMockPorts.out0 ~> sinkSuccess
+      httpRetryMockPorts.out1 ~> sinkFailure
+
+      ClosedShape
+    })
+    
+    (graph, probeSuccess, probeFailure)
+  }
+  
+  it should "emit an Http response on success port if request is successful" in {
+    val mockResponse = HttpResponse(status = StatusCodes.OK)
+    
+    val httpMock = new HttpMock[Any](MockHttpResponse(Success(mockResponse), 1)).httpMock()
+    val httpRetryMock = HttpFlowWithRetry[Any](httpMock)
+    
+    val (graph, probeSuccess, probeFailure) = makeRunnableGraph(httpRetryMock)
+
+    graph.run()
+
+    probeFailure.expectNoMsg(1 second)
+    probeSuccess.expectMsgPF(1 second) {
+      case (response: HttpResponse, _) => response.status shouldBe StatusCodes.OK
+    }
+
+    stopProbe(probeSuccess)
+    stopProbe(probeFailure)
+  }
+
+  it should "emit an Http response on failure port if request is not successful and not retryable" in {
+    val mockResponse = HttpResponse(status = StatusCodes.BadRequest)
+    
+    val httpMock = new HttpMock[Any](MockHttpResponse(Success(mockResponse), 1)).httpMock()
+    val httpRetryMock = HttpFlowWithRetry[Any](httpMock)
+
+    val (graph, probeSuccess, probeFailure) = makeRunnableGraph(httpRetryMock)
+
+    graph.run()
+
+    probeSuccess.expectNoMsg(1 second)
+    probeFailure.expectMsgPF(1 second) {
+      case (response: HttpResponse, _) => response.status shouldBe StatusCodes.BadRequest
+    }
+
+    stopProbe(probeSuccess)
+    stopProbe(probeFailure)
+  }
+
+  it should "retry a request if it fails" in {
+    val mockFailure = Failure[HttpResponse](new Exception("Request failed - part of test flow"))
+    val mockOK = HttpResponse(status = StatusCodes.OK)
+    
+    val httpMock = new HttpMock[Any](MockHttpResponse(mockFailure, 1), MockHttpResponse(Success(mockOK), 1)).httpMock()
+    val httpRetryMock = HttpFlowWithRetry[Any](httpMock)
+
+    val (graph, probeSuccess, probeFailure) = makeRunnableGraph(httpRetryMock)
+
+    graph.run()
+
+    probeFailure.expectNoMsg(1 second)
+    probeSuccess.expectMsgPF(1 second) {
+      case (response: HttpResponse, _) => response.status shouldBe StatusCodes.OK
+    }
+
+    stopProbe(probeSuccess)
+    stopProbe(probeFailure)
+  }
+  
+  it should "retry a request if the response is not a success and is retryable" in {
+    def isRetryable(httpResponse: HttpResponse) = httpResponse.status == StatusCodes.Unauthorized
+    
+    val mockFailure = HttpResponse(status = StatusCodes.Unauthorized)
+    val mockOK = HttpResponse(status = StatusCodes.OK)
+    val httpMock = new HttpMock[Any](MockHttpResponse(Success(mockFailure), 1), MockHttpResponse(Success(mockOK), 1)).httpMock()
+    val httpRetryMock = HttpFlowWithRetry[Any](httpMock, isRetryable = isRetryable)
+
+    val (graph, probeSuccess, probeFailure) = makeRunnableGraph(httpRetryMock)
+
+    graph.run()
+
+    probeFailure.expectNoMsg(1 second)
+    probeSuccess.expectMsgPF(1 second) {
+      case (response: HttpResponse, _) => response.status shouldBe StatusCodes.OK
+    }
+
+    stopProbe(probeSuccess)
+    stopProbe(probeFailure)
+  }
+
+  it should "handle flaky connections with a sufficiently large retry buffer" in {
+    val mock200 = HttpResponse(status = StatusCodes.OK)
+    val mock429 = HttpResponse(status = StatusCodes.TooManyRequests)
+    
+    // only 20% of requests come back successful, rest are out of quota errors
+    val mock200s = MockHttpResponse(Success(mock200), 1)
+    val mock429s = MockHttpResponse(Success(mock429), 5)
+
+    val httpMock = new HttpMock[Any](mock200s, mock429s).httpMock()
+    // Give a retry buffer of 1000 (= queues up to 1000 retry requests before starting dropping them) 
+    val httpRetryMock = new HttpFlowWithRetry[Any](httpMock, retryBufferSize = 1000)
+
+    // Send 1000 requests
+    val nbMessagesToSend = 1000
+    val mockRequest = HttpRequest()
+    val sourceIterator = 1 to nbMessagesToSend map { _ => (mockRequest, ()) } toIterator
+    val source = Source.fromIterator { () => sourceIterator }
+    val (graph, probeSuccess, probeFailure) = makeRunnableGraph(httpRetryMock, source)
+
+    graph.run()
+
+    probeFailure.expectNoMsg(5 seconds)
+    var successes: Int = 0
+    
+    probeSuccess.receiveWhile(5 seconds, 500 millis, nbMessagesToSend) {
+      case _ => successes += 1
+    }
+
+    // All responses should come back successfully
+    successes == 1000 shouldBe true
+
+    stopProbe(probeSuccess)
+    stopProbe(probeFailure)
+  }
+
+  it should "not deadlock even under high retry pressure" in {
+    val mock200 = HttpResponse(status = StatusCodes.OK)
+    val mock429 = HttpResponse(status = StatusCodes.TooManyRequests)
+
+    // only 20% of requests come back successful, rest are out of quota errors
+    val mock200s = MockHttpResponse(Success(mock200), 1)
+    val mock429s = MockHttpResponse(Success(mock429), 5)
+
+    val httpMock = new HttpMock[Any](mock200s, mock429s).httpMock()
+    // Give a retry buffer of 10 (= queues up to 10 retry requests before starting dropping them)
+    val httpRetryMock = new HttpFlowWithRetry[Any](httpMock, retryBufferSize = 10)
+
+    // Send 1000 requests, the retry buffer will overflow, and start dropping requests
+    // Note that this is an extreme case, 80% failure rate with 100 * retry buffer size elements and no throttling
+    val nbMessagesToSend = 1000
+    val mockRequest = HttpRequest()
+    val sourceIterator = 1 to nbMessagesToSend map { _ => (mockRequest, ()) } toIterator
+    val source = Source.fromIterator { () => sourceIterator }
+    val (graph, probeSuccess, probeFailure) = makeRunnableGraph(httpRetryMock, source)
+
+    graph.run()
+
+    probeFailure.expectNoMsg(5 seconds)
+    var successes: Int = 0
+    // The second parameter makes sure that the stream keeps emitting responses at least every 500 milliseconds
+    probeSuccess.receiveWhile(5 seconds, 500 millis, nbMessagesToSend) {
+      case _ => successes += 1
+    }
+
+    // A minimum amount of requests should come back
+    successes > 200 shouldBe true
+
+    stopProbe(probeSuccess)
+    stopProbe(probeFailure)
+  }
+}
diff --git a/engine/src/main/scala/cromwell/engine/EngineWorkflowDescriptor.scala b/engine/src/main/scala/cromwell/engine/EngineWorkflowDescriptor.scala
index 071b4d70d..c6fd821ae 100644
--- a/engine/src/main/scala/cromwell/engine/EngineWorkflowDescriptor.scala
+++ b/engine/src/main/scala/cromwell/engine/EngineWorkflowDescriptor.scala
@@ -6,7 +6,7 @@ import cromwell.core.callcaching.CallCachingMode
 import cromwell.core.path.PathBuilder
 import wdl4s._
 
-final case class EngineWorkflowDescriptor(namespace: WdlNamespaceWithWorkflow,
+case class EngineWorkflowDescriptor(namespace: WdlNamespaceWithWorkflow,
                                           backendDescriptor: BackendWorkflowDescriptor,
                                           backendAssignments: Map[TaskCall, String],
                                           failureMode: WorkflowFailureMode,
diff --git a/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala b/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala
index eefa9ceeb..ad97eb0f5 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala
@@ -141,11 +141,12 @@ object WorkflowActor {
             jobStoreActor: ActorRef,
             subWorkflowStoreActor: ActorRef,
             callCacheReadActor: ActorRef,
+            dockerHashActor: ActorRef,
             jobTokenDispenserActor: ActorRef,
             backendSingletonCollection: BackendSingletonCollection,
             serverMode: Boolean): Props = {
     Props(new WorkflowActor(workflowId, startMode, wdlSource, conf, serviceRegistryActor, workflowLogCopyRouter,
-      jobStoreActor, subWorkflowStoreActor, callCacheReadActor, jobTokenDispenserActor, backendSingletonCollection, serverMode)).withDispatcher(EngineDispatcher)
+      jobStoreActor, subWorkflowStoreActor, callCacheReadActor, dockerHashActor, jobTokenDispenserActor, backendSingletonCollection, serverMode)).withDispatcher(EngineDispatcher)
   }
 }
 
@@ -161,6 +162,7 @@ class WorkflowActor(val workflowId: WorkflowId,
                     jobStoreActor: ActorRef,
                     subWorkflowStoreActor: ActorRef,
                     callCacheReadActor: ActorRef,
+                    dockerHashActor: ActorRef,
                     jobTokenDispenserActor: ActorRef,
                     backendSingletonCollection: BackendSingletonCollection,
                     serverMode: Boolean)
@@ -211,6 +213,7 @@ class WorkflowActor(val workflowId: WorkflowId,
         jobStoreActor,
         subWorkflowStoreActor,
         callCacheReadActor,
+        dockerHashActor,
         jobTokenDispenserActor,
         backendSingletonCollection,
         initializationData,
diff --git a/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala b/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala
index e92b537d6..fb87406a1 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala
@@ -46,12 +46,13 @@ object WorkflowManagerActor {
             jobStoreActor: ActorRef,
             subWorkflowStoreActor: ActorRef,
             callCacheReadActor: ActorRef,
+            dockerHashActor: ActorRef,
             jobTokenDispenserActor: ActorRef,
             backendSingletonCollection: BackendSingletonCollection,
             abortJobsOnTerminate: Boolean,
             serverMode: Boolean): Props = {
     val params = WorkflowManagerActorParams(ConfigFactory.load, workflowStore, serviceRegistryActor,
-      workflowLogCopyRouter, jobStoreActor, subWorkflowStoreActor, callCacheReadActor, jobTokenDispenserActor, backendSingletonCollection,
+      workflowLogCopyRouter, jobStoreActor, subWorkflowStoreActor, callCacheReadActor, dockerHashActor, jobTokenDispenserActor, backendSingletonCollection,
       abortJobsOnTerminate, serverMode)
     Props(new WorkflowManagerActor(params)).withDispatcher(EngineDispatcher)
   }
@@ -91,6 +92,7 @@ case class WorkflowManagerActorParams(config: Config,
                                       jobStoreActor: ActorRef,
                                       subWorkflowStoreActor: ActorRef,
                                       callCacheReadActor: ActorRef,
+                                      dockerHashActor: ActorRef,
                                       jobTokenDispenserActor: ActorRef,
                                       backendSingletonCollection: BackendSingletonCollection,
                                       abortJobsOnTerminate: Boolean,
@@ -283,7 +285,8 @@ class WorkflowManagerActor(params: WorkflowManagerActorParams)
     }
 
     val wfProps = WorkflowActor.props(workflowId, startMode, workflow.sources, config, params.serviceRegistryActor,
-      params.workflowLogCopyRouter, params.jobStoreActor, params.subWorkflowStoreActor, params.callCacheReadActor, params.jobTokenDispenserActor,
+      params.workflowLogCopyRouter, params.jobStoreActor, params.subWorkflowStoreActor, params.callCacheReadActor,
+      params.dockerHashActor, params.jobTokenDispenserActor,
       params.backendSingletonCollection, params.serverMode)
     val wfActor = context.actorOf(wfProps, name = s"WorkflowActor-$workflowId")
 
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala
index 20a2ee40c..a9a111df8 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala
@@ -13,11 +13,12 @@ import cromwell.core.callcaching._
 import cromwell.core.logging.WorkflowLogging
 import cromwell.core.simpleton.WdlValueSimpleton
 import cromwell.database.sql.tables.CallCachingEntry
-import cromwell.engine.workflow.lifecycle.execution.CallPreparationActor.{BackendJobPreparationSucceeded, CallPreparationFailed}
 import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor._
 import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.{CacheHit, CacheMiss, CallCacheHashes, HashError}
 import cromwell.engine.workflow.lifecycle.execution.callcaching.FetchCachedResultsActor.{CachedOutputLookupFailed, CachedOutputLookupSucceeded}
 import cromwell.engine.workflow.lifecycle.execution.callcaching._
+import cromwell.engine.workflow.lifecycle.execution.preparation.CallPreparation.{BackendJobPreparationSucceeded, CallPreparationFailed}
+import cromwell.engine.workflow.lifecycle.execution.preparation.{CallPreparation, JobPreparationActor}
 import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor.{JobExecutionTokenDenied, JobExecutionTokenDispensed, JobExecutionTokenRequest, JobExecutionTokenReturn}
 import cromwell.jobstore.JobStoreActor._
 import cromwell.jobstore.{Pending => _, _}
@@ -37,6 +38,7 @@ class EngineJobExecutionActor(replyTo: ActorRef,
                               val serviceRegistryActor: ActorRef,
                               jobStoreActor: ActorRef,
                               callCacheReadActor: ActorRef,
+                              dockerHashActor: ActorRef,
                               jobTokenDispenserActor: ActorRef,
                               backendSingletonActor: Option[ActorRef],
                               backendName: String,
@@ -117,12 +119,8 @@ class EngineJobExecutionActor(replyTo: ActorRef,
     case Event(BackendJobPreparationSucceeded(jobDescriptor, bjeaProps), NoData) =>
       val updatedData = ResponsePendingData(jobDescriptor, bjeaProps)
       effectiveCallCachingMode match {
-        case activity: CallCachingActivity if activity.readFromCache =>
-          initializeJobHashing(jobDescriptor, activity)
-          goto(CheckingCallCache) using updatedData
-        case activity: CallCachingActivity =>
-          initializeJobHashing(jobDescriptor, activity)
-          runJob(updatedData)
+        case activity: CallCachingActivity if activity.readFromCache => handleReadFromCacheOn(jobDescriptor, activity, updatedData)
+        case activity: CallCachingActivity => handleReadFromCacheOff(jobDescriptor, activity, updatedData)
         case CallCachingOff => runJob(updatedData)
       }
     case Event(CallPreparationFailed(jobKey: BackendJobDescriptorKey, throwable), NoData) =>
@@ -140,7 +138,7 @@ class EngineJobExecutionActor(replyTo: ActorRef,
       fetchCachedResults(jobDescriptorKey.call.task.outputs, hit.cacheResultIds.head, data.copy(cacheHit = Option(hit)))
     case Event(HashError(t), data: ResponsePendingData) =>
       writeToMetadata(Map(callCachingReadResultMetadataKey -> s"Hashing Error: ${t.getMessage}"))
-      disableCallCaching(t)
+      disableCallCaching(Option(t))
       runJob(data)
   }
 
@@ -213,12 +211,16 @@ class EngineJobExecutionActor(replyTo: ActorRef,
       saveCacheResults(hashes, data)
     case Event(hashes: CallCacheHashes, data: ResponsePendingData) =>
       addHashesAndStay(data, hashes)
+    case Event(CacheMiss, _) =>
+      stay()
+    case Event(_: CacheHit, _) =>
+      stay()
 
     case Event(HashError(t), data: SucceededResponseData) =>
-      disableCallCaching(t)
+      disableCallCaching(Option(t))
       saveJobCompletionToJobStore(data.copy(hashes = Option(Failure(t))))
     case Event(HashError(t), data: ResponsePendingData) =>
-      disableCallCaching(t)
+      disableCallCaching(Option(t))
       stay using data.copy(hashes = Option(Failure(t)))
 
     case Event(response: JobSucceededResponse, data @ ResponsePendingData(_, _, Some(Success(hashes)), _)) if effectiveCallCachingMode.writeToCache =>
@@ -263,6 +265,31 @@ class EngineJobExecutionActor(replyTo: ActorRef,
       log.error("Bad message from {} to EngineJobExecutionActor in state {}(with data {}): {}", sender, stateName, stateData, msg)
       stay
   }
+  
+  private def handleReadFromCacheOn(jobDescriptor: BackendJobDescriptor, activity: CallCachingActivity, updatedData: ResponsePendingData) = {
+    jobDescriptor.callCachingEligibility match {
+        // If the job is eligible, initialize job hashing and go to CheckingCallCache state
+      case CallCachingEligible =>
+        initializeJobHashing(jobDescriptor, activity)
+        goto(CheckingCallCache) using updatedData
+      case ineligible: CallCachingIneligible =>
+        // If the job is ineligible, turn call caching off
+        writeToMetadata(Map(callCachingReadResultMetadataKey -> s"Cache Miss: ${ineligible.message}"))
+        disableCallCaching()
+        runJob(updatedData)
+    }
+  }
+
+  private def handleReadFromCacheOff(jobDescriptor: BackendJobDescriptor, activity: CallCachingActivity, updatedData: ResponsePendingData) = {
+    jobDescriptor.callCachingEligibility match {
+        // If the job is eligible, initialize job hashing so it can be written to the cache
+      case CallCachingEligible => initializeJobHashing(jobDescriptor, activity)
+      // Don't even initialize hashing to write to the cache if the job is ineligible
+      case ineligible: CallCachingIneligible => disableCallCaching()
+    }
+    // If read from cache is off, always run the job
+    runJob(updatedData)
+  }
 
   private def requestExecutionToken(): Unit = {
     jobTokenDispenserActor ! JobExecutionTokenRequest(factory.jobExecutionTokenType)
@@ -288,10 +315,11 @@ class EngineJobExecutionActor(replyTo: ActorRef,
     stay()
   }
 
-  private def disableCallCaching(reason: Throwable) = {
-    log.error(reason, "{}: Hash error, disabling call caching for this job.", jobTag)
+  private def disableCallCaching(reason: Option[Throwable] = None) = {
+    reason foreach { r => log.error(r, "{}: Hash error, disabling call caching for this job.", jobTag) }
     effectiveCallCachingMode = CallCachingOff
     writeCallCachingModeToMetadata()
+    writeToMetadata(Map(callCachingHitResultMetadataKey -> false))
   }
 
   private def disableCacheWrite(reason: Throwable) = {
@@ -310,9 +338,9 @@ class EngineJobExecutionActor(replyTo: ActorRef,
   def createJobPreparationActor(jobPrepProps: Props, name: String): ActorRef = context.actorOf(jobPrepProps, name)
   def prepareJob() = {
     val jobPreparationActorName = s"BackendPreparationActor_for_$jobTag"
-    val jobPrepProps = JobPreparationActor.props(executionData, jobDescriptorKey, factory, initializationData, serviceRegistryActor, backendSingletonActor)
+    val jobPrepProps = JobPreparationActor.props(executionData, jobDescriptorKey, factory, dockerHashActor, initializationData, serviceRegistryActor, backendSingletonActor)
     val jobPreparationActor = createJobPreparationActor(jobPrepProps, jobPreparationActorName)
-    jobPreparationActor ! CallPreparationActor.Start
+    jobPreparationActor ! CallPreparation.Start
     goto(PreparingJob)
   }
 
@@ -366,8 +394,8 @@ class EngineJobExecutionActor(replyTo: ActorRef,
 
   private def handleCacheInvalidatedResponse(response: CallCacheInvalidatedResponse, data: ResponsePendingData) = {
     def updateMetadataForInvalidatedEntry(entry: CallCachingEntry) = {
-      import cromwell.services.metadata.MetadataService.implicits.MetadataAutoPutter
       import cromwell.core.ExecutionIndex._
+      import cromwell.services.metadata.MetadataService.implicits.MetadataAutoPutter
       
       val workflowId = WorkflowId.fromString(entry.workflowExecutionUuid)
       // If the entry doesn't have an attempt, it means that this cache entry was added before this change
@@ -499,6 +527,7 @@ object EngineJobExecutionActor {
             serviceRegistryActor: ActorRef,
             jobStoreActor: ActorRef,
             callCacheReadActor: ActorRef,
+            dockerHashActor: ActorRef,
             jobTokenDispenserActor: ActorRef,
             backendSingletonActor: Option[ActorRef],
             backendName: String,
@@ -513,6 +542,7 @@ object EngineJobExecutionActor {
       serviceRegistryActor = serviceRegistryActor,
       jobStoreActor = jobStoreActor,
       callCacheReadActor = callCacheReadActor,
+      dockerHashActor = dockerHashActor,
       jobTokenDispenserActor = jobTokenDispenserActor,
       backendSingletonActor = backendSingletonActor,
       backendName = backendName: String,
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/JobPreparationActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/JobPreparationActor.scala
deleted file mode 100644
index 4bb6c2ab9..000000000
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/JobPreparationActor.scala
+++ /dev/null
@@ -1,138 +0,0 @@
-package cromwell.engine.workflow.lifecycle.execution
-
-import akka.actor.{Actor, ActorRef, Props}
-import cromwell.backend._
-import cromwell.core.Dispatcher.EngineDispatcher
-import cromwell.core.logging.WorkflowLogging
-import cromwell.core.{CallKey, JobKey, WorkflowId}
-import cromwell.engine.EngineWorkflowDescriptor
-import cromwell.engine.workflow.lifecycle.execution.CallPreparationActor._
-import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor.SubWorkflowKey
-import wdl4s._
-import wdl4s.exception.VariableLookupException
-import wdl4s.expression.WdlStandardLibraryFunctions
-import wdl4s.values.WdlValue
-
-import scala.util.{Failure, Success, Try}
-
-abstract class CallPreparationActor(val workflowDescriptor: EngineWorkflowDescriptor,
-                                    val outputStore: OutputStore,
-                                    callKey: CallKey) extends Actor with WorkflowLogging {
-  lazy val workflowIdForLogging = workflowDescriptor.id
-  def expressionLanguageFunctions: WdlStandardLibraryFunctions
-  def prepareExecutionActor(inputEvaluation: Map[Declaration, WdlValue]): CallPreparationActorResponse
-  
-  override def receive = {
-    case Start =>
-      val response = resolveAndEvaluateInputs() map { prepareExecutionActor }
-      context.parent ! (response recover { case f => CallPreparationFailed(callKey, f) }).get
-      context stop self
-
-    case unhandled => workflowLogger.warn(self.path.name + " received an unhandled message: " + unhandled)
-  }
-
-  def resolveAndEvaluateInputs(): Try[Map[Declaration, WdlValue]] = {
-
-    val call = callKey.scope
-    val scatterMap = callKey.index flatMap { i =>
-      // Will need update for nested scatters
-      call.ancestry collectFirst { case s: Scatter => Map(s -> i) }
-    } getOrElse Map.empty[Scatter, Int]
-
-    call.evaluateTaskInputs(
-      workflowDescriptor.backendDescriptor.knownValues,
-      expressionLanguageFunctions,
-      outputStore.fetchNodeOutputEntries,
-      scatterMap
-    ).recoverWith {
-      case t: Throwable => Failure(new VariableLookupException(s"Couldn't resolve all inputs for ${callKey.scope.fullyQualifiedName} at index ${callKey.index}.", List(t)))
-    }
-  }
-}
-
-final case class JobPreparationActor(executionData: WorkflowExecutionActorData,
-                                     jobKey: BackendJobDescriptorKey,
-                                     factory: BackendLifecycleActorFactory,
-                                     initializationData: Option[BackendInitializationData],
-                                     serviceRegistryActor: ActorRef,
-                                     backendSingletonActor: Option[ActorRef])
-  extends CallPreparationActor(executionData.workflowDescriptor, executionData.outputStore, jobKey) {
-
-  override lazy val expressionLanguageFunctions = factory.expressionLanguageFunctions(workflowDescriptor.backendDescriptor, jobKey, initializationData)
-  
-  override def prepareExecutionActor(inputEvaluation: Map[Declaration, WdlValue]): CallPreparationActorResponse = {
-    import RuntimeAttributeDefinition.{addDefaultsToAttributes, evaluateRuntimeAttributes}
-    val curriedAddDefaultsToAttributes = addDefaultsToAttributes(factory.runtimeAttributeDefinitions(initializationData), workflowDescriptor.backendDescriptor.workflowOptions) _
-
-    (for {
-      unevaluatedRuntimeAttributes <- Try(jobKey.call.task.runtimeAttributes)
-      evaluatedRuntimeAttributes <- evaluateRuntimeAttributes(unevaluatedRuntimeAttributes, expressionLanguageFunctions, inputEvaluation)
-      attributesWithDefault = curriedAddDefaultsToAttributes(evaluatedRuntimeAttributes)
-      jobDescriptor = BackendJobDescriptor(workflowDescriptor.backendDescriptor, jobKey, attributesWithDefault, inputEvaluation)
-    } yield BackendJobPreparationSucceeded(jobDescriptor, factory.jobExecutionActorProps(jobDescriptor, initializationData, serviceRegistryActor, backendSingletonActor))) match {
-      case Success(s) => s
-      case Failure(f) => CallPreparationFailed(jobKey, f)
-    }
-  }
-}
-
-final case class SubWorkflowPreparationActor(executionData: WorkflowExecutionActorData,
-                                             key: SubWorkflowKey,
-                                             subWorkflowId: WorkflowId)
-  extends CallPreparationActor(executionData.workflowDescriptor, executionData.outputStore, key) {
-
-  override lazy val expressionLanguageFunctions = executionData.expressionLanguageFunctions
-
-  override def prepareExecutionActor(inputEvaluation: Map[Declaration, WdlValue]): CallPreparationActorResponse = {
-    val oldBackendDescriptor = workflowDescriptor.backendDescriptor
-    
-    val newBackendDescriptor = oldBackendDescriptor.copy(
-      id = subWorkflowId,
-      workflow = key.scope.calledWorkflow,
-      knownValues = workflowDescriptor.knownValues ++ (inputEvaluation map { case (k, v) => k.fullyQualifiedName -> v }),
-      breadCrumbs = oldBackendDescriptor.breadCrumbs :+ BackendJobBreadCrumb(workflowDescriptor.workflow, workflowDescriptor.id, key)
-    )
-    val engineDescriptor = workflowDescriptor.copy(backendDescriptor = newBackendDescriptor, parentWorkflow = Option(workflowDescriptor))
-    SubWorkflowPreparationSucceeded(engineDescriptor, inputEvaluation)
-  }
-}
-
-object CallPreparationActor {
-  sealed trait CallPreparationActorCommands
-  case object Start extends CallPreparationActorCommands
-
-  sealed trait CallPreparationActorResponse
-  
-  case class BackendJobPreparationSucceeded(jobDescriptor: BackendJobDescriptor, bjeaProps: Props) extends CallPreparationActorResponse
-  case class SubWorkflowPreparationSucceeded(workflowDescriptor: EngineWorkflowDescriptor, inputs: EvaluatedTaskInputs) extends CallPreparationActorResponse
-  case class JobCallPreparationFailed(jobKey: JobKey, throwable: Throwable) extends CallPreparationActorResponse
-  case class CallPreparationFailed(jobKey: JobKey, throwable: Throwable) extends CallPreparationActorResponse
-}
-
-object JobPreparationActor {
-  def props(executionData: WorkflowExecutionActorData,
-            jobKey: BackendJobDescriptorKey,
-            factory: BackendLifecycleActorFactory,
-            initializationData: Option[BackendInitializationData],
-            serviceRegistryActor: ActorRef,
-            backendSingletonActor: Option[ActorRef]) = {
-    // Note that JobPreparationActor doesn't run on the engine dispatcher as it mostly executes backend-side code
-    // (WDL expression evaluation using Backend's expressionLanguageFunctions)
-    Props(new JobPreparationActor(executionData,
-      jobKey,
-      factory,
-      initializationData,
-      serviceRegistryActor,
-      backendSingletonActor)).withDispatcher(EngineDispatcher)
-  }
-}
-
-object SubWorkflowPreparationActor {
-  def props(executionData: WorkflowExecutionActorData,
-            key: SubWorkflowKey,
-            subWorkflowId: WorkflowId) = {
-    // Note that JobPreparationActor doesn't run on the engine dispatcher as it mostly executes backend-side code
-    // (WDL expression evaluation using Backend's expressionLanguageFunctions)
-    Props(new SubWorkflowPreparationActor(executionData, key, subWorkflowId)).withDispatcher(EngineDispatcher)
-  }
-}
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/SubWorkflowExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/SubWorkflowExecutionActor.scala
index 1fcf670c8..007326908 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/SubWorkflowExecutionActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/SubWorkflowExecutionActor.scala
@@ -8,9 +8,11 @@ import cromwell.core.Dispatcher.EngineDispatcher
 import cromwell.core.logging.JobLogging
 import cromwell.engine.EngineWorkflowDescriptor
 import cromwell.engine.backend.{BackendConfiguration, BackendSingletonCollection}
-import cromwell.engine.workflow.lifecycle.execution.CallPreparationActor.{CallPreparationFailed, Start, SubWorkflowPreparationSucceeded}
+import cromwell.engine.workflow.lifecycle.execution.preparation.CallPreparation.{CallPreparationFailed, Start}
 import cromwell.engine.workflow.lifecycle.execution.SubWorkflowExecutionActor._
 import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor._
+import cromwell.engine.workflow.lifecycle.execution.preparation.SubWorkflowPreparationActor
+import cromwell.engine.workflow.lifecycle.execution.preparation.SubWorkflowPreparationActor.SubWorkflowPreparationSucceeded
 import cromwell.services.metadata.MetadataService._
 import cromwell.services.metadata._
 import cromwell.subworkflowstore.SubWorkflowStoreActor._
@@ -23,6 +25,7 @@ class SubWorkflowExecutionActor(key: SubWorkflowKey,
                                 jobStoreActor: ActorRef,
                                 subWorkflowStoreActor: ActorRef,
                                 callCacheReadActor: ActorRef,
+                                dockerHashActor: ActorRef,
                                 jobTokenDispenserActor: ActorRef,
                                 backendSingletonCollection: BackendSingletonCollection,
                                 initializationData: AllBackendInitializationData,
@@ -149,6 +152,7 @@ class SubWorkflowExecutionActor(key: SubWorkflowKey,
       jobStoreActor,
       subWorkflowStoreActor,
       callCacheReadActor,
+      dockerHashActor,
       jobTokenDispenserActor,
       backendSingletonCollection,
       initializationData,
@@ -251,6 +255,7 @@ object SubWorkflowExecutionActor {
             jobStoreActor: ActorRef,
             subWorkflowStoreActor: ActorRef,
             callCacheReadActor: ActorRef,
+            dockerHashActor: ActorRef,
             jobTokenDispenserActor: ActorRef,
             backendSingletonCollection: BackendSingletonCollection,
             initializationData: AllBackendInitializationData,
@@ -263,6 +268,7 @@ object SubWorkflowExecutionActor {
       jobStoreActor,
       subWorkflowStoreActor,
       callCacheReadActor,
+      dockerHashActor,
       jobTokenDispenserActor,
       backendSingletonCollection,
       initializationData,
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala
index 02114ceb4..a3829f347 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala
@@ -34,6 +34,7 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor,
                                   jobStoreActor: ActorRef,
                                   subWorkflowStoreActor: ActorRef,
                                   callCacheReadActor: ActorRef,
+                                  dockerHashActor: ActorRef,
                                   jobTokenDispenserActor: ActorRef,
                                   backendSingletonCollection: BackendSingletonCollection,
                                   initializationData: AllBackendInitializationData,
@@ -445,7 +446,7 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor,
             val backendSingleton = backendSingletonCollection.backendSingletonActors(backendName)
             val ejeaProps = EngineJobExecutionActor.props(
               self, jobKey, data, factory, initializationData.get(backendName), restarting, serviceRegistryActor,
-              jobStoreActor, callCacheReadActor, jobTokenDispenserActor, backendSingleton, backendName, workflowDescriptor.callCachingMode)
+              jobStoreActor, callCacheReadActor, dockerHashActor, jobTokenDispenserActor, backendSingleton, backendName, workflowDescriptor.callCachingMode)
             val ejeaRef = context.actorOf(ejeaProps, ejeaName)
             context watch ejeaRef
             pushNewCallMetadata(jobKey, Option(backendName))
@@ -462,7 +463,7 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor,
   private def processRunnableSubWorkflow(key: SubWorkflowKey, data: WorkflowExecutionActorData): Try[WorkflowExecutionDiff] = {
     val sweaRef = context.actorOf(
       SubWorkflowExecutionActor.props(key, data, backendFactories, serviceRegistryActor, jobStoreActor, subWorkflowStoreActor,
-        callCacheReadActor, jobTokenDispenserActor, backendSingletonCollection, initializationData, restarting),
+        callCacheReadActor, dockerHashActor, jobTokenDispenserActor, backendSingletonCollection, initializationData, restarting),
       s"SubWorkflowExecutionActor-${key.tag}"
     )
 
@@ -732,12 +733,13 @@ object WorkflowExecutionActor {
             jobStoreActor: ActorRef,
             subWorkflowStoreActor: ActorRef,
             callCacheReadActor: ActorRef,
+            dockerHashActor: ActorRef,
             jobTokenDispenserActor: ActorRef,
             backendSingletonCollection: BackendSingletonCollection,
             initializationData: AllBackendInitializationData,
             restarting: Boolean): Props = {
     Props(WorkflowExecutionActor(workflowDescriptor, serviceRegistryActor, jobStoreActor, subWorkflowStoreActor,
-      callCacheReadActor, jobTokenDispenserActor, backendSingletonCollection, initializationData, restarting)).withDispatcher(EngineDispatcher)
+      callCacheReadActor, dockerHashActor, jobTokenDispenserActor, backendSingletonCollection, initializationData, restarting)).withDispatcher(EngineDispatcher)
   }
 
   implicit class EnhancedWorkflowOutputs(val outputs: Map[LocallyQualifiedName, WdlValue]) extends AnyVal {
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/preparation/CallPreparation.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/preparation/CallPreparation.scala
new file mode 100644
index 000000000..c39806191
--- /dev/null
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/preparation/CallPreparation.scala
@@ -0,0 +1,45 @@
+package cromwell.engine.workflow.lifecycle.execution.preparation
+
+import akka.actor.Props
+import cromwell.backend.BackendJobDescriptor
+import cromwell.core.{CallKey, JobKey}
+import cromwell.engine.EngineWorkflowDescriptor
+import cromwell.engine.workflow.lifecycle.execution.OutputStore
+import wdl4s.exception.VariableLookupException
+import wdl4s.expression.WdlStandardLibraryFunctions
+import wdl4s.values.WdlValue
+import wdl4s.{Declaration, Scatter}
+
+import scala.util.{Failure, Try}
+
+object CallPreparation {
+  sealed trait CallPreparationActorCommands
+  case object Start extends CallPreparationActorCommands
+
+  trait CallPreparationActorResponse
+
+  case class BackendJobPreparationSucceeded(jobDescriptor: BackendJobDescriptor, bjeaProps: Props) extends CallPreparationActorResponse
+
+  case class JobCallPreparationFailed(jobKey: JobKey, throwable: Throwable) extends CallPreparationActorResponse
+  case class CallPreparationFailed(jobKey: JobKey, throwable: Throwable) extends CallPreparationActorResponse
+
+  def resolveAndEvaluateInputs(callKey: CallKey,
+                               workflowDescriptor: EngineWorkflowDescriptor,
+                               expressionLanguageFunctions: WdlStandardLibraryFunctions,
+                               outputStore: OutputStore): Try[Map[Declaration, WdlValue]] = {
+    val call = callKey.scope
+    val scatterMap = callKey.index flatMap { i =>
+      // Will need update for nested scatters
+      call.ancestry collectFirst { case s: Scatter => Map(s -> i) }
+    } getOrElse Map.empty[Scatter, Int]
+
+    call.evaluateTaskInputs(
+      workflowDescriptor.backendDescriptor.knownValues,
+      expressionLanguageFunctions,
+      outputStore.fetchNodeOutputEntries,
+      scatterMap
+    ) recoverWith {
+      case t: Throwable => Failure[Map[Declaration, WdlValue]](new VariableLookupException(s"Couldn't resolve all inputs for ${callKey.scope.fullyQualifiedName} at index ${callKey.index}.", List(t)))
+    }
+  }
+}
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/preparation/JobPreparationActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/preparation/JobPreparationActor.scala
new file mode 100644
index 000000000..5524ff640
--- /dev/null
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/preparation/JobPreparationActor.scala
@@ -0,0 +1,215 @@
+package cromwell.engine.workflow.lifecycle.execution.preparation
+
+import akka.actor.{ActorRef, Cancellable, FSM, Props}
+import cromwell.backend._
+import cromwell.backend.validation.RuntimeAttributesKeys
+import cromwell.core.Dispatcher.EngineDispatcher
+import cromwell.core.callcaching.docker.DockerHashActor.{DockerHashBackPressure, DockerHashFailureResponse, DockerHashResponseSuccess}
+import cromwell.core.callcaching.docker._
+import cromwell.core.callcaching._
+import cromwell.core.logging.WorkflowLogging
+import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActorData
+import cromwell.engine.workflow.lifecycle.execution.preparation.CallPreparation._
+import cromwell.engine.workflow.lifecycle.execution.preparation.JobPreparationActor._
+import wdl4s._
+import wdl4s.values.{WdlString, WdlValue}
+
+import scala.concurrent.duration._
+import scala.language.postfixOps
+import scala.util.{Failure, Success, Try}
+
+case class JobPreparationActor(executionData: WorkflowExecutionActorData,
+                                     jobKey: BackendJobDescriptorKey,
+                                     factory: BackendLifecycleActorFactory,
+                                     dockerHashingActor: ActorRef,
+                                     initializationData: Option[BackendInitializationData],
+                                     serviceRegistryActor: ActorRef,
+                                     backendSingletonActor: Option[ActorRef])
+  extends FSM[JobPreparationActorState, Option[JobPreparationActorData]] with WorkflowLogging {
+
+  lazy val workflowIdForLogging = workflowDescriptor.id
+  
+  // Number of attempts to make contact with the docker actor before giving up if no response
+  private [preparation] lazy val maxDockerAttempts = 3
+  // Amount of time after which the docker request should be considered lost and sent agagin
+  private [preparation] lazy val dockerNoResponseTimeout = 30 seconds
+  // Amount of time to wait when we get a Backpressure response before sending the request again
+  private [preparation] lazy val backpressureWaitTime = 15 seconds
+  
+  private lazy val workflowDescriptor = executionData.workflowDescriptor
+  private val ec = context.system.dispatcher
+  private lazy val expressionLanguageFunctions = factory.expressionLanguageFunctions(workflowDescriptor.backendDescriptor, jobKey, initializationData)
+  private lazy val dockerHashCredentials = factory.dockerHashCredentials(initializationData)
+  
+  startWith(Idle, None)
+
+  when(Idle) {
+    case Event(Start, _) =>
+      evaluateInputsAndAttributes match {
+        case Success((inputs, attributes)) => startPreparation(inputs, attributes)
+        case Failure(failure) => sendFailureAndStop(failure)
+      }
+  }
+
+  when(WaitingForDockerHash) {
+    case Event(DockerHashResponseSuccess(dockerHash), Some(data)) =>
+      handleDockerHashSuccess(dockerHash, data)
+    case Event(failureResponse: DockerHashFailureResponse, Some(data)) =>
+      log.warning(failureResponse.reason)
+      handleDockerHashFailed(failureResponse.reason, data)
+    case Event(DockerHashBackPressure(request), Some(data)) =>
+      handleDockHashBackPressure(request, data)
+    case Event(DockerNoResponseTimeout, Some(data)) =>
+      if (data.maxAttemptsReached) handleDockerHashFailed("Docker Hash service is unreachable", data)
+      else {
+        dockerHashingActor ! data.dockerHashRequest
+        stay() using data.newAttempt(newRequestLostTimer)
+      }
+  }
+  
+  whenUnhandled {
+    case Event(unexpectedMessage, _) =>
+      workflowLogger.warn(s"JobPreparation actor received an unexpected message in state $stateName: $unexpectedMessage")
+      stay()
+  }
+  
+  private [preparation] def evaluateInputsAndAttributes = {
+    for {
+      evaluatedInputs <- resolveAndEvaluateInputs(jobKey, workflowDescriptor, expressionLanguageFunctions, executionData.outputStore)
+      runtimeAttributes <- prepareRuntimeAttributes(evaluatedInputs)
+    } yield (evaluatedInputs, runtimeAttributes)
+  }
+  
+  private def startPreparation(inputs: Map[Declaration, WdlValue], attributes: Map[LocallyQualifiedName, WdlValue]) = {
+    def sendDockerRequest(dockerImageId: DockerImageIdentifierWithoutHash) = {
+      val dockerHashRequest = DockerHashRequest(dockerImageId, dockerHashCredentials)
+      val newData = JobPreparationActorData(dockerHashRequest, inputs, attributes, newRequestLostTimer, maxDockerAttempts)
+      dockerHashingActor ! dockerHashRequest
+      goto(WaitingForDockerHash) using Option(newData)
+    }
+    
+    def handleDockerValue(value: String) = DockerImageIdentifier.fromString(value) match {
+      case Success(dockerImageId: DockerImageIdentifierWithoutHash) => sendDockerRequest(dockerImageId)
+      case Success(dockerImageIdWithHash: DockerImageIdentifierWithHash) =>
+        // If the docker value already has a hash - we're ok for call caching
+        val response = prepareBackendDescriptor(inputs, attributes, CallCachingEligible)
+        sendResponseAndStop(response)
+      case Failure(failure) => sendFailureAndStop(failure)
+    }
+
+    attributes.get(RuntimeAttributesKeys.DockerKey) match {
+      case Some(dockerValue) => handleDockerValue(dockerValue.valueString)
+      case None =>
+        // If there is no docker attribute at all - we're ok for call caching
+        val response = prepareBackendDescriptor(inputs, attributes, CallCachingEligible)
+        sendResponseAndStop(response)
+    }
+  }
+  
+  // Schedule a DockerNoResponseTimeout message to be sent to self
+  private final def newRequestLostTimer = context.system.scheduler.scheduleOnce(dockerNoResponseTimeout, self, DockerNoResponseTimeout)(ec, self)
+  // Schedule a DockerHashRequest to be sent to the docker hashing actor
+  private final def newBackPressureTimer(dockerHashRequest: DockerHashRequest) = context.system.scheduler.scheduleOnce(backpressureWaitTime, dockerHashingActor, dockerHashRequest)(ec, self)
+
+  /*
+   * If we get backpressured, recreate both timers
+   */
+  private def handleDockHashBackPressure(request: DockerHashRequest, data: JobPreparationActorData) = {
+    data.cancelTimers()
+    val requestLostTimer = newRequestLostTimer
+    val backpressureTimer = newBackPressureTimer(data.dockerHashRequest)
+    stay() using data.backpressure(requestLostTimer, backpressureTimer)
+  }
+  
+  private def handleDockerHashSuccess(dockerHashResult: DockerHashResult, data: JobPreparationActorData) = {
+    data.cancelTimers()
+    val dockerValueWithHash = data.dockerHashRequest.dockerImageID.withHash(dockerHashResult).fullName
+    val newRuntimeAttributes = data.attributes updated (RuntimeAttributesKeys.DockerKey, WdlString(dockerValueWithHash))
+    // We had to ask for the docker hash, which means the attribute had a floating tag, we're NOT ok for call caching
+    val response = prepareBackendDescriptor(data.inputs, newRuntimeAttributes, FloatingDockerTagWithHash(dockerValueWithHash))
+    sendResponseAndStop(response)
+  }
+
+  private def handleDockerHashFailed(failure: String, data: JobPreparationActorData) = {
+    data.cancelTimers()
+    val response = prepareBackendDescriptor(data.inputs, data.attributes, FloatingDockerTagWithoutHash)
+    sendResponseAndStop(response)
+  }
+
+  private def sendResponseAndStop(response: CallPreparationActorResponse) = {
+    context.parent ! response
+    stay()
+  }
+  
+  private def sendFailureAndStop(failure: Throwable) = {
+    sendResponseAndStop(CallPreparationFailed(jobKey, failure))
+  }
+  
+  private [preparation] def prepareBackendDescriptor(inputEvaluation: Map[Declaration, WdlValue],
+                               runtimeAttributes: Map[LocallyQualifiedName, WdlValue],
+                               callCachingEligibility: CallCachingEligibility) = {
+    val jobDescriptor = BackendJobDescriptor(workflowDescriptor.backendDescriptor, jobKey, runtimeAttributes, inputEvaluation, callCachingEligibility)
+    BackendJobPreparationSucceeded(jobDescriptor, factory.jobExecutionActorProps(jobDescriptor, initializationData, serviceRegistryActor, backendSingletonActor))
+  }
+  
+  private [preparation] def prepareRuntimeAttributes(inputEvaluation: Map[Declaration, WdlValue]): Try[Map[LocallyQualifiedName, WdlValue]] = {
+    import RuntimeAttributeDefinition.{addDefaultsToAttributes, evaluateRuntimeAttributes}
+    val curriedAddDefaultsToAttributes = addDefaultsToAttributes(factory.runtimeAttributeDefinitions(initializationData), workflowDescriptor.backendDescriptor.workflowOptions) _
+
+    for {
+      unevaluatedRuntimeAttributes <- Try(jobKey.call.task.runtimeAttributes)
+      evaluatedRuntimeAttributes <- evaluateRuntimeAttributes(unevaluatedRuntimeAttributes, expressionLanguageFunctions, inputEvaluation)
+    } yield curriedAddDefaultsToAttributes(evaluatedRuntimeAttributes)
+  }
+}
+
+object JobPreparationActor {
+  case class JobPreparationActorData(dockerHashRequest: DockerHashRequest,
+                                     inputs: Map[Declaration, WdlValue],
+                                     attributes: Map[LocallyQualifiedName, WdlValue],
+                                     requestLostTimer: Cancellable,
+                                     maxAttempts: Int,
+                                     currentAttempts: Int = 0,
+                                     backpressureTimer: Option[Cancellable] = None
+                                    ) {
+    def cancelTimers(): Unit = {
+      requestLostTimer.cancel()
+      backpressureTimer foreach { _.cancel() }
+      ()
+    }
+    
+    def backpressure(newBackpressureTimer: Cancellable, newRequestLostTimer: Cancellable) = {
+      Option(copy(requestLostTimer = newRequestLostTimer, backpressureTimer = Option(newBackpressureTimer)))
+    }
+    
+    def newAttempt(newBackpressureTimer: Cancellable) = {
+      Option(copy(requestLostTimer = newBackpressureTimer, currentAttempts = currentAttempts + 1))
+    }
+    
+    def maxAttemptsReached = currentAttempts == maxAttempts
+  }
+
+  sealed trait JobPreparationActorState
+  case object Idle extends JobPreparationActorState
+  case object WaitingForDockerHash extends JobPreparationActorState
+  
+  private case object DockerNoResponseTimeout
+  
+  def props(executionData: WorkflowExecutionActorData,
+            jobKey: BackendJobDescriptorKey,
+            factory: BackendLifecycleActorFactory,
+            dockerHashingActor: ActorRef,
+            initializationData: Option[BackendInitializationData],
+            serviceRegistryActor: ActorRef,
+            backendSingletonActor: Option[ActorRef]) = {
+    // Note that JobPreparationActor doesn't run on the engine dispatcher as it mostly executes backend-side code
+    // (WDL expression evaluation using Backend's expressionLanguageFunctions)
+    Props(new JobPreparationActor(executionData,
+      jobKey,
+      factory,
+      dockerHashingActor,
+      initializationData,
+      serviceRegistryActor,
+      backendSingletonActor)).withDispatcher(EngineDispatcher)
+  }
+}
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/preparation/SubWorkflowPreparationActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/preparation/SubWorkflowPreparationActor.scala
new file mode 100644
index 000000000..f4cbec0f5
--- /dev/null
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/preparation/SubWorkflowPreparationActor.scala
@@ -0,0 +1,60 @@
+package cromwell.engine.workflow.lifecycle.execution.preparation
+
+import akka.actor.{Actor, Props}
+import cromwell.backend.BackendJobBreadCrumb
+import cromwell.core.Dispatcher._
+import cromwell.core.WorkflowId
+import cromwell.core.logging.WorkflowLogging
+import cromwell.engine.EngineWorkflowDescriptor
+import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor.SubWorkflowKey
+import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActorData
+import cromwell.engine.workflow.lifecycle.execution.preparation.CallPreparation.{CallPreparationFailed, Start, _}
+import cromwell.engine.workflow.lifecycle.execution.preparation.SubWorkflowPreparationActor.SubWorkflowPreparationSucceeded
+import wdl4s._
+import wdl4s.values.WdlValue
+
+class SubWorkflowPreparationActor(executionData: WorkflowExecutionActorData,
+                                   callKey: SubWorkflowKey,
+                                   subWorkflowId: WorkflowId) extends Actor with WorkflowLogging {
+  
+  private val workflowDescriptor = executionData.workflowDescriptor
+  lazy val outputStore = executionData.outputStore
+  lazy val expressionLanguageFunctions = executionData.expressionLanguageFunctions
+
+  lazy val workflowIdForLogging = workflowDescriptor.id
+  
+  def prepareExecutionActor(inputEvaluation: Map[Declaration, WdlValue]): CallPreparationActorResponse = {
+    val oldBackendDescriptor = workflowDescriptor.backendDescriptor
+
+    val newBackendDescriptor = oldBackendDescriptor.copy(
+      id = subWorkflowId,
+      workflow = callKey.scope.calledWorkflow,
+      knownValues = workflowDescriptor.knownValues ++ (inputEvaluation map { case (k, v) => k.fullyQualifiedName -> v }),
+      breadCrumbs = oldBackendDescriptor.breadCrumbs :+ BackendJobBreadCrumb(workflowDescriptor.workflow, workflowDescriptor.id, callKey)
+    )
+    val engineDescriptor = workflowDescriptor.copy(backendDescriptor = newBackendDescriptor, parentWorkflow = Option(workflowDescriptor))
+    SubWorkflowPreparationSucceeded(engineDescriptor, inputEvaluation)
+  }
+
+  override def receive = {
+    case Start =>
+      val evaluatedInputs = resolveAndEvaluateInputs(callKey, workflowDescriptor, expressionLanguageFunctions, outputStore)
+      val response = evaluatedInputs map { prepareExecutionActor }
+      context.parent ! (response recover { case f => CallPreparationFailed(callKey, f) }).get
+      context stop self
+
+    case unhandled => workflowLogger.warn(self.path.name + " received an unhandled message: " + unhandled)
+  }
+}
+
+object SubWorkflowPreparationActor {
+  case class SubWorkflowPreparationSucceeded(workflowDescriptor: EngineWorkflowDescriptor, inputs: EvaluatedTaskInputs) extends CallPreparationActorResponse
+
+  def props(executionData: WorkflowExecutionActorData,
+            key: SubWorkflowKey,
+            subWorkflowId: WorkflowId) = {
+    // Note that JobPreparationActor doesn't run on the engine dispatcher as it mostly executes backend-side code
+    // (WDL expression evaluation using Backend's expressionLanguageFunctions)
+    Props(new SubWorkflowPreparationActor(executionData, key, subWorkflowId)).withDispatcher(EngineDispatcher)
+  }
+}
diff --git a/engine/src/main/scala/cromwell/server/CromwellRootActor.scala b/engine/src/main/scala/cromwell/server/CromwellRootActor.scala
index 780b7c7ad..e9e8878f7 100644
--- a/engine/src/main/scala/cromwell/server/CromwellRootActor.scala
+++ b/engine/src/main/scala/cromwell/server/CromwellRootActor.scala
@@ -1,10 +1,18 @@
 package cromwell.server
 
-import akka.actor.SupervisorStrategy.Escalate
+import akka.actor.SupervisorStrategy.{Escalate, Restart}
 import akka.actor.{Actor, ActorInitializationException, ActorRef, OneForOneStrategy}
 import akka.event.Logging
+import akka.http.scaladsl.Http
+import akka.http.scaladsl.model.HttpRequest
 import akka.routing.RoundRobinPool
+import akka.stream.ActorMaterializer
 import com.typesafe.config.ConfigFactory
+import cromwell.core.Dispatcher
+import cromwell.core.callcaching.docker.DockerHashActor
+import cromwell.core.callcaching.docker.DockerHashActor.{DockerHashActorException, DockerHashContext}
+import cromwell.core.callcaching.docker.registryv2.flows.dockerhub.DockerHubFlow
+import cromwell.core.callcaching.docker.registryv2.flows.gcr.GoogleFlow
 import cromwell.engine.backend.{BackendSingletonCollection, CromwellBackends}
 import cromwell.engine.workflow.WorkflowManagerActor
 import cromwell.engine.workflow.lifecycle.CopyWorkflowLogsActor
@@ -15,6 +23,10 @@ import cromwell.jobstore.{JobStore, JobStoreActor, SqlJobStore}
 import cromwell.services.{ServiceRegistryActor, SingletonServicesStore}
 import cromwell.subworkflowstore.{SqlSubWorkflowStore, SubWorkflowStoreActor}
 import net.ceedubs.ficus.Ficus._
+
+import scala.concurrent.duration._
+import scala.language.postfixOps
+
 /**
   * An actor which serves as the lord protector for the rest of Cromwell, allowing us to have more fine grain
   * control on top level supervision, etc.
@@ -30,6 +42,9 @@ import net.ceedubs.ficus.Ficus._
 
   private val logger = Logging(context.system, this)
   private val config = ConfigFactory.load()
+  private implicit val system = context.system
+  private implicit val materializer = ActorMaterializer()
+  
   val serverMode: Boolean
 
   lazy val serviceRegistryActor: ActorRef = context.actorOf(ServiceRegistryActor.props(config), "ServiceRegistryActor")
@@ -53,6 +68,20 @@ import net.ceedubs.ficus.Ficus._
   lazy val callCacheReadActor = context.actorOf(RoundRobinPool(25)
     .props(CallCacheReadActor.props(callCache)),
     "CallCacheReadActor")
+  
+  // Docker Actor
+  lazy val ioEc = context.system.dispatchers.lookup(Dispatcher.IoDispatcher)
+  lazy val gcrQueriesPer100Sec = config.getAs[Int]("docker.gcr-api-queries-per-100-seconds") getOrElse 1000
+  lazy val dockerCacheEntryTTL = config.as[Option[FiniteDuration]]("docker.cache-entry-ttl").getOrElse(DefaultCacheTTL)
+  lazy val dockerCacheSize = config.getAs[Long]("docker.cache-size") getOrElse 200L
+  // Sets the number of requests that the docker actor will accept before it starts backpressuring (modulo the number of in flight requests)
+  lazy val dockerActorQueueSize = 500
+  
+  lazy val dockerHttpPool = Http().superPool[(DockerHashContext, HttpRequest)]()
+  lazy val googleFlow = new GoogleFlow(dockerHttpPool, gcrQueriesPer100Sec)(ioEc, materializer)
+  lazy val dockerHubFlow = new DockerHubFlow(dockerHttpPool)(ioEc, materializer)
+  lazy val dockerFlows = Seq(dockerHubFlow, googleFlow)
+  lazy val dockerHashActor = context.actorOf(DockerHashActor.props(dockerFlows, dockerActorQueueSize, dockerCacheEntryTTL, dockerCacheSize)(materializer).withDispatcher(Dispatcher.IoDispatcher))
 
   lazy val backendSingletons = CromwellBackends.instance.get.backendLifecycleActorFactories map {
     case (name, factory) => name -> (factory.backendSingletonActorProps map context.actorOf)
@@ -66,7 +95,7 @@ import net.ceedubs.ficus.Ficus._
   lazy val workflowManagerActor = context.actorOf(
     WorkflowManagerActor.props(
       workflowStoreActor, serviceRegistryActor, workflowLogCopyRouter, jobStoreActor, subWorkflowStoreActor, callCacheReadActor,
-      jobExecutionTokenDispenserActor, backendSingletonCollection, abortJobsOnTerminate, serverMode),
+      dockerHashActor, jobExecutionTokenDispenserActor, backendSingletonCollection, abortJobsOnTerminate, serverMode),
     "WorkflowManagerActor")
 
   override def receive = {
@@ -79,10 +108,12 @@ import net.ceedubs.ficus.Ficus._
     */
   override val supervisorStrategy = OneForOneStrategy() {
     case actorInitializationException: ActorInitializationException => Escalate
+    case dockerHash: DockerHashActorException => Restart
     case t => super.supervisorStrategy.decider.applyOrElse(t, (_: Any) => Escalate)
   }
 }
 
 object CromwellRootActor {
   val DefaultNumberOfWorkflowLogCopyWorkers = 10
+  val DefaultCacheTTL = 20 minutes
 }
diff --git a/engine/src/test/scala/cromwell/CromwellTestKitSpec.scala b/engine/src/test/scala/cromwell/CromwellTestKitSpec.scala
index ddfeafb07..b28a61a3f 100644
--- a/engine/src/test/scala/cromwell/CromwellTestKitSpec.scala
+++ b/engine/src/test/scala/cromwell/CromwellTestKitSpec.scala
@@ -10,6 +10,8 @@ import com.typesafe.config.{Config, ConfigFactory}
 import cromwell.CromwellTestKitSpec._
 import cromwell.backend._
 import cromwell.core._
+import cromwell.core.callcaching.docker.DockerHashActor.DockerHashResponseSuccess
+import cromwell.core.callcaching.docker.{DockerHashRequest, DockerHashResult}
 import cromwell.core.path.BetterFileMethods.Cmds
 import cromwell.core.path.DefaultPathBuilder
 import cromwell.engine.backend.BackendConfigurationEntry
@@ -473,3 +475,13 @@ class EmptyCallCacheReadActor extends Actor {
 object EmptyCallCacheReadActor {
   def props: Props = Props(new EmptyCallCacheReadActor)
 }
+
+class EmptyDockerHashActor extends Actor {
+  override def receive: Receive = {
+    case DockerHashRequest(image, _) => sender ! DockerHashResponseSuccess(DockerHashResult("alg", "hash"))
+  }
+}
+
+object EmptyDockerHashActor {
+  def props: Props = Props(new EmptyDockerHashActor)
+}
diff --git a/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala b/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala
index 8eb60d45f..511cbaa27 100644
--- a/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala
+++ b/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala
@@ -45,6 +45,7 @@ class SimpleWorkflowActorSpec extends CromwellTestKitSpec with BeforeAndAfter {
         jobStoreActor = system.actorOf(AlwaysHappyJobStoreActor.props),
         subWorkflowStoreActor = system.actorOf(AlwaysHappySubWorkflowStoreActor.props),
         callCacheReadActor = system.actorOf(EmptyCallCacheReadActor.props),
+        dockerHashActor = system.actorOf(EmptyDockerHashActor.props),
         jobTokenDispenserActor = system.actorOf(JobExecutionTokenDispenserActor.props),
         backendSingletonCollection = BackendSingletonCollection(Map("Local" -> None)),
         serverMode = true),
diff --git a/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala
index fbe1880b1..820091fef 100644
--- a/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala
+++ b/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala
@@ -17,7 +17,7 @@ import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor
 import cromwell.engine.workflow.workflowstore.{InMemoryWorkflowStore, WorkflowStoreActor}
 import cromwell.util.SampleWdl
 import cromwell.util.SampleWdl.{ExpressionsInInputs, GoodbyeWorld, ThreeStep}
-import cromwell.{AlwaysHappyJobStoreActor, AlwaysHappySubWorkflowStoreActor, CromwellTestKitSpec, EmptyCallCacheReadActor}
+import cromwell._
 import org.scalatest.prop.{TableDrivenPropertyChecks, TableFor3}
 import spray.json._
 
@@ -57,6 +57,7 @@ abstract class SingleWorkflowRunnerActorSpec extends CromwellTestKitSpec {
   private val jobStore = system.actorOf(AlwaysHappyJobStoreActor.props)
   private val subWorkflowStore = system.actorOf(AlwaysHappySubWorkflowStoreActor.props)
   private val callCacheReadActor = system.actorOf(EmptyCallCacheReadActor.props)
+  private val dockerHashActor = system.actorOf(EmptyDockerHashActor.props)
   private val jobTokenDispenserActor = system.actorOf(JobExecutionTokenDispenserActor.props)
 
 
@@ -68,6 +69,7 @@ abstract class SingleWorkflowRunnerActorSpec extends CromwellTestKitSpec {
       jobStore,
       subWorkflowStore,
       callCacheReadActor,
+      dockerHashActor,
       jobTokenDispenserActor,
       BackendSingletonCollection(Map.empty),
       abortJobsOnTerminate = false,
@@ -118,6 +120,7 @@ class SingleWorkflowRunnerActorWithMetadataSpec extends SingleWorkflowRunnerActo
         outputFile = Option(metadataFile))
         TestKit.shutdownActorSystem(system, TimeoutDuration)
     }
+    
     eventually {
       val metadataFileContent = metadataFile.contentAsString
       val metadata = metadataFileContent.parseJson.asJsObject.fields
diff --git a/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala
index 1995b480b..35bcb2ca1 100644
--- a/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala
+++ b/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala
@@ -60,6 +60,7 @@ class WorkflowActorSpec extends CromwellTestKitSpec with WorkflowDescriptorBuild
         jobStoreActor = system.actorOf(AlwaysHappyJobStoreActor.props),
         subWorkflowStoreActor = system.actorOf(AlwaysHappySubWorkflowStoreActor.props),
         callCacheReadActor = system.actorOf(EmptyCallCacheReadActor.props),
+        dockerHashActor = system.actorOf(EmptyDockerHashActor.props),
         jobTokenDispenserActor = TestProbe().ref
       ),
       supervisor = supervisorProbe.ref)
@@ -173,7 +174,8 @@ class MockWorkflowActor(val finalizationProbe: TestProbe,
                         jobStoreActor: ActorRef,
                         subWorkflowStoreActor: ActorRef,
                         callCacheReadActor: ActorRef,
-                        jobTokenDispenserActor: ActorRef) extends WorkflowActor(workflowId, startMode, workflowSources, conf, serviceRegistryActor, workflowLogCopyRouter, jobStoreActor, subWorkflowStoreActor, callCacheReadActor, jobTokenDispenserActor, BackendSingletonCollection(Map.empty), serverMode = true) {
+                        dockerHashActor: ActorRef,
+                        jobTokenDispenserActor: ActorRef) extends WorkflowActor(workflowId, startMode, workflowSources, conf, serviceRegistryActor, workflowLogCopyRouter, jobStoreActor, subWorkflowStoreActor, callCacheReadActor, dockerHashActor, jobTokenDispenserActor, BackendSingletonCollection(Map.empty), serverMode = true) {
 
   override def makeFinalizationActor(workflowDescriptor: EngineWorkflowDescriptor, jobExecutionMap: JobExecutionMap, worfklowOutputs: CallOutputs) = finalizationProbe.ref
 }
diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/SubWorkflowExecutionActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/SubWorkflowExecutionActorSpec.scala
index 32a75e57f..6cae26763 100644
--- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/SubWorkflowExecutionActorSpec.scala
+++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/SubWorkflowExecutionActorSpec.scala
@@ -9,9 +9,11 @@ import cromwell.core._
 import cromwell.core.callcaching.CallCachingOff
 import cromwell.database.sql.tables.SubWorkflowStoreEntry
 import cromwell.engine.backend.BackendSingletonCollection
-import cromwell.engine.workflow.lifecycle.execution.CallPreparationActor.{CallPreparationFailed, SubWorkflowPreparationSucceeded}
 import cromwell.engine.workflow.lifecycle.execution.SubWorkflowExecutionActor._
 import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor._
+import cromwell.engine.workflow.lifecycle.execution.preparation.CallPreparation
+import cromwell.engine.workflow.lifecycle.execution.preparation.CallPreparation.CallPreparationFailed
+import cromwell.engine.workflow.lifecycle.execution.preparation.SubWorkflowPreparationActor.SubWorkflowPreparationSucceeded
 import cromwell.engine.{ContinueWhilePossible, EngineWorkflowDescriptor}
 import cromwell.subworkflowstore.SubWorkflowStoreActor.{QuerySubWorkflow, SubWorkflowFound, SubWorkflowNotFound}
 import org.scalatest.concurrent.Eventually
@@ -30,6 +32,7 @@ class SubWorkflowExecutionActorSpec extends TestKitSuite with FlatSpecLike with
   val jobStoreProbe = TestProbe()
   val subWorkflowStoreProbe = TestProbe()
   val callCacheReadActorProbe = TestProbe()
+  val dockerHashActorProbe = TestProbe()
   val jobTokenDispenserProbe = TestProbe()
   val preparationActor = TestProbe()
   val subWorkflowActor = TestProbe()
@@ -65,6 +68,7 @@ class SubWorkflowExecutionActorSpec extends TestKitSuite with FlatSpecLike with
         jobStoreProbe.ref,
         subWorkflowStoreProbe.ref,
         callCacheReadActorProbe.ref,
+        dockerHashActorProbe.ref,
         jobTokenDispenserProbe.ref,
         BackendSingletonCollection(Map.empty),
         AllBackendInitializationData(Map.empty),
@@ -94,7 +98,7 @@ class SubWorkflowExecutionActorSpec extends TestKitSuite with FlatSpecLike with
     
     val subWorkflowUuid = WorkflowId.randomId()
     ewea ! SubWorkflowFound(SubWorkflowStoreEntry(Option(0), parentWorkflowId.toString, subKey.scope.fullyQualifiedName, subKey.index.fromIndex, subKey.attempt, subWorkflowUuid.toString, None))
-    preparationActor.expectMsg(CallPreparationActor.Start)
+    preparationActor.expectMsg(CallPreparation.Start)
     parentProbe.expectMsg(JobStarting(subKey))
     
     eventually {
@@ -108,7 +112,7 @@ class SubWorkflowExecutionActorSpec extends TestKitSuite with FlatSpecLike with
     ewea.setState(SubWorkflowCheckingStoreState)
 
     ewea ! SubWorkflowNotFound(QuerySubWorkflow(parentWorkflowId, subKey))
-    preparationActor.expectMsg(CallPreparationActor.Start)
+    preparationActor.expectMsg(CallPreparation.Start)
     parentProbe.expectMsg(JobStarting(subKey))
     
     eventually {
@@ -122,7 +126,7 @@ class SubWorkflowExecutionActorSpec extends TestKitSuite with FlatSpecLike with
     ewea.setState(SubWorkflowPendingState)
     
     ewea ! Execute
-    preparationActor.expectMsg(CallPreparationActor.Start)
+    preparationActor.expectMsg(CallPreparation.Start)
     parentProbe.expectMsg(JobStarting(subKey))
     eventually {
       ewea.stateName shouldBe SubWorkflowPreparingState
diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorSpec.scala
index 51007a2df..f3106bc21 100644
--- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorSpec.scala
+++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorSpec.scala
@@ -65,10 +65,11 @@ class WorkflowExecutionActorSpec extends CromwellTestKitSpec with BeforeAndAfter
       val workflowId = WorkflowId.randomId()
       val engineWorkflowDescriptor = createMaterializedEngineWorkflowDescriptor(workflowId, SampleWdl.HelloWorld.asWorkflowSources(runtime = runtimeSection))
       val callCacheReadActor = TestProbe()
+      val dockerHashActor = TestProbe()
 
       val workflowExecutionActor = system.actorOf(
         WorkflowExecutionActor.props(engineWorkflowDescriptor, serviceRegistryActor, jobStoreActor, subWorkflowStoreActor,
-          callCacheReadActor.ref, jobTokenDispenserActor, MockBackendSingletonCollection, AllBackendInitializationData.empty, restarting = false),
+          callCacheReadActor.ref, dockerHashActor.ref, jobTokenDispenserActor, MockBackendSingletonCollection, AllBackendInitializationData.empty, restarting = false),
         "WorkflowExecutionActor")
 
       EventFilter.info(pattern = ".*Final Outputs", occurrences = 1).intercept {
@@ -89,6 +90,7 @@ class WorkflowExecutionActorSpec extends CromwellTestKitSpec with BeforeAndAfter
       val jobStore = system.actorOf(AlwaysHappyJobStoreActor.props)
       val subWorkflowStoreActor = system.actorOf(AlwaysHappySubWorkflowStoreActor.props)
       val callCacheReadActor = system.actorOf(EmptyCallCacheReadActor.props)
+      val dockerHashActor = system.actorOf(EmptyDockerHashActor.props)
       val jobTokenDispenserActor = system.actorOf(JobExecutionTokenDispenserActor.props)
 
       val MockBackendConfigEntry = BackendConfigurationEntry(
@@ -102,7 +104,7 @@ class WorkflowExecutionActorSpec extends CromwellTestKitSpec with BeforeAndAfter
       val engineWorkflowDescriptor = createMaterializedEngineWorkflowDescriptor(workflowId, SampleWdl.SimpleScatterWdl.asWorkflowSources(runtime = runtimeSection))
       val workflowExecutionActor = system.actorOf(
         WorkflowExecutionActor.props(engineWorkflowDescriptor, serviceRegistry, jobStore, subWorkflowStoreActor,
-          callCacheReadActor, jobTokenDispenserActor, MockBackendSingletonCollection, AllBackendInitializationData.empty, restarting = false),
+          callCacheReadActor, dockerHashActor, jobTokenDispenserActor, MockBackendSingletonCollection, AllBackendInitializationData.empty, restarting = false),
         "WorkflowExecutionActor")
 
       val scatterLog = "Starting calls: scatter0.inside_scatter:0:1, scatter0.inside_scatter:1:1, scatter0.inside_scatter:2:1, scatter0.inside_scatter:3:1, scatter0.inside_scatter:4:1"
diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActorSpec.scala
index 34ac835cd..9c718c5d6 100644
--- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActorSpec.scala
+++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActorSpec.scala
@@ -179,6 +179,7 @@ object EngineJobHashingActorSpec extends BackendSpec {
     backendName: String = "whatever"
   )(implicit system: ActorSystem) = {
     val callCacheReadActor = system.actorOf(Props(new PredictableCallCacheReadActor(cacheLookupResponses)))
+    
     system.actorOf(EngineJobHashingActor.props(
       receiver = replyTo,
       jobDescriptor = jobDescriptor,
@@ -199,7 +200,7 @@ object EngineJobHashingActorSpec extends BackendSpec {
     when(task.outputs).thenReturn(List.empty)
     when(call.task).thenReturn(task)
     val workflowDescriptor = mock[BackendWorkflowDescriptor]
-    val jobDescriptor = BackendJobDescriptor(workflowDescriptor, BackendJobDescriptorKey(call, None, 1), Map.empty, fqnMapToDeclarationMap(inputs))
+    val jobDescriptor = BackendJobDescriptor(workflowDescriptor, BackendJobDescriptorKey(call, None, 1), Map.empty, fqnMapToDeclarationMap(inputs), CallCachingEligible)
     jobDescriptor
   }
 
diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaCheckingJobStoreSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaCheckingJobStoreSpec.scala
index 5a9bb4f5c..433f87223 100644
--- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaCheckingJobStoreSpec.scala
+++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaCheckingJobStoreSpec.scala
@@ -2,9 +2,9 @@ package cromwell.engine.workflow.lifecycle.execution.ejea
 
 import cromwell.backend.BackendJobExecutionActor.{JobFailedNonRetryableResponse, JobFailedRetryableResponse, JobSucceededResponse}
 import cromwell.core._
-import cromwell.engine.workflow.lifecycle.execution.CallPreparationActor
 import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor.{CheckingJobStore, NoData, PreparingJob}
 import cromwell.engine.workflow.lifecycle.execution.ejea.EngineJobExecutionActorSpec.EnhancedTestEJEA
+import cromwell.engine.workflow.lifecycle.execution.preparation.CallPreparation
 import cromwell.jobstore.JobStoreActor.{JobComplete, JobNotComplete}
 import cromwell.jobstore.{JobResultFailure, JobResultSuccess}
 
@@ -59,7 +59,7 @@ class EjeaCheckingJobStoreSpec extends EngineJobExecutionActorSpec {
       ejea.setState(CheckingJobStore)
       ejea ! JobNotComplete
 
-      helper.jobPreparationProbe.expectMsg(awaitTimeout, "expecting RecoverJobCommand", CallPreparationActor.Start)
+      helper.jobPreparationProbe.expectMsg(awaitTimeout, "expecting RecoverJobCommand", CallPreparation.Start)
       ejea.stateName should be(PreparingJob)
 
       ejea.stop()
diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaPreparingJobSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaPreparingJobSpec.scala
index bdb426b60..c4b14eec0 100644
--- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaPreparingJobSpec.scala
+++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaPreparingJobSpec.scala
@@ -2,9 +2,9 @@ package cromwell.engine.workflow.lifecycle.execution.ejea
 
 import cromwell.backend.BackendJobExecutionActor.JobFailedNonRetryableResponse
 import cromwell.core.callcaching.CallCachingMode
-import cromwell.engine.workflow.lifecycle.execution.CallPreparationActor.{BackendJobPreparationSucceeded, CallPreparationFailed}
 import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor._
 import cromwell.engine.workflow.lifecycle.execution.ejea.EngineJobExecutionActorSpec._
+import cromwell.engine.workflow.lifecycle.execution.preparation.CallPreparation.{BackendJobPreparationSucceeded, CallPreparationFailed}
 import org.scalatest.concurrent.Eventually
 
 class EjeaPreparingJobSpec extends EngineJobExecutionActorSpec with CanExpectHashingInitialization with Eventually {
diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaRequestingExecutionTokenSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaRequestingExecutionTokenSpec.scala
index 3047de560..543d54550 100644
--- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaRequestingExecutionTokenSpec.scala
+++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaRequestingExecutionTokenSpec.scala
@@ -1,7 +1,7 @@
 package cromwell.engine.workflow.lifecycle.execution.ejea
 
-import cromwell.engine.workflow.lifecycle.execution.CallPreparationActor
 import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor._
+import cromwell.engine.workflow.lifecycle.execution.preparation.CallPreparation
 import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor.{JobExecutionTokenDenied, JobExecutionTokenDispensed}
 import cromwell.jobstore.JobStoreActor.QueryJobCompletion
 import org.scalatest.concurrent.Eventually
@@ -45,7 +45,7 @@ class EjeaRequestingExecutionTokenSpec extends EngineJobExecutionActorSpec with
         ejea = helper.buildEJEA(restarting = false)
         ejea ! JobExecutionTokenDispensed(helper.executionToken)
 
-        helper.jobPreparationProbe.expectMsg(max = awaitTimeout, hint = "Awaiting job preparation", CallPreparationActor.Start)
+        helper.jobPreparationProbe.expectMsg(max = awaitTimeout, hint = "Awaiting job preparation", CallPreparation.Start)
         helper.jobStoreProbe.expectNoMsg(awaitAlmostNothing)
         ejea.stateName should be(PreparingJob)
       }
diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala
index eb312013c..1ceef04c4 100644
--- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala
+++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala
@@ -7,7 +7,7 @@ import akka.testkit.{TestFSMRef, TestProbe}
 import cromwell.backend.BackendJobExecutionActor.JobSucceededResponse
 import cromwell.backend._
 import cromwell.core.JobExecutionToken.JobExecutionTokenType
-import cromwell.core.callcaching.{CallCachingActivity, CallCachingMode, CallCachingOff}
+import cromwell.core.callcaching.{CallCachingActivity, CallCachingEligible, CallCachingMode, CallCachingOff}
 import cromwell.core.{CallOutputs, JobExecutionToken, WorkflowId}
 import cromwell.engine.EngineWorkflowDescriptor
 import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor.{EJEAData, EngineJobExecutionActorState}
@@ -53,7 +53,7 @@ private[ejea] class PerTestHelper(implicit val system: ActorSystem) extends Mock
   val jobDescriptorKey = BackendJobDescriptorKey(call, jobIndex, jobAttempt)
 
   val backendWorkflowDescriptor = BackendWorkflowDescriptor(workflowId, null, null, null, null)
-  val backendJobDescriptor = BackendJobDescriptor(backendWorkflowDescriptor, jobDescriptorKey, runtimeAttributes = Map.empty, inputDeclarations = Map.empty)
+  val backendJobDescriptor = BackendJobDescriptor(backendWorkflowDescriptor, jobDescriptorKey, runtimeAttributes = Map.empty, inputDeclarations = Map.empty, CallCachingEligible)
 
   var fetchCachedResultsActorCreations: ExpectOne[(CallCachingEntryId, Seq[TaskOutput])] = NothingYet
   var jobHashingInitializations: ExpectOne[(BackendJobDescriptor, CallCachingActivity)] = NothingYet
@@ -68,6 +68,7 @@ private[ejea] class PerTestHelper(implicit val system: ActorSystem) extends Mock
   val serviceRegistryProbe = TestProbe()
   val jobStoreProbe = TestProbe()
   val callCacheReadActorProbe = TestProbe()
+  val dockerHashActorProbe = TestProbe()
   val callCacheHitCopyingProbe = TestProbe()
   val jobPreparationProbe = TestProbe()
   val jobTokenDispenserProbe = TestProbe()
@@ -115,6 +116,7 @@ private[ejea] class PerTestHelper(implicit val system: ActorSystem) extends Mock
       serviceRegistryActor = serviceRegistryProbe.ref,
       jobStoreActor = jobStoreProbe.ref,
       callCacheReadActor = callCacheReadActorProbe.ref,
+      dockerHashActor = dockerHashActorProbe.ref,
       jobTokenDispenserActor = jobTokenDispenserProbe.ref,
       backendName = "NOT USED",
       callCachingMode = callCachingMode
@@ -136,9 +138,10 @@ private[ejea] class MockEjea(helper: PerTestHelper,
                              serviceRegistryActor: ActorRef,
                              jobStoreActor: ActorRef,
                              callCacheReadActor: ActorRef,
+                             dockerHashActor: ActorRef,
                              jobTokenDispenserActor: ActorRef,
                              backendName: String,
-                             callCachingMode: CallCachingMode) extends EngineJobExecutionActor(replyTo, jobDescriptorKey, executionData, factory, initializationData, restarting, serviceRegistryActor, jobStoreActor, callCacheReadActor, jobTokenDispenserActor, None, backendName, callCachingMode) {
+                             callCachingMode: CallCachingMode) extends EngineJobExecutionActor(replyTo, jobDescriptorKey, executionData, factory, initializationData, restarting, serviceRegistryActor, jobStoreActor, callCacheReadActor, dockerHashActor, jobTokenDispenserActor, None, backendName, callCachingMode) {
 
   override def makeFetchCachedResultsActor(cacheId: CallCachingEntryId, taskOutputs: Seq[TaskOutput]) = helper.fetchCachedResultsActorCreations = helper.fetchCachedResultsActorCreations.foundOne((cacheId, taskOutputs))
   override def initializeJobHashing(jobDescriptor: BackendJobDescriptor, activity: CallCachingActivity) = helper.jobHashingInitializations = helper.jobHashingInitializations.foundOne((jobDescriptor, activity))
diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/preparation/JobPreparationActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/preparation/JobPreparationActorSpec.scala
new file mode 100644
index 000000000..837f87e96
--- /dev/null
+++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/preparation/JobPreparationActorSpec.scala
@@ -0,0 +1,164 @@
+package cromwell.engine.workflow.lifecycle.execution.preparation
+
+import akka.testkit.{ImplicitSender, TestActorRef, TestProbe}
+import cromwell.core.callcaching.{CallCachingEligible, CallCachingIneligible}
+import cromwell.core.callcaching.docker.DockerHashActor.{DockerHashBackPressure, DockerHashFailedResponse, DockerHashResponseSuccess}
+import cromwell.core.callcaching.docker.{DockerHashRequest, DockerHashResult, DockerImageIdentifier, DockerImageIdentifierWithoutHash}
+import cromwell.core.{LocallyQualifiedName, TestKitSuite}
+import cromwell.engine.workflow.lifecycle.execution.preparation.CallPreparation.{BackendJobPreparationSucceeded, CallPreparationFailed, Start}
+import org.scalatest.{FlatSpecLike, Matchers}
+import wdl4s.Declaration
+import wdl4s.values.{WdlString, WdlValue}
+
+import scala.util.{Failure, Success}
+import scala.concurrent.duration._
+import scala.language.postfixOps
+
+class JobPreparationActorSpec extends TestKitSuite with FlatSpecLike with Matchers with ImplicitSender {
+
+  behavior of "JobPreparationActor"
+  
+  val helper = new JobPreparationTestHelper()
+  val inputs = Map.empty[Declaration, WdlValue]
+  
+  it should "fail preparation if it can't evaluate inputs or runtime attributes" in {
+    val exception = new Exception("Failed to prepare inputs/attributes - part of test flow")
+    val failure = Failure(exception)
+    val expectedResponse = CallPreparationFailed(helper.jobKey, exception)
+    val actor = TestActorRef(helper.buildJobPreparationMock(null, null, null, null, failure), self)
+    actor ! Start
+    expectMsg(expectedResponse)
+  }
+
+  it should "prepare successfully a job without docker attribute" in {
+    val attributes = Map.empty[LocallyQualifiedName, WdlValue]
+    val inputsAndAttributes = Success((inputs, attributes))
+    val dockerHashingActor = TestProbe()
+    val actor = TestActorRef(helper.buildJobPreparationMock(null, null, null, dockerHashingActor.ref, inputsAndAttributes), self)
+    actor ! Start
+    expectMsgPF(5 seconds) {
+      case success: BackendJobPreparationSucceeded =>
+        success.jobDescriptor.callCachingEligibility shouldBe CallCachingEligible
+    }
+    dockerHashingActor.expectNoMsg(1 second)
+  }
+
+  it should "not ask for the docker hash if the attribute already contains a hash" in {
+    val dockerValue = "ubuntu@sha256:71cd81252a3563a03ad8daee81047b62ab5d892ebbfbf71cf53415f29c130950"
+    val attributes = Map (
+      "docker" -> WdlString(dockerValue)
+    )
+    val inputsAndAttributes = Success((inputs, attributes))
+    val dockerHashingActor = TestProbe()
+    val actor = TestActorRef(helper.buildJobPreparationMock(null, null, null, dockerHashingActor.ref, inputsAndAttributes), self)
+    actor ! Start
+    expectMsgPF(5 seconds) {
+      case success: BackendJobPreparationSucceeded =>
+        success.jobDescriptor.runtimeAttributes("docker").valueString shouldBe dockerValue
+        success.jobDescriptor.callCachingEligibility shouldBe CallCachingEligible
+    }
+    dockerHashingActor.expectNoMsg(1 second)
+  }
+
+  it should "replace the tag with a docker hash if the attribute has a floating tag, and make it ineligible for caching" in {
+    val dockerValue = "ubuntu:latest"
+    val attributes = Map (
+      "docker" -> WdlString(dockerValue)
+    )
+    val hashResult = DockerHashResult("sha256", "71cd81252a3563a03ad8daee81047b62ab5d892ebbfbf71cf53415f29c130950")
+    val inputsAndAttributes = Success((inputs, attributes))
+    val finalValue = "library/ubuntu@sha256:71cd81252a3563a03ad8daee81047b62ab5d892ebbfbf71cf53415f29c130950"
+    val dockerHashingActor = TestProbe()
+    val actor = TestActorRef(helper.buildJobPreparationMock(1 minute, 1 minutes, List.empty, dockerHashingActor.ref, inputsAndAttributes), self)
+    actor ! Start
+    dockerHashingActor.expectMsgClass(classOf[DockerHashRequest])
+    dockerHashingActor.reply(DockerHashResponseSuccess(hashResult))
+    expectMsgPF(5 seconds) {
+      case success: BackendJobPreparationSucceeded =>
+        success.jobDescriptor.runtimeAttributes("docker").valueString shouldBe finalValue
+        success.jobDescriptor.callCachingEligibility.isInstanceOf[CallCachingIneligible] shouldBe true
+        success.jobDescriptor.callCachingEligibility.asInstanceOf[CallCachingIneligible].message shouldBe
+          s"""You are using a floating docker tag in this task. Cromwell does not consider tasks with floating tags to be eligible for call caching.
+              |If you want this task to be eligible for call caching in the future, use a docker runtime attribute with a digest instead.
+              |This is the exact docker image that was used for this job: $finalValue
+              |You can replace the docker runtime attribute in your task with the above value to make this task eligible for call caching.""".stripMargin
+    }
+  }
+
+  it should "make a job ineligible for caching if it can't get the docker hash" in {
+    val dockerValue = "ubuntu:latest"
+    val dockerId = DockerImageIdentifier.fromString(dockerValue).get.asInstanceOf[DockerImageIdentifierWithoutHash]
+    val attributes = Map (
+      "docker" -> WdlString(dockerValue)
+    )
+    val inputsAndAttributes = Success((inputs, attributes))
+    val dockerHashingActor = TestProbe()
+    val actor = TestActorRef(helper.buildJobPreparationMock(1 minute, 1 minutes, List.empty, dockerHashingActor.ref, inputsAndAttributes), self)
+    actor ! Start
+    dockerHashingActor.expectMsgClass(classOf[DockerHashRequest])
+    dockerHashingActor.reply(DockerHashFailedResponse(new Exception("Failed to get docker hash - part of test flow"), dockerId))
+    expectMsgPF(5 seconds) {
+      case success: BackendJobPreparationSucceeded =>
+        success.jobDescriptor.runtimeAttributes("docker").valueString shouldBe dockerValue
+        success.jobDescriptor.callCachingEligibility.isInstanceOf[CallCachingIneligible] shouldBe true
+        success.jobDescriptor.callCachingEligibility.asInstanceOf[CallCachingIneligible].message shouldBe
+          s"""You are using a floating docker tag in this task. Cromwell does not consider tasks with floating tags to be eligible for call caching.
+              |If you want this task to be eligible for call caching in the future, use a docker runtime attribute with a digest instead.
+              |Cromwell attempted to retrieve the current hash for this docker image but failed.
+              |This is not necessarily a cause for concern as Cromwell is currently only able to retrieve hashes for Dockerhub and GCR images.
+              |The job will be dispatched to the appropriate backend that will attempt to run it.""".stripMargin
+    }
+  }
+  
+  it should "wait and resubmit the docker request when it gets a backpressure message" in {
+    val dockerValue = "ubuntu:latest"
+    val dockerId = DockerImageIdentifier.fromString(dockerValue).get.asInstanceOf[DockerImageIdentifierWithoutHash]
+    val attributes = Map (
+      "docker" -> WdlString(dockerValue)
+    )
+    val inputsAndAttributes = Success((inputs, attributes))
+    val dockerHashingActor = TestProbe()
+    val backpresureWaitTime = 2 seconds
+    val actor = TestActorRef(helper.buildJobPreparationMock(backpresureWaitTime, 1 minute, List.empty, dockerHashingActor.ref, inputsAndAttributes), self)
+    val request = DockerHashRequest(dockerId)
+    actor ! Start
+    dockerHashingActor.expectMsg(request)
+    dockerHashingActor.reply(DockerHashBackPressure(request))
+    // Give a couple of seconds of margin to account for test latency etc...
+    dockerHashingActor.expectMsg(backpresureWaitTime.+(2 seconds), request)
+  }
+
+  it should "automatically re submit the docker request if no answer and finally give up" in {
+    val dockerValue = "ubuntu:latest"
+    val dockerId = DockerImageIdentifier.fromString(dockerValue).get.asInstanceOf[DockerImageIdentifierWithoutHash]
+    val attributes = Map (
+      "docker" -> WdlString(dockerValue)
+    )
+    val inputsAndAttributes = Success((inputs, attributes))
+    val dockerHashingActor = TestProbe()
+    val requestTimeout = 1 second
+    val actor = TestActorRef(helper.buildJobPreparationMock(1 minute, requestTimeout, List.empty, dockerHashingActor.ref, inputsAndAttributes), self)
+    val request = DockerHashRequest(dockerId)
+    actor ! Start
+    
+    within(10 seconds) {
+      dockerHashingActor.expectMsg(request)
+      dockerHashingActor.expectMsg(request)
+      dockerHashingActor.expectMsg(request)
+      dockerHashingActor.expectMsg(request)
+    }
+
+    expectMsgPF(5 seconds) {
+      case success: BackendJobPreparationSucceeded =>
+        success.jobDescriptor.runtimeAttributes("docker").valueString shouldBe dockerValue
+        success.jobDescriptor.callCachingEligibility.isInstanceOf[CallCachingIneligible] shouldBe true
+        success.jobDescriptor.callCachingEligibility.asInstanceOf[CallCachingIneligible].message shouldBe
+          s"""You are using a floating docker tag in this task. Cromwell does not consider tasks with floating tags to be eligible for call caching.
+              |If you want this task to be eligible for call caching in the future, use a docker runtime attribute with a digest instead.
+              |Cromwell attempted to retrieve the current hash for this docker image but failed.
+              |This is not necessarily a cause for concern as Cromwell is currently only able to retrieve hashes for Dockerhub and GCR images.
+              |The job will be dispatched to the appropriate backend that will attempt to run it.""".stripMargin
+    }
+  }
+  
+}
diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/preparation/JobPreparationTestHelper.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/preparation/JobPreparationTestHelper.scala
new file mode 100644
index 000000000..6c985a2c0
--- /dev/null
+++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/preparation/JobPreparationTestHelper.scala
@@ -0,0 +1,55 @@
+package cromwell.engine.workflow.lifecycle.execution.preparation
+
+import akka.actor.{ActorRef, ActorSystem, Props}
+import akka.testkit.TestProbe
+import cromwell.backend._
+import cromwell.engine.EngineWorkflowDescriptor
+import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActorData
+import org.mockito.Mockito._
+import org.specs2.mock.Mockito
+import wdl4s.Declaration
+import wdl4s.values.WdlValue
+
+import scala.concurrent.duration.FiniteDuration
+import scala.util.Try
+
+class JobPreparationTestHelper(implicit val system: ActorSystem) extends Mockito {
+  val executionData = mock[WorkflowExecutionActorData]
+  val workflowDescriptor = mock[EngineWorkflowDescriptor]
+  val backendDescriptor = mock[BackendWorkflowDescriptor]
+  workflowDescriptor.backendDescriptor returns backendDescriptor
+  executionData.workflowDescriptor returns workflowDescriptor
+  val jobKey = mock[BackendJobDescriptorKey]
+  val serviceRegistryProbe = TestProbe()
+
+  def buildJobPreparationMock(
+                               backpressureTimeout: FiniteDuration,
+                               noResponseTimeout: FiniteDuration,
+                               dockerHashCredentials: List[Any],
+                               dockerHashingActor: ActorRef,
+                               inputsAndAttributes: Try[(Map[Declaration, WdlValue], Map[wdl4s.LocallyQualifiedName, WdlValue])]
+                             ) = {
+    val factory = mock[BackendLifecycleActorFactory]
+    when(factory.dockerHashCredentials(None)).thenReturn(dockerHashCredentials)
+    when(factory.jobExecutionActorProps(
+      any[BackendJobDescriptor],
+      any[Option[BackendInitializationData]],
+      any[ActorRef],
+      any[Option[ActorRef]]
+    )).thenReturn(Props.empty)
+
+    Props(new JobPreparationActor(
+      executionData,
+      jobKey,
+      factory,
+      dockerHashingActor,
+      None,
+      serviceRegistryProbe.ref,
+      None
+    ) {
+      override lazy val dockerNoResponseTimeout = noResponseTimeout
+      override lazy val backpressureWaitTime = backpressureTimeout
+      override def evaluateInputsAndAttributes = inputsAndAttributes
+    })
+  }
+}
diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/auth/GoogleAuthMode.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/auth/GoogleAuthMode.scala
index 26d18833d..67f4c2be7 100644
--- a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/auth/GoogleAuthMode.scala
+++ b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/auth/GoogleAuthMode.scala
@@ -59,6 +59,10 @@ sealed trait GoogleAuthMode {
   def authCredentials(options: WorkflowOptions): AuthCredentials
   // Create a Credential object from the google.api.client.auth library (https://github.com/google/google-api-java-client)
   def credential(options: WorkflowOptions): Credential
+  
+  def credentialBundle(options: WorkflowOptions) = {
+    GoogleCredentialBundle(credential(options), authCredentials(options))
+  }
 
   def requiresAuthFile: Boolean = false
 
diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/auth/GoogleCredentialBundle.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/auth/GoogleCredentialBundle.scala
new file mode 100644
index 000000000..83487e63d
--- /dev/null
+++ b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/auth/GoogleCredentialBundle.scala
@@ -0,0 +1,6 @@
+package cromwell.filesystems.gcs.auth
+
+import com.google.api.client.auth.oauth2.Credential
+import com.google.cloud.AuthCredentials
+
+case class GoogleCredentialBundle(credential: Credential, authCredential: AuthCredentials)
diff --git a/project/Dependencies.scala b/project/Dependencies.scala
index 96646d876..945fcc0d2 100644
--- a/project/Dependencies.scala
+++ b/project/Dependencies.scala
@@ -13,6 +13,7 @@ object Dependencies {
    */
   lazy val sprayJsonV = "1.3.2"
   lazy val akkaV = "2.4.16"
+  lazy val akkaHttpV = "2.4.8"
   lazy val slickV = "3.1.1"
   lazy val googleClientApiV = "1.22.0"
   lazy val googleGenomicsServicesApiV = "1.22.0"
@@ -118,7 +119,10 @@ object Dependencies {
     "com.typesafe.akka" %% "akka-actor" % akkaV,
     "com.typesafe.akka" %% "akka-slf4j" % akkaV,
     "com.typesafe.akka" %% "akka-testkit" % akkaV % Test,
-    "com.google.guava" % "guava" % "20.0"
+    "com.google.guava" % "guava" % "20.0",
+    "com.typesafe.akka" %% "akka-http-core" % akkaHttpV,
+    "com.typesafe.akka" %% "akka-stream-testkit" % akkaHttpV,
+    "com.typesafe.akka" %% "akka-http-spray-json-experimental" % akkaHttpV
   ) ++ baseDependencies ++ googleApiClientDependencies ++
     // TODO: We're not using the "F" in slf4j. Core only supports logback, specifically the WorkflowLogger.
     slf4jBindingDependencies
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendInitializationData.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendInitializationData.scala
index 06eb747b2..de4295a49 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendInitializationData.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendInitializationData.scala
@@ -2,11 +2,13 @@ package cromwell.backend.impl.jes
 
 import com.google.api.services.genomics.Genomics
 import cromwell.backend.standard.{StandardInitializationData, StandardValidatedRuntimeAttributesBuilder}
+import cromwell.filesystems.gcs.auth.GoogleCredentialBundle
 
 case class JesBackendInitializationData
 (
   override val workflowPaths: JesWorkflowPaths,
   override val runtimeAttributesBuilder: StandardValidatedRuntimeAttributesBuilder,
   jesConfiguration: JesConfiguration,
+  gcsCredentials: GoogleCredentialBundle,
   genomics: Genomics
 ) extends StandardInitializationData(workflowPaths, runtimeAttributesBuilder, classOf[JesExpressionFunctions])
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala
index 5bd04e7f9..82d11e271 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala
@@ -8,6 +8,8 @@ import cromwell.backend.standard._
 import cromwell.core.CallOutputs
 import wdl4s.TaskCall
 
+import scala.util.{Success, Try}
+
 case class JesBackendLifecycleActorFactory(name: String, configurationDescriptor: BackendConfigurationDescriptor)
   extends StandardLifecycleActorFactory {
 
@@ -43,4 +45,14 @@ case class JesBackendLifecycleActorFactory(name: String, configurationDescriptor
   override def backendSingletonActorProps = Option(JesBackendSingletonActor.props(jesConfiguration.qps))
 
   override lazy val fileHashingFunction: Option[FileHashingFunction] = Option(FileHashingFunction(JesBackendFileHashing.getCrc32c))
+  
+  override def dockerHashCredentials(initializationData: Option[BackendInitializationData]) = {
+    Try(BackendInitializationData.as[JesBackendInitializationData](initializationData)) match {
+      case Success(jesData) => 
+        val maybeDockerHubCredentials = jesData.jesConfiguration.dockerCredentials
+        val googleCredentials = Option(jesData.gcsCredentials.credential)
+        List(maybeDockerHubCredentials, googleCredentials).flatten
+      case _ => List.empty[Any]
+    }
+  }
 }
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesInitializationActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesInitializationActor.scala
index 3b9ac58e8..3a23a9832 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesInitializationActor.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesInitializationActor.scala
@@ -40,13 +40,16 @@ class JesInitializationActor(jesParams: JesInitializationActorParams)
   }
 
   private lazy val genomics = jesConfiguration.genomicsFactory.withOptions(workflowDescriptor.workflowOptions)
+  // FIXME: workflow paths indirectly re create part of those credentials via the GcsPathBuilder
+  // This is unnecessary duplication of credentials. They are needed here so they can be added to the initialization data
+  // and used to retrieve docker hashes
+  private lazy val gcsCredentials = jesConfiguration.jesAuths.gcs.credentialBundle(workflowDescriptor.workflowOptions)
 
   override lazy val workflowPaths: JesWorkflowPaths =
     new JesWorkflowPaths(workflowDescriptor, jesConfiguration)(context.system)
 
-
   override lazy val initializationData: JesBackendInitializationData =
-    JesBackendInitializationData(workflowPaths, runtimeAttributesBuilder, jesConfiguration, genomics)
+        JesBackendInitializationData(workflowPaths, runtimeAttributesBuilder, jesConfiguration, gcsCredentials, genomics)
 
   override def beforeAll(): Future[Option[BackendInitializationData]] = Future.fromTry(Try {
     if (jesConfiguration.needAuthFileUpload) writeAuthenticationFile(workflowPaths)
diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala
index 592d5b432..eb2082e69 100644
--- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala
+++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala
@@ -15,6 +15,7 @@ import cromwell.backend.impl.jes.statuspolling.JesApiQueryManager.DoPoll
 import cromwell.backend.standard.{DefaultStandardAsyncExecutionActorParams, StandardAsyncExecutionActorParams, StandardAsyncJob, StandardExpressionFunctionsParams}
 import cromwell.backend.wdl.WdlFileMapper
 import cromwell.core._
+import cromwell.core.callcaching.CallCachingEligible
 import cromwell.core.labels.Labels
 import cromwell.core.logging.JobLogger
 import cromwell.core.path.{DefaultPathBuilder, PathBuilder}
@@ -82,7 +83,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
   private def buildInitializationData(jobDescriptor: BackendJobDescriptor, configuration: JesConfiguration) = {
     val workflowPaths = JesWorkflowPaths(jobDescriptor.workflowDescriptor, configuration)(system)
     val runtimeAttributesBuilder = JesRuntimeAttributes.runtimeAttributesBuilder(configuration)
-    JesBackendInitializationData(workflowPaths, runtimeAttributesBuilder, configuration, null)
+    JesBackendInitializationData(workflowPaths, runtimeAttributesBuilder, configuration, null, null)
   }
 
   class TestableJesJobExecutionActor(params: StandardAsyncExecutionActorParams, functions: JesExpressionFunctions)
@@ -139,7 +140,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
     val job = workflowDescriptor.workflow.taskCalls.head
     val key = BackendJobDescriptorKey(job, None, attempt)
     val runtimeAttributes = makeRuntimeAttributes(job)
-    BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, fqnMapToDeclarationMap(Inputs))
+    BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, fqnMapToDeclarationMap(Inputs), CallCachingEligible)
   }
 
   private def executionActor(jobDescriptor: BackendJobDescriptor,
@@ -306,7 +307,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
     val call = workflowDescriptor.workflow.taskCalls.head
     val key = BackendJobDescriptorKey(call, None, 1)
     val runtimeAttributes = makeRuntimeAttributes(call)
-    val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, fqnMapToDeclarationMap(inputs))
+    val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, fqnMapToDeclarationMap(inputs), CallCachingEligible)
 
     val props = Props(new TestableJesJobExecutionActor(jobDescriptor, Promise(), jesConfiguration))
     val testActorRef = TestActorRef[TestableJesJobExecutionActor](
@@ -366,7 +367,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
     val job = workflowDescriptor.workflow.taskCalls.head
     val runtimeAttributes = makeRuntimeAttributes(job)
     val key = BackendJobDescriptorKey(job, None, 1)
-    val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, fqnMapToDeclarationMap(inputs))
+    val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, fqnMapToDeclarationMap(inputs), CallCachingEligible)
 
     val props = Props(new TestableJesJobExecutionActor(jobDescriptor, Promise(), jesConfiguration))
     val testActorRef = TestActorRef[TestableJesJobExecutionActor](
@@ -406,7 +407,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
     val call = workflowDescriptor.workflow.findCallByName(callName).get.asInstanceOf[TaskCall]
     val key = BackendJobDescriptorKey(call, None, 1)
     val runtimeAttributes = makeRuntimeAttributes(call)
-    val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, fqnMapToDeclarationMap(inputs))
+    val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, fqnMapToDeclarationMap(inputs), CallCachingEligible)
 
     val props = Props(new TestableJesJobExecutionActor(jobDescriptor, Promise(), jesConfiguration, functions))
     TestActorRef[TestableJesJobExecutionActor](props, s"TestableJesJobExecutionActor-${jobDescriptor.workflowDescriptor.id}")
@@ -467,7 +468,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
     val job = workflowDescriptor.workflow.taskCalls.head
     val runtimeAttributes = makeRuntimeAttributes(job)
     val key = BackendJobDescriptorKey(job, None, 1)
-    val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, fqnMapToDeclarationMap(inputs))
+    val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, fqnMapToDeclarationMap(inputs), CallCachingEligible)
 
     val props = Props(new TestableJesJobExecutionActor(jobDescriptor, Promise(), jesConfiguration))
     val testActorRef = TestActorRef[TestableJesJobExecutionActor](
@@ -496,7 +497,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
     val job = workflowDescriptor.workflow.taskCalls.head
     val runtimeAttributes = makeRuntimeAttributes(job)
     val key = BackendJobDescriptorKey(job, None, 1)
-    val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, fqnMapToDeclarationMap(inputs))
+    val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, fqnMapToDeclarationMap(inputs), CallCachingEligible)
 
     val props = Props(new TestableJesJobExecutionActor(jobDescriptor, Promise(), jesConfiguration))
     val testActorRef = TestActorRef[TestableJesJobExecutionActor](
@@ -541,7 +542,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
     val call = workflowDescriptor.workflow.taskCalls.head
     val key = BackendJobDescriptorKey(call, None, 1)
     val runtimeAttributes = makeRuntimeAttributes(call)
-    val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, Map.empty)
+    val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, Map.empty, CallCachingEligible)
 
     val props = Props(new TestableJesJobExecutionActor(jobDescriptor, Promise(), jesConfiguration))
     val testActorRef = TestActorRef[TestableJesJobExecutionActor](
@@ -574,7 +575,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
     val job = workflowDescriptor.workflow.taskCalls.head
     val runtimeAttributes = makeRuntimeAttributes(job)
     val key = BackendJobDescriptorKey(job, None, 1)
-    val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, Map.empty)
+    val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, Map.empty, CallCachingEligible)
 
     val props = Props(new TestableJesJobExecutionActor(jobDescriptor, Promise(), jesConfiguration))
     val testActorRef = TestActorRef[TestableJesJobExecutionActor](
@@ -596,7 +597,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
     val job = workflowDescriptor.workflow.taskCalls.head
     val key = BackendJobDescriptorKey(job, None, 1)
     val runtimeAttributes = makeRuntimeAttributes(job)
-    val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, Map.empty)
+    val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, Map.empty, CallCachingEligible)
 
     val props = Props(new TestableJesJobExecutionActor(jobDescriptor, Promise(), jesConfiguration))
     val testActorRef = TestActorRef[TestableJesJobExecutionActor](
@@ -618,7 +619,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
     val call = workflowDescriptor.workflow.findCallByName("hello").get.asInstanceOf[TaskCall]
     val key = BackendJobDescriptorKey(call, None, 1)
     val runtimeAttributes = makeRuntimeAttributes(call)
-    val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, Map.empty)
+    val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, Map.empty, CallCachingEligible)
 
     val props = Props(new TestableJesJobExecutionActor(jobDescriptor, Promise(), jesConfiguration))
     val testActorRef = TestActorRef[TestableJesJobExecutionActor](
@@ -650,7 +651,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
     val call = workflowDescriptor.workflow.findCallByName("B").get.asInstanceOf[TaskCall]
     val key = BackendJobDescriptorKey(call, Option(2), 1)
     val runtimeAttributes = makeRuntimeAttributes(call)
-    val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, Map.empty)
+    val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, Map.empty, CallCachingEligible)
 
     val props = Props(new TestableJesJobExecutionActor(jobDescriptor, Promise(), jesConfiguration))
     val testActorRef = TestActorRef[TestableJesJobExecutionActor](
diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala
index 86deb9649..a0f8d44a2 100644
--- a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala
+++ b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala
@@ -12,6 +12,7 @@ import cromwell.backend.sfs.TestLocalAsyncJobExecutionActor._
 import cromwell.backend.standard.StandardValidatedRuntimeAttributesBuilder
 import cromwell.core.Tags._
 import cromwell.core._
+import cromwell.core.callcaching.CallCachingEligible
 import cromwell.core.path.{DefaultPathBuilder, Path}
 import cromwell.services.keyvalue.KeyValueServiceActor.{KvJobKey, KvPair, ScopedKey}
 import lenthall.exception.AggregatedException
@@ -230,7 +231,7 @@ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSyst
       val runtimeAttributes = RuntimeAttributeDefinition.addDefaultsToAttributes(runtimeAttributeDefinitions, WorkflowOptions.empty)(call.task.runtimeAttributes.attrs)
 
       val jobDescriptor: BackendJobDescriptor =
-        BackendJobDescriptor(workflowDescriptor, BackendJobDescriptorKey(call, Option(shard), 1), runtimeAttributes, fqnMapToDeclarationMap(symbolMaps))
+        BackendJobDescriptor(workflowDescriptor, BackendJobDescriptorKey(call, Option(shard), 1), runtimeAttributes, fqnMapToDeclarationMap(symbolMaps), CallCachingEligible)
       val backend = createBackend(jobDescriptor, emptyBackendConfig)
       val response =
         JobSucceededResponse(mock[BackendJobDescriptorKey], Some(0), Map("out" -> JobOutput(WdlInteger(shard))), None, Seq.empty)

From 7e183cba162730041cb8995cdf381f8de1f8043e Mon Sep 17 00:00:00 2001
From: Khalid Shakir <kshakir@broadinstitute.org>
Date: Fri, 24 Feb 2017 13:20:28 -0500
Subject: [PATCH 73/84] Updated wdl4s to import patch for glob indexing.

---
 project/Dependencies.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/project/Dependencies.scala b/project/Dependencies.scala
index 945fcc0d2..e59df1939 100644
--- a/project/Dependencies.scala
+++ b/project/Dependencies.scala
@@ -2,7 +2,7 @@ import sbt._
 
 object Dependencies {
   lazy val lenthallV = "0.21-e1b7822-SNAP"
-  lazy val wdl4sV = "0.10-e040cf8-SNAP"
+  lazy val wdl4sV = "0.10-785f16a-SNAP"
   lazy val sprayV = "1.3.3"
   /*
   spray-json is an independent project from the "spray suite"

From ba56af3b4ac8e0dff1239fb838e59065bd9bcf92 Mon Sep 17 00:00:00 2001
From: Khalid Shakir <kshakir@broadinstitute.org>
Date: Sat, 25 Feb 2017 22:48:21 -0500
Subject: [PATCH 74/84] Updates for memory validations. Fixed SGE memory
 parameter in reference.conf. Split `MemoryDeclarationValidation` out of
 `DeclarationValidation`. Added tests for `MemoryDeclarationValidation`.
 `OptionalRuntimeAttributesValidation` were not returning the defaults from
 the underlying `RuntimeAttributesValidation`.

---
 .../validation/RuntimeAttributesValidation.scala   |   2 +
 core/src/main/resources/reference.conf             |   2 +-
 .../impl/sfs/config/DeclarationValidation.scala    | 101 +-----------------
 .../sfs/config/MemoryDeclarationValidation.scala   | 115 +++++++++++++++++++++
 .../config/MemoryDeclarationValidationSpec.scala   |  98 ++++++++++++++++++
 5 files changed, 217 insertions(+), 101 deletions(-)
 create mode 100644 supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/MemoryDeclarationValidation.scala
 create mode 100644 supportedBackends/sfs/src/test/scala/cromwell/backend/impl/sfs/config/MemoryDeclarationValidationSpec.scala

diff --git a/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala b/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala
index 322fe0e43..f8ff1441a 100644
--- a/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala
+++ b/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala
@@ -115,6 +115,8 @@ object RuntimeAttributesValidation {
       override protected def missingValueMessage: String = validation.missingValueMessage
 
       override protected def usedInCallCaching: Boolean = validation.usedInCallCachingPackagePrivate
+
+      override protected def staticDefaultOption = validation.staticDefaultOption
     }
   }
 
diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf
index 3aab8d3a4..873380b0b 100644
--- a/core/src/main/resources/reference.conf
+++ b/core/src/main/resources/reference.conf
@@ -246,7 +246,7 @@ backend {
     #    -o ${out} \
     #    -e ${err} \
     #    -pe smp ${cpu} \
-    #    ${"-l m_mem_free=" + memory_gb + "gb"} \
+    #    ${"-l mem_free=" + memory_gb + "g"} \
     #    ${"-q " + sge_queue} \
     #    ${"-P " + sge_project} \
     #    ${script}
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/DeclarationValidation.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/DeclarationValidation.scala
index 904705180..587042d5e 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/DeclarationValidation.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/DeclarationValidation.scala
@@ -1,12 +1,9 @@
 package cromwell.backend.impl.sfs.config
 
-import cromwell.backend.MemorySize
-import cromwell.backend.impl.sfs.config.ConfigConstants._
 import cromwell.backend.validation._
 import wdl4s.expression.NoFunctions
-import wdl4s.parser.MemoryUnit
 import wdl4s.types._
-import wdl4s.values.{WdlFloat, WdlInteger, WdlValue}
+import wdl4s.values.WdlValue
 import wdl4s.{Declaration, NoLookup, WdlExpression}
 
 /**
@@ -107,99 +104,3 @@ class DeclarationValidation(declaration: Declaration, instanceValidation: Runtim
     }
   }
 }
-
-/**
-  * Maps declarations of memory in WDL runtime attributes to the commands used to submit.
-  *
-  * The wdl runtime attributes for memory specified as strings such as:
-  *
-  * {{{
-  *   runtime {
-  *     memory: "500 MB"
-  *   }
-  * }}}
-  *
-  * However, the backend configuration only supports specifying memory in amounts in Float or Int. To specify the unit
-  * of the amount, the string "memory_" is suffixed with the unit, for example "memory_mb" or "memory_gb", or even
-  * "memory_ki".
-  *
-  * This class and companion object will do the conversion. The backend configuration should use the runtime attribute
-  * "Float? memory_gb", meaning that "memory" is now an optional runtime attribute, and will be converted to GB.
-  *
-  * Just like the runtime attribute, when no units are specified the config, the default unit is bytes.
-  *
-  * @param declaration The declaration used to create this memory validation.
-  */
-class MemoryDeclarationValidation(declaration: Declaration)
-  extends DeclarationValidation(declaration, MemoryValidation.instance) {
-
-  import MemoryDeclarationValidation._
-
-  /**
-    * Converts the validation to a version with the default from the memory expression.
-    *
-    * If the backend configuration contains a runtime attribute such as "Float memory_gb = 1.0", then the default will
-    * be set to 1 GB of memory when the attribute is not set.
-    *
-    * @param validation    The validation to set the default for.
-    * @param wdlExpression The declaration expression to retrieve the default.
-    * @return The new validation.
-    */
-  override protected def default(validation: RuntimeAttributesValidation[_],
-                                 wdlExpression: WdlExpression): RuntimeAttributesValidation[_] = {
-    val wdlValue = declaration.expression.get.evaluate(NoLookup, NoFunctions).get
-    val amount: Double = wdlValue match {
-      case WdlInteger(value) => value.toDouble
-      case WdlFloat(value) => value
-      case other => throw new RuntimeException(s"Unsupported memory default: $other")
-    }
-    val memorySize = MemorySize(amount, declarationMemoryUnit)
-    validation.withDefault(WdlInteger(memorySize.bytes.toInt))
-  }
-
-  private lazy val declarationMemoryUnit: MemoryUnit = {
-    val suffix = memoryUnitSuffix(declaration.unqualifiedName)
-    val memoryUnitOption = MemoryUnit.values.find(_.suffixes.map(_.toLowerCase).contains(suffix.toLowerCase))
-    memoryUnitOption match {
-      case Some(memoryUnit) => memoryUnit
-      case None => throw new IllegalArgumentException(s"MemoryUnit with suffix $suffix was not found.")
-    }
-  }
-
-  /**
-    * Converts the memory value from a `MemorySize` to a `Float` or `Int` based on the units.
-    *
-    * @param validatedRuntimeAttributes The validated attributes.
-    * @return The value from the collection wrapped in `Some`, or `None` if the value wasn't found.
-    */
-  override def extractWdlValueOption(validatedRuntimeAttributes: ValidatedRuntimeAttributes): Option[WdlValue] = {
-    RuntimeAttributesValidation.extractOption(MemoryValidation.instance, validatedRuntimeAttributes) map { value =>
-      declaration.wdlType match {
-        case WdlIntegerType => WdlInteger(value.to(declarationMemoryUnit).amount.toInt)
-        case WdlFloatType => WdlFloat(value.to(declarationMemoryUnit).amount)
-        case other => throw new RuntimeException(s"Unsupported wdl type for memory: $other")
-      }
-    }
-  }
-}
-
-object MemoryDeclarationValidation {
-  def isMemoryDeclaration(name: String): Boolean = {
-    name match {
-      case MemoryRuntimeAttribute => true
-      case prefixed if prefixed.startsWith(MemoryRuntimeAttributePrefix) =>
-        val suffix = memoryUnitSuffix(name)
-        MemoryUnit.values exists {
-          _.suffixes.map(_.toLowerCase).contains(suffix)
-        }
-      case _ => false
-    }
-  }
-
-  private def memoryUnitSuffix(name: String) = {
-    if (name == MemoryRuntimeAttribute)
-      MemoryUnit.Bytes.suffixes.head
-    else
-      name.substring(MemoryRuntimeAttributePrefix.length)
-  }
-}
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/MemoryDeclarationValidation.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/MemoryDeclarationValidation.scala
new file mode 100644
index 000000000..39fd88046
--- /dev/null
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/MemoryDeclarationValidation.scala
@@ -0,0 +1,115 @@
+package cromwell.backend.impl.sfs.config
+
+import cromwell.backend.MemorySize
+import cromwell.backend.impl.sfs.config.ConfigConstants._
+import cromwell.backend.validation._
+import wdl4s.expression.NoFunctions
+import wdl4s.parser.MemoryUnit
+import wdl4s.types._
+import wdl4s.values.{WdlFloat, WdlInteger, WdlOptionalValue, WdlValue}
+import wdl4s.{Declaration, NoLookup, WdlExpression}
+
+/**
+  * Maps declarations of memory in WDL runtime attributes to the commands used to submit.
+  *
+  * The wdl runtime attributes for memory specified as strings such as:
+  *
+  * {{{
+  *   runtime {
+  *     memory: "500 MB"
+  *   }
+  * }}}
+  *
+  * However, the backend configuration only supports specifying memory in amounts in Float or Int. To specify the unit
+  * of the amount, the string "memory_" is suffixed with the unit, for example "memory_mb" or "memory_gb", or even
+  * "memory_ki".
+  *
+  * This class and companion object will do the conversion. The backend configuration should use the runtime attribute
+  * "Float? memory_gb", meaning that "memory" is now an optional runtime attribute, and will be converted to GB.
+  *
+  * Just like the runtime attribute, when no units are specified the config, the default unit is bytes.
+  *
+  * @param declaration The declaration used to create this memory validation.
+  */
+class MemoryDeclarationValidation(declaration: Declaration)
+  extends DeclarationValidation(declaration, MemoryValidation.instance) {
+
+  import MemoryDeclarationValidation._
+
+  /**
+    * Converts the validation to a version with the default from the memory expression.
+    *
+    * If the backend configuration contains a runtime attribute such as "Float memory_gb = 1.0", then the default will
+    * be set to 1 GB of memory when the attribute is not set.
+    *
+    * @param validation    The validation to set the default for.
+    * @param wdlExpression The declaration expression to retrieve the default.
+    * @return The new validation.
+    */
+  override protected def default(validation: RuntimeAttributesValidation[_],
+                                 wdlExpression: WdlExpression): RuntimeAttributesValidation[_] = {
+    val wdlValue = declaration.expression.get.evaluate(NoLookup, NoFunctions).get
+    val amount: Double = defaultAmount(wdlValue)
+    val memorySize = MemorySize(amount, declarationMemoryUnit)
+    validation.withDefault(WdlInteger(memorySize.bytes.toInt))
+  }
+
+  private def defaultAmount(wdlValue: WdlValue): Double = {
+    wdlValue match {
+      case WdlInteger(value) => value.toDouble
+      case WdlFloat(value) => value
+      case WdlOptionalValue(_, Some(optionalWdlValue)) => defaultAmount(optionalWdlValue)
+      case other => throw new RuntimeException(s"Unsupported memory default: $other")
+    }
+  }
+
+  private lazy val declarationMemoryUnit: MemoryUnit = {
+    val suffix = memoryUnitSuffix(declaration.unqualifiedName)
+    val memoryUnitOption = MemoryUnit.values.find(_.suffixes.map(_.toLowerCase).contains(suffix.toLowerCase))
+    memoryUnitOption match {
+      case Some(memoryUnit) => memoryUnit
+      case None => throw new IllegalArgumentException(s"MemoryUnit with suffix $suffix was not found.")
+    }
+  }
+
+  /**
+    * Converts the memory value from a `MemorySize` to a `Float` or `Int` based on the units.
+    *
+    * @param validatedRuntimeAttributes The validated attributes.
+    * @return The value from the collection wrapped in `Some`, or `None` if the value wasn't found.
+    */
+  override def extractWdlValueOption(validatedRuntimeAttributes: ValidatedRuntimeAttributes): Option[WdlValue] = {
+    RuntimeAttributesValidation.extractOption(MemoryValidation.instance, validatedRuntimeAttributes) map
+      coerceMemorySize(declaration.wdlType)
+  }
+
+  private def coerceMemorySize(wdlType: WdlType)(value: MemorySize): WdlValue = {
+    wdlType match {
+      case WdlIntegerType => WdlInteger(value.to(declarationMemoryUnit).amount.toInt)
+      case WdlFloatType => WdlFloat(value.to(declarationMemoryUnit).amount)
+      case WdlOptionalType(optionalType) => coerceMemorySize(optionalType)(value)
+      case other => throw new RuntimeException(s"Unsupported wdl type for memory: $other")
+    }
+  }
+}
+
+object MemoryDeclarationValidation {
+  def isMemoryDeclaration(name: String): Boolean = {
+    name match {
+      case MemoryRuntimeAttribute => true
+      case prefixed if prefixed.startsWith(MemoryRuntimeAttributePrefix) =>
+        val suffix = memoryUnitSuffix(name)
+        MemoryUnit.values exists {
+          _.suffixes.map(_.toLowerCase).contains(suffix)
+        }
+      case _ => false
+    }
+  }
+
+  private def memoryUnitSuffix(name: String) = {
+    if (name == MemoryRuntimeAttribute)
+      MemoryUnit.Bytes.suffixes.head
+    else
+      name.substring(MemoryRuntimeAttributePrefix.length)
+  }
+}
diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/impl/sfs/config/MemoryDeclarationValidationSpec.scala b/supportedBackends/sfs/src/test/scala/cromwell/backend/impl/sfs/config/MemoryDeclarationValidationSpec.scala
new file mode 100644
index 000000000..3cc444c53
--- /dev/null
+++ b/supportedBackends/sfs/src/test/scala/cromwell/backend/impl/sfs/config/MemoryDeclarationValidationSpec.scala
@@ -0,0 +1,98 @@
+package cromwell.backend.impl.sfs.config
+
+import com.typesafe.config.ConfigFactory
+import cromwell.backend.MemorySize
+import cromwell.backend.validation.{RuntimeAttributesKeys, ValidatedRuntimeAttributes}
+import org.scalatest.prop.TableDrivenPropertyChecks
+import org.scalatest.{FlatSpec, Matchers}
+import wdl4s.parser.MemoryUnit
+import wdl4s.values.{WdlFloat, WdlInteger}
+
+class MemoryDeclarationValidationSpec extends FlatSpec with Matchers with TableDrivenPropertyChecks {
+  behavior of "MemoryDeclarationValidation"
+
+  val validDeclaredAmounts = Table(
+    ("declaration", "runtimeAmount", "expectedDefaultAmount", "expectedExtracted"),
+    ("Int memory", Option(2), None, Option(WdlInteger(2 * 1000 * 1000 * 1000))),
+    ("Int memory_gb", Option(2), None, Option(WdlInteger(2))),
+    ("Int memory_gb = 3", None, Option(3), None),
+    ("Int memory_gb = 3", Option(2), Option(3), Option(WdlInteger(2))),
+    ("Int? memory_gb", None, None, None),
+    ("Int? memory_gb", Option(2), None, Option(WdlInteger(2))),
+    ("Int? memory_gb = 3", None, Option(3), None),
+    ("Int? memory_gb = 3", Option(2), Option(3), Option(WdlInteger(2))),
+    ("Float memory", Option(2), None, Option(WdlFloat(2 * 1000 * 1000 * 1000))),
+    ("Float memory_gb", Option(2), None, Option(WdlFloat(2))),
+    ("Float memory_gb = 3.0", None, Option(3), None),
+    ("Float memory_gb = 3.0", Option(2), Option(3), Option(WdlFloat(2))),
+    ("Float? memory_gb", None, None, None),
+    ("Float? memory_gb", Option(2), None, Option(WdlFloat(2))),
+    ("Float? memory_gb = 3.0", None, Option(3), None),
+    ("Float? memory_gb = 3.0", Option(2), Option(3), Option(WdlFloat(2)))
+  )
+
+  forAll(validDeclaredAmounts) { (declaration, runtimeAmount, expectedDefaultAmount, expectedExtracted) =>
+    it should s"extract memory from declared $declaration with memory set to ${runtimeAmount.getOrElse("none")}" in {
+      val config = ConfigFactory.parseString(
+        s"""|submit = "anything"
+            |${ConfigConstants.RuntimeAttributesConfig} = "$declaration"
+            |""".stripMargin)
+
+      val configWdlNamespace = new ConfigWdlNamespace(config)
+      val runtimeDeclaration = configWdlNamespace.runtimeDeclarations.head
+      val memoryDeclarationValidation = new MemoryDeclarationValidation(runtimeDeclaration)
+      val attributes = runtimeAmount
+        .map(amount => RuntimeAttributesKeys.MemoryKey -> MemorySize(amount.toDouble, MemoryUnit.GB))
+        .toMap
+      val validatedRuntimeAttributes = ValidatedRuntimeAttributes(attributes)
+
+      val default = memoryDeclarationValidation.makeValidation().runtimeAttributeDefinition.factoryDefault
+      val extracted = memoryDeclarationValidation.extractWdlValueOption(validatedRuntimeAttributes)
+
+      val expectedDefault = expectedDefaultAmount
+        .map(amount => WdlInteger(MemorySize(amount.toDouble, MemoryUnit.GB).bytes.toInt))
+
+      MemoryDeclarationValidation.isMemoryDeclaration(runtimeDeclaration.unqualifiedName) should be(true)
+      default should be(expectedDefault)
+      extracted should be(expectedExtracted)
+    }
+  }
+
+  val badSyntaxDeclarations = Table(
+    "declaration",
+    "Int memory_gb = 3.0",
+    "Float memory_gb = 3"
+  )
+
+  forAll(badSyntaxDeclarations) { declaration =>
+    it should s"throw a syntax error for memory declaration $declaration" in {
+      val config = ConfigFactory.parseString(
+        s"""|submit = "anything"
+            |${ConfigConstants.RuntimeAttributesConfig} = "$declaration"
+            |""".stripMargin)
+
+      val expectedException = intercept[RuntimeException](new ConfigWdlNamespace(config))
+      expectedException.getMessage should startWith("Error parsing generated wdl:\n")
+    }
+  }
+
+  val invalidDeclarations = Table(
+    "declaration",
+    "Int mem",
+    "Int memory_badunit",
+    "Float memory_badunit"
+  )
+
+  forAll(invalidDeclarations) { declaration =>
+    it should s"not identify $declaration as a memory declaration" in {
+      val config = ConfigFactory.parseString(
+        s"""|submit = "anything"
+            |${ConfigConstants.RuntimeAttributesConfig} = "$declaration"
+            |""".stripMargin)
+
+      val configWdlNamespace = new ConfigWdlNamespace(config)
+      val runtimeDeclaration = configWdlNamespace.runtimeDeclarations.head
+      MemoryDeclarationValidation.isMemoryDeclaration(runtimeDeclaration.unqualifiedName) should be(false)
+    }
+  }
+}

From e397e2542586ab43f4108caa085d95e3dfbde65f Mon Sep 17 00:00:00 2001
From: Khalid Shakir <kshakir@broadinstitute.org>
Date: Sun, 26 Feb 2017 13:24:02 -0500
Subject: [PATCH 75/84] Slightly better abort behavior in standard backend.
 Stopped closing a scala `Future` over akka's `context.become`. Flipped the
 default for `requestsAbortAndDiesImmediately` from `false` to `true`. When
 killing a Standard backend job with rADI false, both the rc and the standard
 error are required. Writing the stderr on abort for the SFS backend. When
 rADI is true, sending a backend status of `"Aborted"`. In the engine, set and
 store the `ExecutionStatus.Aborted`.

---
 .../standard/StandardAsyncExecutionActor.scala       | 20 ++++++++++++--------
 .../lifecycle/execution/CallMetadataHelper.scala     |  6 ++++++
 .../lifecycle/execution/WorkflowExecutionActor.scala | 11 +++++++++--
 .../impl/jes/JesAsyncBackendJobExecutionActor.scala  |  2 --
 .../sfs/SharedFileSystemAsyncJobExecutionActor.scala | 11 +++++++++++
 .../impl/tes/TesAsyncBackendJobExecutionActor.scala  |  2 --
 6 files changed, 38 insertions(+), 14 deletions(-)

diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala b/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala
index fa9ffffd3..6fa5d58f9 100644
--- a/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala
+++ b/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala
@@ -334,7 +334,7 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
   /**
     * Attempts to abort a job when an abort signal is retrieved.
     *
-    * If `abortAndDieImmediately` is true, then the actor will die immediately after this method returns.
+    * If `requestsAbortAndDiesImmediately` is true, then the actor will die immediately after this method returns.
     *
     * @param jobId The job to abort.
     */
@@ -344,11 +344,14 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
     * Returns true if when an abort signal is retrieved, the actor makes an attempt to abort and then immediately stops
     * itself _without_ polling for an aborted status.
     *
-    * The default is false.
+    * NOTE: When this value is set to `false`, `tryAbort` must write the rc and stderr files, as they will still be
+    * processed during a poll that returns a terminal status.
+    *
+    * The default is true.
     *
     * @return true if actor should request an abort and then die immediately.
     */
-  def requestsAbortAndDiesImmediately: Boolean = false
+  def requestsAbortAndDiesImmediately: Boolean = true
 
   /**
     * Return true if the return code is an abort code.
@@ -363,10 +366,11 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
   /**
     * Custom behavior to run after an abort signal is processed.
     *
-    * By default handles the behavior of `abortAndDieImmediately`.
+    * By default handles the behavior of `requestsAbortAndDiesImmediately`.
     */
   def postAbort(): Unit = {
     if (requestsAbortAndDiesImmediately) {
+      tellMetadata(Map(CallMetadataKeys.BackendStatus -> "Aborted"))
       context.parent ! AbortedResponse(jobDescriptor.key)
       context.stop(self)
     }
@@ -475,7 +479,7 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
 
   // See executeOrRecoverSuccess
   private var missedAbort = false
-  private case object CheckMissedAbort
+  private case class CheckMissedAbort(jobId: StandardAsyncJob)
 
   context.become(standardReceiveBehavior(None) orElse receive)
 
@@ -490,7 +494,8 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
         case None => missedAbort = true
       }
       postAbort()
-    case CheckMissedAbort =>
+    case CheckMissedAbort(jobId: StandardAsyncJob) =>
+      context.become(standardReceiveBehavior(Option(jobId)) orElse receive)
       if (missedAbort)
         self ! AbortJobCommand
     case KvPutSuccess(_) => // expected after the KvPut for the operation ID
@@ -529,8 +534,7 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
         queued up and may still be run by the thread pool anytime in the future. Issue #1218 may address this
         inconsistency at a later time. For now, just go back and check if we missed the abort command.
         */
-        context.become(standardReceiveBehavior(Option(handle.pendingJob)) orElse receive)
-        self ! CheckMissedAbort
+        self ! CheckMissedAbort(handle.pendingJob)
       case _ =>
     }
     executionHandle
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/CallMetadataHelper.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/CallMetadataHelper.scala
index 67db2d926..bdc6eaf7c 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/CallMetadataHelper.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/CallMetadataHelper.scala
@@ -90,6 +90,12 @@ trait CallMetadataHelper {
     serviceRegistryActor ! PutMetadataAction(completionEvents ++ failureEvents)
   }
 
+  def pushAbortedCallMetadata(jobKey: JobKey) = {
+    val completionEvents = completedCallMetadataEvents(jobKey, ExecutionStatus.Aborted, None)
+
+    serviceRegistryActor ! PutMetadataAction(completionEvents)
+  }
+
   def pushExecutionEventsToMetadataService(jobKey: JobKey, eventList: Seq[ExecutionEvent]) = {
     def metadataEvent(k: String, value: Any) = {
       val metadataValue = MetadataValue(value)
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala
index a3829f347..aef406278 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala
@@ -134,12 +134,16 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor,
 
   when(WorkflowExecutionAbortingState) {
     case Event(AbortedResponse(jobKey), stateData) =>
+      pushAbortedCallMetadata(jobKey)
       handleCallAborted(stateData, jobKey, Map.empty)
     case Event(SubWorkflowAbortedResponse(jobKey, executedKeys), stateData) =>
+      pushAbortedCallMetadata(jobKey)
       handleCallAborted(stateData, jobKey, executedKeys)
     case Event(SubWorkflowSucceededResponse(subKey, executedKeys, _), stateData) =>
+      pushAbortedCallMetadata(subKey)
       handleCallAborted(stateData, subKey, executedKeys)
     case Event(JobSucceededResponse(jobKey, returnCode, callOutputs, _, _), stateData) =>
+      pushAbortedCallMetadata(jobKey)
       handleCallAborted(stateData, jobKey, Map.empty)
   }
   
@@ -195,7 +199,7 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor,
       if (stateData.hasRunningActors) {
         log.info(s"$tag: Abort received. " +
           s"Aborting ${stateData.backendJobExecutionActors.size} Job Execution Actors" +
-          s"and ${stateData.subWorkflowExecutionActors.size} Sub Workflow Execution Actors"
+          s" and ${stateData.subWorkflowExecutionActors.size} Sub Workflow Execution Actors"
         )
         stateData.backendJobExecutionActors.values foreach { _ ! AbortJobCommand }
         stateData.subWorkflowExecutionActors.values foreach { _ ! EngineLifecycleActorAbortCommand }
@@ -334,7 +338,10 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor,
   
   private def handleCallAborted(data: WorkflowExecutionActorData, jobKey: JobKey, jobExecutionMap: JobExecutionMap) = {
     workflowLogger.info(s"$tag job aborted: ${jobKey.tag}")
-    val newStateData = data.removeCallExecutionActor(jobKey).addExecutions(jobExecutionMap)
+    val newStateData = data
+      .mergeExecutionDiff(WorkflowExecutionDiff(Map(jobKey -> ExecutionStatus.Aborted)))
+      .removeCallExecutionActor(jobKey)
+      .addExecutions(jobExecutionMap)
     if (!newStateData.hasRunningActors) {
       workflowLogger.info(s"$tag all jobs aborted")
       goto(WorkflowExecutionAbortedState)
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala
index dbdccd846..bc958475d 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala
@@ -74,8 +74,6 @@ class JesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn
     Run(job.jobId, initializationData.genomics).abort()
   }
 
-  override def requestsAbortAndDiesImmediately: Boolean = true
-
   override def receive: Receive = pollingActorClientReceive orElse super.receive
 
   private def gcsAuthParameter: Option[JesInput] = {
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala
index 01aa20d9c..731ad6efa 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala
@@ -172,6 +172,8 @@ trait SharedFileSystemAsyncJobExecutionActor
     checkAlive.run() == 0
   }
 
+  override def requestsAbortAndDiesImmediately: Boolean = false
+
   override def tryAbort(job: StandardAsyncJob): Unit = {
     val returnCodeTmp = jobPaths.returnCode.plusExt("kill")
     returnCodeTmp.write(s"$SIGTERM\n")
@@ -182,6 +184,15 @@ trait SharedFileSystemAsyncJobExecutionActor
         // If the process has already completed, there will be an existing rc file.
         returnCodeTmp.delete(true)
     }
+    val stderrTmp = jobPaths.stderr.plusExt("kill")
+    stderrTmp.touch()
+    try {
+      stderrTmp.moveTo(jobPaths.stderr)
+    } catch {
+      case _: FileAlreadyExistsException =>
+        // If the process has already started, there will be an existing stderr file.
+        stderrTmp.delete(true)
+    }
     val argv = killArgs(job).argv
     val stdout = jobPaths.stdout.plusExt("kill")
     val stderr = jobPaths.stderr.plusExt("kill")
diff --git a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesAsyncBackendJobExecutionActor.scala b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesAsyncBackendJobExecutionActor.scala
index a0a9c5230..67c45f175 100644
--- a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesAsyncBackendJobExecutionActor.scala
+++ b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesAsyncBackendJobExecutionActor.scala
@@ -143,8 +143,6 @@ class TesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn
     ()
   }
 
-  override def requestsAbortAndDiesImmediately: Boolean = true
-
   override def pollStatusAsync(handle: StandardAsyncPendingExecutionHandle)(implicit ec: ExecutionContext): Future[TesRunStatus] = {
     val pollTask = pipeline[TesGetResponse].apply(Get(s"$tesEndpoint/${handle.pendingJob.jobId}"))
 

From 2e79a2598404e2520aa85941014128ea5a2b55f1 Mon Sep 17 00:00:00 2001
From: Miguel Covarrubias <mcovarr@broadinstitute.org>
Date: Fri, 24 Feb 2017 11:21:39 -0500
Subject: [PATCH 76/84] add a knob

---
 core/src/main/resources/reference.conf                        | 3 +++
 engine/src/main/scala/cromwell/server/CromwellRootActor.scala | 5 ++++-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf
index 3aab8d3a4..b20beed31 100644
--- a/core/src/main/resources/reference.conf
+++ b/core/src/main/resources/reference.conf
@@ -82,6 +82,9 @@ system {
 
   # Since the WorkflowLogCopyRouter is initialized in code, this is the number of workers
   number-of-workflow-log-copy-workers = 10
+
+  # Default number of cache read workers
+  number-of-cache-read-workers = 25
 }
 
 workflow-options {
diff --git a/engine/src/main/scala/cromwell/server/CromwellRootActor.scala b/engine/src/main/scala/cromwell/server/CromwellRootActor.scala
index e9e8878f7..0a6b708b2 100644
--- a/engine/src/main/scala/cromwell/server/CromwellRootActor.scala
+++ b/engine/src/main/scala/cromwell/server/CromwellRootActor.scala
@@ -65,7 +65,9 @@ import scala.language.postfixOps
   lazy val subWorkflowStoreActor = context.actorOf(SubWorkflowStoreActor.props(subWorkflowStore), "SubWorkflowStoreActor")
 
   lazy val callCache: CallCache = new CallCache(SingletonServicesStore.databaseInterface)
-  lazy val callCacheReadActor = context.actorOf(RoundRobinPool(25)
+
+  lazy val numberOfCacheReadWorkers = config.getConfig("system").as[Option[Int]]("number-of-cache-read-workers").getOrElse(DefaultNumberOfCacheReadWorkers)
+  lazy val callCacheReadActor = context.actorOf(RoundRobinPool(numberOfCacheReadWorkers)
     .props(CallCacheReadActor.props(callCache)),
     "CallCacheReadActor")
   
@@ -116,4 +118,5 @@ import scala.language.postfixOps
 object CromwellRootActor {
   val DefaultNumberOfWorkflowLogCopyWorkers = 10
   val DefaultCacheTTL = 20 minutes
+  val DefaultNumberOfCacheReadWorkers = 25
 }

From 9f6180f161fc3de2005025d62c0b8e135f7c1603 Mon Sep 17 00:00:00 2001
From: delocalizer <conrad.leonard@hotmail.com>
Date: Tue, 28 Feb 2017 10:38:59 +1000
Subject: [PATCH 77/84] * make sure dir that TMPDIR is set to, exists

---
 .../scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala    | 1 +
 1 file changed, 1 insertion(+)

diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala b/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala
index 6fa5d58f9..261bda4a9 100644
--- a/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala
+++ b/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala
@@ -179,6 +179,7 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
     s"""|#!/bin/bash
         |export _JAVA_OPTIONS=-Djava.io.tmpdir=$tmpDir
         |export TMPDIR=$tmpDir
+        |mkdir -p $tmpDir
         |$commandScriptPreamble
         |(
         |cd $cwd

From a89ce0aabf0e76074caa82819cd045ce0e1824bf Mon Sep 17 00:00:00 2001
From: Khalid Shakir <kshakir@broadinstitute.org>
Date: Thu, 2 Mar 2017 13:31:50 -0500
Subject: [PATCH 78/84] Message, not throw, failures in
 WEA.startRunnableScopes. Other syntax fixes.

---
 .../execution/WorkflowExecutionActor.scala         | 119 +++++++++++++--------
 1 file changed, 75 insertions(+), 44 deletions(-)

diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala
index aef406278..9135696cf 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala
@@ -13,7 +13,7 @@ import cromwell.core.WorkflowOptions.WorkflowFailureMode
 import cromwell.core._
 import cromwell.core.logging.WorkflowLogging
 import cromwell.engine.backend.{BackendSingletonCollection, CromwellBackends}
-import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor.{apply => _, _}
+import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor._
 import cromwell.engine.workflow.lifecycle.{EngineLifecycleActorAbortCommand, EngineLifecycleActorAbortedResponse}
 import cromwell.engine.{ContinueWhilePossible, EngineWorkflowDescriptor}
 import cromwell.util.StopAndLogSupervisor
@@ -121,7 +121,7 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor,
       pushFailedCallMetadata(jobKey, returnCode, reason, retryableFailure = false)
       handleNonRetryableFailure(stateData, jobKey, reason, Map.empty)
       // Job Retryable
-    case Event(JobFailedRetryableResponse(jobKey, reason, returnCode), stateData) =>
+    case Event(JobFailedRetryableResponse(jobKey, reason, returnCode), _) =>
       pushFailedCallMetadata(jobKey, None, reason, retryableFailure = true)
       handleRetryableFailure(jobKey, reason, returnCode)
       // Sub Workflow - sub workflow failures are always non retryable
@@ -142,7 +142,7 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor,
     case Event(SubWorkflowSucceededResponse(subKey, executedKeys, _), stateData) =>
       pushAbortedCallMetadata(subKey)
       handleCallAborted(stateData, subKey, executedKeys)
-    case Event(JobSucceededResponse(jobKey, returnCode, callOutputs, _, _), stateData) =>
+    case Event(JobSucceededResponse(jobKey, _, _, _, _), stateData) =>
       pushAbortedCallMetadata(jobKey)
       handleCallAborted(stateData, jobKey, Map.empty)
   }
@@ -161,16 +161,16 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor,
     * Mop up function to handle a set of incoming results if this workflow has already failed:
     */
   private def alreadyFailedMopUp: StateFunction = {
-    case Event(JobInitializationFailed(jobKey, reason), stateData) =>
+    case Event(JobInitializationFailed(jobKey, reason), _) =>
       pushFailedCallMetadata(jobKey, None, reason, retryableFailure = false)
       stay
-    case Event(JobFailedNonRetryableResponse(jobKey, reason, returnCode), stateData) =>
+    case Event(JobFailedNonRetryableResponse(jobKey, reason, returnCode), _) =>
       pushFailedCallMetadata(jobKey, returnCode, reason, retryableFailure = false)
       stay
-    case Event(JobFailedRetryableResponse(jobKey, reason, returnCode), stateData) =>
+    case Event(JobFailedRetryableResponse(jobKey, reason, returnCode), _) =>
       pushFailedCallMetadata(jobKey, returnCode, reason, retryableFailure = true)
       stay
-    case Event(JobSucceededResponse(jobKey, returnCode, callOutputs, _, _), stateData) =>
+    case Event(JobSucceededResponse(jobKey, returnCode, callOutputs, _, _), _) =>
       pushSuccessfulCallMetadata(jobKey, returnCode, callOutputs)
       stay
   }
@@ -263,7 +263,10 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor,
     import cromwell.util.JsonFormatting.WdlValueJsonFormatter._
     import spray.json._
 
-     val (response, finalState) = workflowDescriptor.workflow.evaluateOutputs(
+    case class ResponseAndFinalState(response: WorkflowExecutionActorResponse,
+                                     finalState: WorkflowExecutionActorTerminalState)
+
+    val responseAndState = workflowDescriptor.workflow.evaluateOutputs(
       workflowDescriptor.knownValues,
       data.expressionLanguageFunctions,
       data.outputStore.fetchNodeOutputEntries
@@ -277,20 +280,22 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor,
              |${workflowScopeOutputs.stripLarge.toJson.prettyPrint}""".stripMargin
        )
        pushWorkflowOutputMetadata(workflowScopeOutputs)
-       
+
        // For cromwell internal storage of outputs
        val unqualifiedWorkflowOutputs = workflowOutputs map {
          // JobOutput is poorly named here - a WorkflowOutput type would be better
          case (output, value) => output.unqualifiedName -> JobOutput(value)
        }
-       (WorkflowExecutionSucceededResponse(data.jobExecutionMap, unqualifiedWorkflowOutputs), WorkflowExecutionSuccessfulState)
+      ResponseAndFinalState(
+        WorkflowExecutionSucceededResponse(data.jobExecutionMap, unqualifiedWorkflowOutputs),
+        WorkflowExecutionSuccessfulState)
     } recover {
        case ex =>
-         (WorkflowExecutionFailedResponse(data.jobExecutionMap, ex), WorkflowExecutionFailedState)
+         ResponseAndFinalState(WorkflowExecutionFailedResponse(data.jobExecutionMap, ex), WorkflowExecutionFailedState)
     } get
     
-    context.parent ! response
-    goto(finalState) using data
+    context.parent ! responseAndState.response
+    goto(responseAndState.finalState) using data
   }
 
   private def handleRetryableFailure(jobKey: BackendJobDescriptorKey, reason: Throwable, returnCode: Option[Int]) = {
@@ -328,7 +333,7 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor,
     data.workflowCompletionStatus match {
       case Some(ExecutionStatus.Done) =>
         handleWorkflowSuccessful(data)
-      case Some(sts) =>
+      case Some(_) =>
         context.parent ! WorkflowExecutionFailedResponse(data.jobExecutionMap, new Exception("One or more jobs failed in fail-slow mode"))
         goto(WorkflowExecutionFailedState) using data
       case _ =>
@@ -362,34 +367,58 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor,
     if (runnableCalls.nonEmpty) workflowLogger.info("Starting calls: " + runnableCalls.mkString(", "))
 
     // Each process returns a Try[WorkflowExecutionDiff], which, upon success, contains potential changes to be made to the execution store.
-    val executionDiffs = runnableScopes map {
-      case k: CallKey if isInBypassedScope(k, data) => processBypassedScope(k, data)
-      case k: DeclarationKey if isInBypassedScope(k, data) => processBypassedScope(k, data)
-      case k: BackendJobDescriptorKey => processRunnableJob(k, data)
-      case k: ScatterKey => processRunnableScatter(k, data, isInBypassedScope(k, data))
-      case k: ConditionalKey => processRunnableConditional(k, data)
-      case k: CollectorKey => processRunnableCollector(k, data, isInBypassedScope(k, data))
-      case k: SubWorkflowKey => processRunnableSubWorkflow(k, data)
-      case k: StaticDeclarationKey => processRunnableStaticDeclaration(k)
-      case k: DynamicDeclarationKey => processRunnableDynamicDeclaration(k, data)
-      case k =>
-        val exception = new UnsupportedOperationException(s"Unknown entry in execution store: ${k.tag}")
-        self ! JobInitializationFailed(k, exception)
-        Failure(exception)
+    val diffs = runnableScopes map { scope =>
+      scope -> Try(scope match {
+        case k: CallKey if isInBypassedScope(k, data) => processBypassedScope(k, data)
+        case k: DeclarationKey if isInBypassedScope(k, data) => processBypassedScope(k, data)
+        case k: BackendJobDescriptorKey => processRunnableJob(k, data)
+        case k: ScatterKey => processRunnableScatter(k, data, isInBypassedScope(k, data))
+        case k: ConditionalKey => processRunnableConditional(k, data)
+        case k: CollectorKey => processRunnableCollector(k, data, isInBypassedScope(k, data))
+        case k: SubWorkflowKey => processRunnableSubWorkflow(k, data)
+        case k: StaticDeclarationKey => processRunnableStaticDeclaration(k)
+        case k: DynamicDeclarationKey => processRunnableDynamicDeclaration(k, data)
+        case k => Failure(new UnsupportedOperationException(s"Unknown entry in execution store: ${k.tag}"))
+      }).flatten
+    } map {
+      case (_, Success(value)) => Success(value)
+      case (scope, Failure(throwable)) =>
+        self ! JobInitializationFailed(scope, throwable)
+        Failure(throwable)
+    } collect {
+      /*
+      NOTE: This is filtering out all errors and only returning the successes, but only after the map above sent a
+      message that something is wrong.
+
+      We used to throw an aggregation exception of all the collected errors, but _nothing_ in cromwell is actually
+      expecting that. Thus the workflows were being left in a Running state, jobs were left dispatched, etc.
+
+      Meanwhile this actor and its children died or restarted, and one couldn't even attempt to abort the other jobs.
+
+      Now, in the previous map, we send a message to ourselves about _every_ failure. But this method does not attempt
+      to further process the errors. The failures enqueue in the actor mailbox, and are handled by this actor's receive.
+
+      At the moment, there is an issue in how this actor handles failure messages. That issue is tracked in:
+      https://github.com/broadinstitute/cromwell/issues/2029
+
+      Separately, we may also want to institute better supervision of actors, in general. But just throwing an exception
+      here doesn't actually force the correct handling.
+
+      See also:
+      https://github.com/broadinstitute/cromwell/issues/1414
+      https://github.com/broadinstitute/cromwell/issues/1874
+      */
+      case Success(value) => value
     }
 
-    TryUtil.sequence(executionDiffs) match {
-      case Success(diffs) =>
-        // Update the metadata for the jobs we just sent to EJEAs (they'll start off queued up waiting for tokens):
-        pushQueuedCallMetadata(diffs)
-        if (diffs.exists(_.containsNewEntry)) {
-          val newData = data.mergeExecutionDiffs(diffs)
-          startRunnableScopes(newData)
-        } else {
-          val result = data.mergeExecutionDiffs(diffs)
-          result
-        }
-      case Failure(e) => throw new RuntimeException("Unexpected engine failure", e)
+    // Update the metadata for the jobs we just sent to EJEAs (they'll start off queued up waiting for tokens):
+    pushQueuedCallMetadata(diffs)
+    if (diffs.exists(_.containsNewEntry)) {
+      val newData = data.mergeExecutionDiffs(diffs)
+      startRunnableScopes(newData)
+    } else {
+      val result = data.mergeExecutionDiffs(diffs)
+      result
     }
   }
 
@@ -500,7 +529,8 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor,
         val conditionalStatus = if (b.value) ExecutionStatus.Done else ExecutionStatus.Bypassed
         val result = WorkflowExecutionDiff(conditionalKey.populate(workflowDescriptor.knownValues) + (conditionalKey -> conditionalStatus))
         result
-      case v: WdlValue => throw new RuntimeException("'if' condition must evaluate to a boolean")
+      case v: WdlValue => throw new RuntimeException(
+        s"'if' condition must evaluate to a boolean but instead got ${v.wdlType.toWdlString}")
     }
   }
 
@@ -517,7 +547,8 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor,
       scatterKey.scope.collection.evaluate(lookup, data.expressionLanguageFunctions) map {
         case a: WdlArray =>
           WorkflowExecutionDiff(scatterKey.populate(a.value.size, workflowDescriptor.knownValues) + (scatterKey -> ExecutionStatus.Done))
-        case v: WdlValue => throw new RuntimeException("Scatter collection must evaluate to an array")
+        case v: WdlValue => throw new RuntimeException(
+          s"Scatter collection must evaluate to an array but instead got ${v.wdlType.toWdlString}")
       }
     }
   }
@@ -644,7 +675,7 @@ object WorkflowExecutionActor {
       def makeCollectors(scope: Scope): Seq[CollectorKey] = scope match {
         case call: Call => List(CollectorKey(call, scatter, count))
         case decl: Declaration => List(CollectorKey(decl, scatter, count))
-        case i: If => i.children.flatMap(makeCollectors(_))
+        case i: If => i.children.flatMap(makeCollectors)
       }
 
       (scope match {
@@ -652,7 +683,7 @@ object WorkflowExecutionActor {
         case call: WorkflowCall => (0 until count) map { i => SubWorkflowKey(call, Option(i), 1) }
         case declaration: Declaration => (0 until count) map { i => DeclarationKey(declaration, Option(i), workflowCoercedInputs) }
         case conditional: If => (0 until count) map { i => ConditionalKey(conditional, Option(i)) }
-        case scatter: Scatter =>
+        case _: Scatter =>
           throw new UnsupportedOperationException("Nested Scatters are not supported (yet) ... but you might try a sub workflow to achieve the same effect!")
         case e =>
           throw new UnsupportedOperationException(s"Scope ${e.getClass.getName} is not supported.")

From c152015058d81e70d22d0e83f26d84d9701bd854 Mon Sep 17 00:00:00 2001
From: Chris Llanwarne <cjllanwarne@users.noreply.github.com>
Date: Thu, 2 Mar 2017 21:16:56 +0000
Subject: [PATCH 79/84] JES API Manager: run creation and retries (#2026)

---
 CHANGELOG.md                                       |   2 +-
 .../cromwell/backend/async/ExecutionHandle.scala   |   4 +-
 .../standard/StandardAsyncExecutionActor.scala     |   8 +-
 .../scala/cromwell/util/StopAndLogSupervisor.scala |  11 +-
 project/Dependencies.scala                         |   5 +
 project/Settings.scala                             |   6 +-
 .../jes/JesAsyncBackendJobExecutionActor.scala     |  70 ++++++---
 .../cromwell/backend/impl/jes/JesAttributes.scala  |  24 ++-
 .../impl/jes/JesBackendSingletonActor.scala        |  16 +-
 .../main/scala/cromwell/backend/impl/jes/Run.scala | 164 +++++----------------
 .../jes/statuspolling/JesApiQueryManager.scala     | 120 ++++++++++-----
 .../impl/jes/statuspolling/JesPollingActor.scala   | 101 +++++--------
 .../jes/statuspolling/JesRunCreationClient.scala   |  46 ++++++
 ...orClient.scala => JesStatusRequestClient.scala} |  12 +-
 .../impl/jes/statuspolling/RunCreation.scala       |  42 ++++++
 .../impl/jes/statuspolling/StatusPolling.scala     | 130 ++++++++++++++++
 .../jes/JesAsyncBackendJobExecutionActorSpec.scala |  18 ++-
 .../jes/statuspolling/JesApiQueryManagerSpec.scala |  15 +-
 .../jes/statuspolling/JesPollingActorSpec.scala    |  53 +++----
 .../StatusPollingSpec.scala}                       |  24 +--
 20 files changed, 534 insertions(+), 337 deletions(-)
 create mode 100644 supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesRunCreationClient.scala
 rename supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/{JesPollingActorClient.scala => JesStatusRequestClient.scala} (77%)
 create mode 100644 supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/RunCreation.scala
 create mode 100644 supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/StatusPolling.scala
 rename supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/{RunSpec.scala => statuspolling/StatusPollingSpec.scala} (87%)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index bb8f2b731..a8791a696 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,7 +4,7 @@
 
 ### Breaking Changes
 
-* Metadata keys for call caching are changed. All call caching keys are now in a `callCaching` stanze. `Call cache read result` has moved here and is now `result`. The `allowResultReuse` and `effectiveCallCachingMode` have moved here. The `hit` boolean is a simple indication of whether or not it was a hit, with no additional information. An example using the new format is:
+* Metadata keys for call caching are changed. All call caching keys are now in a `callCaching` stanza. `Call cache read result` has moved here and is now `result`. The `allowResultReuse` and `effectiveCallCachingMode` have moved here. The `hit` boolean is a simple indication of whether or not it was a hit, with no additional information. An example using the new format is:
 ```
 "callCaching": {
   "hit": false,
diff --git a/backend/src/main/scala/cromwell/backend/async/ExecutionHandle.scala b/backend/src/main/scala/cromwell/backend/async/ExecutionHandle.scala
index 1a0262d36..80e74c960 100644
--- a/backend/src/main/scala/cromwell/backend/async/ExecutionHandle.scala
+++ b/backend/src/main/scala/cromwell/backend/async/ExecutionHandle.scala
@@ -9,12 +9,12 @@ import cromwell.core.{CallOutputs, ExecutionEvent}
  * Trait to encapsulate whether an execution is complete and if so provide a result.  Useful in conjunction
  * with the `poll` API to feed results of previous job status queries forward.
  */
-trait ExecutionHandle {
+sealed trait ExecutionHandle {
   def isDone: Boolean
   def result: ExecutionResult
 }
 
-case class PendingExecutionHandle[BackendJobId <: JobId, BackendRunInfo, BackendRunStatus]
+final case class PendingExecutionHandle[BackendJobId <: JobId, BackendRunInfo, BackendRunStatus]
 (
   jobDescriptor: BackendJobDescriptor,
   pendingJob: BackendJobId,
diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala b/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala
index 261bda4a9..0d5b73684 100644
--- a/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala
+++ b/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala
@@ -576,7 +576,7 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
       the state names.
        */
       val prevStateName = previousStatus.map(_.toString).getOrElse("-")
-      jobLogger.info(s"$tag Status change from $prevStateName to $status")
+      jobLogger.info(s"Status change from $prevStateName to $status")
       tellMetadata(Map(CallMetadataKeys.BackendStatus -> status))
     }
 
@@ -604,11 +604,11 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
           customPollStatusFailure orElse {
             case (_: ExecutionHandle, exception: Exception) if isFatal(exception) =>
               // Log exceptions and return the original handle to try again.
-              jobLogger.warn(s"Caught fatal exception attempting to poll", exception)
+              jobLogger.warn(s"Fatal exception polling for status. Job will fail.")
               FailedNonRetryableExecutionHandle(exception)
             case (handle: ExecutionHandle, exception: Exception) =>
               // Log exceptions and return the original handle to try again.
-              jobLogger.warn(s"Caught exception trying to poll, retrying", exception)
+              jobLogger.warn(s"Caught non-fatal ${exception.getClass.getSimpleName} exception trying to poll, retrying", exception)
               handle
           }
         handler((oldHandle, exception))
@@ -708,4 +708,4 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
   *
   * @param jobId The job id.
   */
-case class StandardAsyncJob(jobId: String) extends JobId
+final case class StandardAsyncJob(jobId: String) extends JobId
diff --git a/core/src/main/scala/cromwell/util/StopAndLogSupervisor.scala b/core/src/main/scala/cromwell/util/StopAndLogSupervisor.scala
index 4b64e5d8b..929ae6dd2 100644
--- a/core/src/main/scala/cromwell/util/StopAndLogSupervisor.scala
+++ b/core/src/main/scala/cromwell/util/StopAndLogSupervisor.scala
@@ -2,9 +2,8 @@ package cromwell.util
 
 import akka.actor.SupervisorStrategy.{Decider, Stop}
 import akka.actor.{Actor, ActorRef, OneForOneStrategy, SupervisorStrategy}
-import cromwell.core.logging.WorkflowLogging
 
-trait StopAndLogSupervisor { this: Actor with WorkflowLogging =>
+trait StopAndLogSupervisor { this: Actor =>
 
   private var failureLog: Map[ActorRef, Throwable] = Map.empty
 
@@ -15,10 +14,14 @@ trait StopAndLogSupervisor { this: Actor with WorkflowLogging =>
         failureLog += failer -> e
         Stop
     }
-    OneForOneStrategy()(stoppingDecider)
+    OneForOneStrategy(loggingEnabled = false)(stoppingDecider)
   }
 
-  final def getFailureCause(actorRef: ActorRef): Option[Throwable] = failureLog.get(actorRef)
+  final def getFailureCause(actorRef: ActorRef): Option[Throwable] = {
+    val result = failureLog.get(actorRef)
+    failureLog -= actorRef
+    result
+  }
 
   override final val supervisorStrategy = stopAndLogStrategy
 }
diff --git a/project/Dependencies.scala b/project/Dependencies.scala
index e59df1939..83fa0c1d1 100644
--- a/project/Dependencies.scala
+++ b/project/Dependencies.scala
@@ -136,6 +136,11 @@ object Dependencies {
     "org.mongodb" %% "casbah" % "3.0.0"
   )
 
+  val jesBackendDependencies = List(
+    "org.scala-lang" % "scala-compiler" % Settings.ScalaVersion,
+    "eu.timepit" %% "refined" % "0.7.0"
+  )
+
   val tesBackendDependencies = List(
     "io.spray" %% "spray-client" % sprayV
   ) ++ sprayServerDependencies
diff --git a/project/Settings.scala b/project/Settings.scala
index c86689b84..aa217201a 100644
--- a/project/Settings.scala
+++ b/project/Settings.scala
@@ -100,10 +100,11 @@ object Settings {
     )
   )
 
+  val ScalaVersion = "2.11.8"
   val commonSettings = ReleasePlugin.projectSettings ++ testSettings ++ assemblySettings ++
     dockerSettings ++ cromwellVersionWithGit ++ publishingSettings ++ List(
     organization := "org.broadinstitute",
-    scalaVersion := "2.11.8",
+    scalaVersion := ScalaVersion,
     resolvers ++= commonResolvers,
     scalacOptions ++= compilerSettings,
     scalacOptions in (Compile, doc) ++= docSettings,
@@ -158,7 +159,8 @@ object Settings {
   ) ++ commonSettings
 
   val jesBackendSettings = List(
-    name := "cromwell-jes-backend"
+    name := "cromwell-jes-backend",
+    libraryDependencies ++= jesBackendDependencies
   ) ++ commonSettings
 
   val engineSettings = List(
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala
index bc958475d..65050a67c 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala
@@ -4,16 +4,19 @@ import java.net.SocketTimeoutException
 
 import akka.actor.ActorRef
 import com.google.api.client.googleapis.json.GoogleJsonResponseException
+import com.google.api.services.genomics.model.RunPipelineRequest
 import cromwell.backend._
 import cromwell.backend.async.{AbortedExecutionHandle, ExecutionHandle, FailedNonRetryableExecutionHandle, FailedRetryableExecutionHandle, PendingExecutionHandle}
 import cromwell.backend.impl.jes.RunStatus.TerminalRunStatus
 import cromwell.backend.impl.jes.io._
-import cromwell.backend.impl.jes.statuspolling.JesPollingActorClient
+import cromwell.backend.impl.jes.statuspolling.{JesRunCreationClient, JesStatusRequestClient}
 import cromwell.backend.standard.{StandardAsyncExecutionActor, StandardAsyncExecutionActorParams, StandardAsyncJob}
 import cromwell.core._
+import cromwell.core.logging.JobLogger
 import cromwell.core.path.{DefaultPathBuilder, Path}
 import cromwell.core.retry.SimpleExponentialBackoff
 import cromwell.filesystems.gcs.GcsPath
+import org.slf4j.LoggerFactory
 import wdl4s._
 import wdl4s.expression.NoFunctions
 import wdl4s.values._
@@ -21,7 +24,8 @@ import wdl4s.values._
 import scala.concurrent.duration._
 import scala.concurrent.{ExecutionContext, Future}
 import scala.language.postfixOps
-import scala.util.{Failure, Success, Try}
+import scala.util.{Success, Try}
+import scala.collection.JavaConverters._
 
 object JesAsyncBackendJobExecutionActor {
   val JesOperationIdKey = "__jes_operation_id"
@@ -36,14 +40,18 @@ object JesAsyncBackendJobExecutionActor {
     PendingExecutionHandle[StandardAsyncJob, Run, RunStatus]
 
   private val ExtraConfigParamName = "__extra_config_gcs_path"
+  private def stringifyMap(m: Map[String, String]): String = m map { case(k, v) => s"  $k -> $v"} mkString "\n"
 }
 
 class JesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyncExecutionActorParams)
   extends BackendJobLifecycleActor with StandardAsyncExecutionActor with JesJobCachingActorHelper
-    with JesPollingActorClient {
+    with JesStatusRequestClient with JesRunCreationClient  {
 
   import JesAsyncBackendJobExecutionActor._
 
+  val slf4jLogger = LoggerFactory.getLogger(JesAsyncBackendJobExecutionActor.getClass)
+  val logger = new JobLogger("JesRun", jobDescriptor.workflowDescriptor.id, jobDescriptor.key.tag, None, Set(slf4jLogger))
+
   val jesBackendSingletonActor: ActorRef =
     standardParams.backendSingletonActorOption.getOrElse(
       throw new RuntimeException("JES Backend actor cannot exist without the JES backend singleton actor"))
@@ -71,10 +79,10 @@ class JesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn
   private lazy val dockerConfiguration = jesConfiguration.dockerCredentials
 
   override def tryAbort(job: StandardAsyncJob): Unit = {
-    Run(job.jobId, initializationData.genomics).abort()
+    Run(job, initializationData.genomics).abort()
   }
 
-  override def receive: Receive = pollingActorClientReceive orElse super.receive
+  override def receive: Receive = pollingActorClientReceive orElse runCreationClientReceive orElse super.receive
 
   private def gcsAuthParameter: Option[JesInput] = {
     if (jesAttributes.auths.gcs.requiresAuthFile || dockerConfiguration.isDefined)
@@ -218,9 +226,8 @@ class JesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn
     }
   }
 
-  private def createJesRun(jesParameters: Seq[JesParameter], runIdForResumption: Option[String]): Run = {
-    Run(
-      runIdForResumption,
+  private def createJesRunPipelineRequest(jesParameters: Seq[JesParameter]): RunPipelineRequest = {
+    val runPipelineParameters = Run.makeRunPipelineRequest(
       jobDescriptor = jobDescriptor,
       runtimeAttributes = runtimeAttributes,
       callRootPath = callRootPath.pathAsString,
@@ -232,41 +239,58 @@ class JesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn
       retryable,
       initializationData.genomics
     )
+    logger.debug(s"Inputs:\n${stringifyMap(runPipelineParameters.getPipelineArgs.getInputs.asScala.toMap)}")
+    logger.debug(s"Outputs:\n${stringifyMap(runPipelineParameters.getPipelineArgs.getOutputs.asScala.toMap)}")
+    runPipelineParameters
   }
 
-  override def isFatal(throwable: Throwable): Boolean = isFatalJesException(throwable)
+  override def isFatal(throwable: Throwable): Boolean = super.isFatal(throwable) || isFatalJesException(throwable)
 
   override def isTransient(throwable: Throwable): Boolean = isTransientJesException(throwable)
 
-  override def execute(): ExecutionHandle = {
+  override def executeAsync()(implicit ec: ExecutionContext): Future[ExecutionHandle] = {
     runWithJes(None)
   }
 
-  protected def runWithJes(runIdForResumption: Option[String]): ExecutionHandle = {
-    // Force runtimeAttributes to evaluate so we can fail quickly now if we need to:
-    Try(runtimeAttributes) match {
-      case Success(_) =>
-        val jesInputs: Set[JesInput] = generateJesInputs(jobDescriptor) ++ monitoringScript + cmdInput
-        val jesOutputs: Set[JesFileOutput] = generateJesOutputs(jobDescriptor) ++ monitoringOutput
+  override def recoverAsync(jobId: StandardAsyncJob)(implicit ec: ExecutionContext): Future[ExecutionHandle] = {
+    runWithJes(Option(jobId))
+  }
+
+  private def runWithJes(jobForResumption: Option[StandardAsyncJob]): Future[ExecutionHandle] = {
 
-        val jesParameters = standardParameters ++ gcsAuthParameter ++ jesInputs ++ jesOutputs
+    // Want to force runtimeAttributes to evaluate so we can fail quickly now if we need to:
+    def makeRpr: Try[RunPipelineRequest] = Try(runtimeAttributes) flatMap { _ => Try {
+      val jesInputs: Set[JesInput] = generateJesInputs(jobDescriptor) ++ monitoringScript + cmdInput
+      val jesOutputs: Set[JesFileOutput] = generateJesOutputs(jobDescriptor) ++ monitoringOutput
 
-        jobPaths.script.writeAsText(commandScriptContents)
-        val run = createJesRun(jesParameters, runIdForResumption)
-        PendingExecutionHandle(jobDescriptor, StandardAsyncJob(run.runId), Option(run), previousStatus = None)
-      case Failure(e) => FailedNonRetryableExecutionHandle(e)
+      val jesParameters = standardParameters ++ gcsAuthParameter ++ jesInputs ++ jesOutputs
+
+      jobPaths.script.writeAsText(commandScriptContents)
+      createJesRunPipelineRequest(jesParameters)
+    }}
+
+    jobForResumption match {
+      case Some(job) =>
+        val run = Run(job, initializationData.genomics)
+        Future.successful(PendingExecutionHandle(jobDescriptor, job, Option(run), previousStatus = None))
+      case None =>
+        for {
+          rpr <- Future.fromTry(makeRpr)
+          runId <- runPipeline(initializationData.genomics, rpr)
+          run = Run(runId, initializationData.genomics)
+        } yield PendingExecutionHandle(jobDescriptor, runId, Option(run), previousStatus = None)
     }
   }
 
   override def pollStatusAsync(handle: JesPendingExecutionHandle)
                               (implicit ec: ExecutionContext): Future[RunStatus] = {
-    super[JesPollingActorClient].pollStatus(handle.runInfo.get)
+    super[JesStatusRequestClient].pollStatus(handle.runInfo.get)
   }
 
 
   override def customPollStatusFailure: PartialFunction[(ExecutionHandle, Exception), ExecutionHandle] = {
     case (oldHandle: JesPendingExecutionHandle@unchecked, e: GoogleJsonResponseException) if e.getStatusCode == 404 =>
-      jobLogger.error(s"$tag JES Job ID ${oldHandle.runInfo.get.runId} has not been found, failing call")
+      jobLogger.error(s"JES Job ID ${oldHandle.runInfo.get.job} has not been found, failing call")
       FailedNonRetryableExecutionHandle(e)
   }
 
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala
index dc9534546..5b85dc05e 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala
@@ -9,6 +9,8 @@ import cats.syntax.validated._
 import com.typesafe.config.{Config, ConfigValue}
 import cromwell.backend.impl.jes.authentication.JesAuths
 import cromwell.filesystems.gcs.GoogleConfiguration
+import eu.timepit.refined.api.Refined
+import eu.timepit.refined.numeric.Positive
 import lenthall.exception.MessageAggregation
 import lenthall.validation.ErrorOr._
 import lenthall.validation.Validation._
@@ -24,7 +26,7 @@ case class JesAttributes(project: String,
                          executionBucket: String,
                          endpointUrl: URL,
                          maxPollingInterval: Int,
-                         qps: Int,
+                         qps: Int Refined Positive,
                          defaultZones: NonEmptyList[String])
 
 object JesAttributes {
@@ -64,12 +66,12 @@ object JesAttributes {
     val computeServiceAccount: String = backendConfig.as[Option[String]]("genomics.compute-service-account").getOrElse("default")
     val genomicsAuthName: ErrorOr[String] = validate { backendConfig.as[String]("genomics.auth") }
     val gcsFilesystemAuthName: ErrorOr[String] = validate { backendConfig.as[String]("filesystems.gcs.auth") }
-    val qps = backendConfig.as[Option[Int]]("genomics-api-queries-per-100-seconds").getOrElse(GenomicsApiDefaultQps) / 100
+    val qpsValidation = validateQps(backendConfig)
     val defaultZones = defaultZonesFromConfig(backendConfig)
 
-    (project |@| executionBucket |@| endpointUrl |@| genomicsAuthName |@| gcsFilesystemAuthName |@| defaultZones) map {
-      (_, _, _, _, _, _)
-    } flatMap { case (p, b, u, genomicsName, gcsName, d) =>
+    (project |@| executionBucket |@| endpointUrl |@| genomicsAuthName |@| gcsFilesystemAuthName |@| defaultZones |@| qpsValidation) map {
+      (_, _, _, _, _, _, _)
+    } flatMap { case (p, b, u, genomicsName, gcsName, d, qps) =>
       (googleConfig.auth(genomicsName) |@| googleConfig.auth(gcsName)) map { case (genomicsAuth, gcsAuth) =>
         JesAttributes(p, computeServiceAccount, JesAuths(genomicsAuth, gcsAuth), b, u, maxPollingInterval, qps, d)
       }
@@ -90,4 +92,16 @@ object JesAttributes {
       case _ => "genomics.default-zones was set but no values were provided".invalidNel
     }
   }
+
+  def validateQps(config: Config): ErrorOr[Int Refined Positive] = {
+    import eu.timepit.refined._
+
+    val qp100s = config.as[Option[Int]]("genomics-api-queries-per-100-seconds").getOrElse(GenomicsApiDefaultQps)
+    val qpsCandidate = qp100s / 100
+
+    refineV[Positive](qpsCandidate) match {
+      case Left(_) => s"Calculated QPS for Google Genomics API ($qpsCandidate/s) was not a positive integer (supplied value was $qp100s per 100s)".invalidNel
+      case Right(refined) => refined.validNel
+    }
+  }
 }
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendSingletonActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendSingletonActor.scala
index 5099cc172..f1a5eda4a 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendSingletonActor.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendSingletonActor.scala
@@ -3,19 +3,21 @@ package cromwell.backend.impl.jes
 import akka.actor.{Actor, ActorLogging, Props}
 import cromwell.core.Dispatcher.BackendDispatcher
 import cromwell.backend.impl.jes.statuspolling.JesApiQueryManager
-import cromwell.backend.impl.jes.statuspolling.JesApiQueryManager.DoPoll
+import cromwell.backend.impl.jes.statuspolling.JesApiQueryManager.JesApiQueryManagerRequest
+import eu.timepit.refined.api.Refined
+import eu.timepit.refined.numeric.Positive
 
-final case class JesBackendSingletonActor(qps: Int) extends Actor with ActorLogging {
+final case class JesBackendSingletonActor(qps: Int Refined Positive) extends Actor with ActorLogging {
 
-  val pollingActor = context.actorOf(JesApiQueryManager.props(qps))
+  val jesApiQueryManager = context.actorOf(JesApiQueryManager.props(qps))
 
   override def receive = {
-    case poll: DoPoll =>
-      log.debug("Forwarding status poll to JES polling actor")
-      pollingActor.forward(poll)
+    case apiQuery: JesApiQueryManagerRequest =>
+      log.debug("Forwarding API query to JES API query manager actor")
+      jesApiQueryManager.forward(apiQuery)
   }
 }
 
 object JesBackendSingletonActor {
-  def props(qps: Int): Props = Props(JesBackendSingletonActor(qps)).withDispatcher(BackendDispatcher)
+  def props(qps: Int Refined Positive): Props = Props(JesBackendSingletonActor(qps)).withDispatcher(BackendDispatcher)
 }
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala
index 70047bfe7..acafef4f5 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala
@@ -1,20 +1,13 @@
 package cromwell.backend.impl.jes
 
-import java.time.OffsetDateTime
-import java.util.{ArrayList => JArrayList}
-
-import com.google.api.client.util.{ArrayMap => GArrayMap}
 import com.google.api.services.genomics.Genomics
 import com.google.api.services.genomics.model._
 import cromwell.backend.BackendJobDescriptor
-import cromwell.backend.impl.jes.RunStatus.{Failed, Initializing, Running, Success}
-import cromwell.core.ExecutionEvent
+import cromwell.backend.standard.StandardAsyncJob
 import cromwell.core.labels.Labels
-import cromwell.core.logging.JobLogger
 import org.slf4j.LoggerFactory
 
 import scala.collection.JavaConverters._
-import scala.language.postfixOps
 
 object Run {
   private val GenomicsScopes = List(
@@ -22,41 +15,39 @@ object Run {
     "https://www.googleapis.com/auth/compute"
   ).asJava
 
-  private val AcceptableEvents = Set("start", "pulling-image", "localizing-files", "running-docker", "delocalizing-files", "ok", "fail", "start-shutdown", "preempted")
-
   val NoAddressFieldName = "noAddress"
 
   val slf4jLogger = LoggerFactory.getLogger(Run.getClass)
 
-  def apply(runIdForResumption: Option[String],
-            jobDescriptor: BackendJobDescriptor,
-            runtimeAttributes: JesRuntimeAttributes,
-            callRootPath: String,
-            commandLine: String,
-            logFileName: String,
-            jesParameters: Seq[JesParameter],
-            projectId: String,
-            computeServiceAccount: String,
-            preemptible: Boolean,
-            genomicsInterface: Genomics): Run = {
-    val logger = new JobLogger("JesRun", jobDescriptor.workflowDescriptor.id, jobDescriptor.key.tag, None, Set(slf4jLogger))
+  def makeRunPipelineRequest(jobDescriptor: BackendJobDescriptor,
+                             runtimeAttributes: JesRuntimeAttributes,
+                             callRootPath: String,
+                             commandLine: String,
+                             logFileName: String,
+                             jesParameters: Seq[JesParameter],
+                             projectId: String,
+                             computeServiceAccount: String,
+                             preemptible: Boolean,
+                             genomicsInterface: Genomics): RunPipelineRequest = {
 
     lazy val workflow = jobDescriptor.workflowDescriptor
     val pipelineInfoBuilder = if (preemptible) PreemptibleJesPipelineInfoBuilder else NonPreemptibleJesPipelineInfoBuilder
     val pipelineInfo = pipelineInfoBuilder.build(commandLine, runtimeAttributes)
 
     val pipeline = new Pipeline()
-             .setProjectId(projectId)
-             .setDocker(pipelineInfo.docker)
-             .setResources(pipelineInfo.resources)
-             .setName(workflow.workflow.unqualifiedName)
-             .setInputParameters(jesParameters.collect({ case i: JesInput => i.toGooglePipelineParameter }).toVector.asJava)
-             .setOutputParameters(jesParameters.collect({ case i: JesFileOutput => i.toGooglePipelineParameter }).toVector.asJava)
+      .setProjectId(projectId)
+      .setDocker(pipelineInfo.docker)
+      .setResources(pipelineInfo.resources)
+      .setName(workflow.workflow.unqualifiedName)
+      .setInputParameters(jesParameters.collect({ case i: JesInput => i.toGooglePipelineParameter }).toVector.asJava)
+      .setOutputParameters(jesParameters.collect({ case i: JesFileOutput => i.toGooglePipelineParameter }).toVector.asJava)
 
     // disks cannot have mount points at runtime, so set them null
     val runtimePipelineResources = {
       val resources = pipelineInfoBuilder.build(commandLine, runtimeAttributes).resources
-      val disksWithoutMountPoint = resources.getDisks.asScala map { _.setMountPoint(null) }
+      val disksWithoutMountPoint = resources.getDisks.asScala map {
+        _.setMountPoint(null)
+      }
       resources.setDisks(disksWithoutMountPoint.asJava)
     }
 
@@ -81,120 +72,31 @@ object Run {
       ) ++ subWorkflowLabels ++ aliasLabels ++ jobDescriptor.workflowDescriptor.customLabels
     }
 
-    def runPipeline: String = {
-      val svcAccount = new ServiceAccount().setEmail(computeServiceAccount).setScopes(GenomicsScopes)
-      val rpargs = new RunPipelineArgs().setProjectId(projectId).setServiceAccount(svcAccount).setResources(runtimePipelineResources)
-
-      rpargs.setInputs(jesParameters.collect({ case i: JesInput => i.name -> i.toGoogleRunParameter }).toMap.asJava)
-      logger.debug(s"Inputs:\n${stringifyMap(rpargs.getInputs.asScala.toMap)}")
-
-      rpargs.setOutputs(jesParameters.collect({ case i: JesFileOutput => i.name -> i.toGoogleRunParameter }).toMap.asJava)
-      logger.debug(s"Outputs:\n${stringifyMap(rpargs.getOutputs.asScala.toMap)}")
-
-      rpargs.setLabels(labels.asJesLabels)
-
-      val rpr = new RunPipelineRequest().setEphemeralPipeline(pipeline).setPipelineArgs(rpargs)
-
-      val logging = new LoggingOptions()
-      logging.setGcsPath(s"$callRootPath/$logFileName")
-      rpargs.setLogging(logging)
-
-      val runId = genomicsInterface.pipelines().run(rpr).execute().getName
-      logger.info(s"JES Run ID is $runId")
-      runId
-    }
-
-    // If runIdForResumption is defined use that, otherwise we'll create a new Run with an ephemeral pipeline.
-    val runId = runIdForResumption getOrElse runPipeline
-    if (runIdForResumption.isDefined) { logger.info(s"JES Run is resuming with Run ID: ${runIdForResumption.get.toString}") }
-    new Run(runId, genomicsInterface)
-  }
-
-  private def stringifyMap(m: Map[String, String]): String = m map { case(k, v) => s"  $k -> $v"} mkString "\n"
-
-  implicit class RunOperationExtension(val operation: Operation) extends AnyVal {
-    def hasStarted = operation.getMetadata.asScala.get("startTime") isDefined
-  }
-
-  def interpretOperationStatus(op: Operation): RunStatus = {
-    require(op != null, "Operation must not be null.")
-    try {
-      if (op.getDone) {
-        lazy val eventList = getEventList(op)
-        lazy val computeEngineOption = for {
-          runtimeMetadata <- op.getMetadata.asScala.get("runtimeMetadata")
-          computeEngine <- runtimeMetadata.asInstanceOf[GArrayMap[String, Object]].asScala.get("computeEngine")
-        } yield computeEngine.asInstanceOf[GArrayMap[String, String]].asScala
-        lazy val machineType = computeEngineOption.flatMap(_.get("machineType"))
-        lazy val instanceName = computeEngineOption.flatMap(_.get("instanceName"))
-        lazy val zone = computeEngineOption.flatMap(_.get("zone"))
-
-        // If there's an error, generate a Failed status. Otherwise, we were successful!
-        Option(op.getError) match {
-          case None => Success(eventList, machineType, zone, instanceName)
-          case Some(error) =>
-            Failed(error.getCode, Option(error.getMessage), eventList, machineType, zone, instanceName)
-        }
-      } else if (op.hasStarted) {
-        Running
-      } else {
-        Initializing
-      }
-    } catch {
-      case npe: NullPointerException =>
-        throw new RuntimeException(s"Caught NPE while processing operation ${op.getName}: $op", npe)
-    }
-  }
-
-  def getEventList(op: Operation): Seq[ExecutionEvent] = {
-    val metadata = op.getMetadata.asScala.toMap
+    val svcAccount = new ServiceAccount().setEmail(computeServiceAccount).setScopes(GenomicsScopes)
+    val rpargs = new RunPipelineArgs().setProjectId(projectId).setServiceAccount(svcAccount).setResources(runtimePipelineResources)
 
-    val starterEvents: Seq[ExecutionEvent] = Seq(
-      eventIfExists("createTime", metadata, "waiting for quota"),
-      eventIfExists("startTime", metadata, "initializing VM")).flatten
+    rpargs.setInputs(jesParameters.collect({ case i: JesInput => i.name -> i.toGoogleRunParameter }).toMap.asJava)
+    rpargs.setOutputs(jesParameters.collect({ case i: JesFileOutput => i.name -> i.toGoogleRunParameter }).toMap.asJava)
 
-    val eventsList = for {
-      events <- metadata.get("events").toSeq
-      entry <- events.asInstanceOf[JArrayList[GArrayMap[String, String]]].asScala
-    } yield ExecutionEvent(entry.get("description"), OffsetDateTime.parse(entry.get("startTime")))
+    rpargs.setLabels(labels.asJesLabels)
 
-    val filteredEventsList: Seq[ExecutionEvent] = eventsList filter { i => AcceptableEvents.contains(i.name) }
+    val rpr = new RunPipelineRequest().setEphemeralPipeline(pipeline).setPipelineArgs(rpargs)
 
-    // A little bit ugly... the endTime of the jes operation can actually be before the final "event" time, due to differences
-    // in the reported precision. As a result, we have to make sure it all lines up nicely:
-    val finalEvent = getCromwellPollIntervalEvent(metadata, filteredEventsList)
-
-    starterEvents ++ filteredEventsList :+ finalEvent
-  }
-
-  private def getCromwellPollIntervalEvent(metadata: Map[String, AnyRef], eventsList: Seq[ExecutionEvent]) = {
-    {
-      val jesReportedEndTime = eventIfExists("endTime", metadata, "cromwell poll interval")
-      val finalEventsListTime = if (eventsList.nonEmpty) Some(eventsList.last.offsetDateTime) else None
-
-      (jesReportedEndTime, finalEventsListTime) match {
-        case (Some(jesEndTime), Some(finalEventTime)) =>
-          if (jesEndTime.offsetDateTime isAfter finalEventTime) jesEndTime else jesEndTime.copy(offsetDateTime = finalEventTime)
-        case (Some(jesEndTime), None) => jesEndTime
-        case (None, Some(finalEventTime)) => ExecutionEvent("cromwell poll interval", finalEventTime)
-        case (None, None) =>
-          throw new IllegalArgumentException("Both jesReportedEndTime and finalEventsListTime were None.")
-      }
-    }
-  }
+    val logging = new LoggingOptions()
+    logging.setGcsPath(s"$callRootPath/$logFileName")
+    rpargs.setLogging(logging)
 
-  private def eventIfExists(name: String, metadata: Map[String, AnyRef], eventName: String): Option[ExecutionEvent] = {
-    metadata.get(name) map { time => ExecutionEvent(eventName, OffsetDateTime.parse(time.toString)) }
+    rpr
   }
 }
 
-case class Run(runId: String, genomicsInterface: Genomics) {
+final case class Run(job: StandardAsyncJob, genomicsInterface: Genomics) {
 
-  def getOperationCommand = genomicsInterface.operations().get(runId)
+  def getOperationCommand = genomicsInterface.operations().get(job.jobId)
 
   def abort(): Unit = {
     val cancellationRequest: CancelOperationRequest = new CancelOperationRequest()
-    genomicsInterface.operations().cancel(runId, cancellationRequest).execute
+    genomicsInterface.operations().cancel(job.jobId, cancellationRequest).execute
     ()
   }
 }
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesApiQueryManager.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesApiQueryManager.scala
index 9d7a7226c..c24c9d76e 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesApiQueryManager.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesApiQueryManager.scala
@@ -1,65 +1,86 @@
 package cromwell.backend.impl.jes.statuspolling
 
-import akka.actor.{Actor, ActorLogging, ActorRef, Props, SupervisorStrategy, Terminated}
+import java.io.IOException
+
+import akka.actor.{Actor, ActorLogging, ActorRef, Props, Terminated}
 import cats.data.NonEmptyList
+import com.google.api.client.googleapis.json.GoogleJsonError
+import com.google.api.client.http.HttpHeaders
+import com.google.api.services.genomics.Genomics
+import com.google.api.services.genomics.model.RunPipelineRequest
 import cromwell.backend.impl.jes.Run
 import cromwell.backend.impl.jes.statuspolling.JesApiQueryManager._
+import cromwell.core.CromwellFatalExceptionMarker
 import cromwell.core.Dispatcher.BackendDispatcher
+import cromwell.core.retry.{Backoff, SimpleExponentialBackoff}
+import cromwell.util.StopAndLogSupervisor
+import eu.timepit.refined.api.Refined
+import eu.timepit.refined.numeric._
 
+import scala.concurrent.duration._
 import scala.collection.immutable.Queue
 
 /**
-  * Currently, just holds a set of JES status poll requests until a PollingActor pulls the work.
-  * TODO: Could eventually move all of the JES queries into a single work-pulling model.
+  * Holds a set of JES API requests until a JesQueryActor pulls the work.
   */
-class JesApiQueryManager(val qps: Int) extends Actor with ActorLogging {
+class JesApiQueryManager(val qps: Int Refined Positive) extends Actor with ActorLogging with StopAndLogSupervisor {
+
+  private implicit val ec = context.dispatcher
+  private val maxRetries = 10
 
   // workQueue is protected for the unit tests, not intended to be generally overridden
-  protected[statuspolling] var workQueue: Queue[JesStatusPollQuery] = Queue.empty
+  protected[statuspolling] var workQueue: Queue[JesApiQuery] = Queue.empty
   private var workInProgress: Map[ActorRef, JesPollingWorkBatch] = Map.empty
 
-  // If the statusPoller dies, we want to stop it and handle the termination ourselves.
-  override val supervisorStrategy = SupervisorStrategy.stoppingStrategy
   private def statusPollerProps = JesPollingActor.props(self, qps)
 
   // statusPoller is protected for the unit tests, not intended to be generally overridden
   protected[statuspolling] var statusPoller: ActorRef = _
 
-  resetStatusPoller()
+  resetWorker()
 
   override def receive = {
     case DoPoll(run) => workQueue :+= JesStatusPollQuery(sender, run)
+    case DoCreateRun(genomics, rpr) =>
+      workQueue :+= JesRunCreationQuery(sender, genomics, rpr)
+    case q: JesApiQuery => workQueue :+= q
     case RequestJesPollingWork(maxBatchSize) =>
-      log.debug(s"Request for JES Polling Work received (max batch: $maxBatchSize, current queue size is ${workQueue.size})")
+      log.debug("Request for JES Polling Work received (max batch: {}, current queue size is {})", maxBatchSize, workQueue.size)
       handleJesPollingRequest(sender, maxBatchSize)
+    case failure: JesApiQueryFailed => handleQueryFailure(failure)
     case Terminated(actorRef) => handleTerminated(actorRef)
     case other => log.error(s"Unexpected message to JesPollingManager: $other")
   }
 
-  /**
-    * !! Function Impurity Warning: Modifies Actor Data !!
-    */
+  private def handleQueryFailure(failure: JesApiQueryFailed) = failure match {
+    case JesApiQueryFailed(request, _) if request.failedAttempts < maxRetries =>
+      val nextRequest = request.withFailedAttempt
+      val delay = nextRequest.backoff.backoffMillis.millis
+      context.system.scheduler.scheduleOnce(delay, self, nextRequest)
+      ()
+    case JesApiQueryFailed(request, _) =>
+      request.requester ! failure
+      ()
+  }
+
   private def handleJesPollingRequest(workPullingJesPollingActor: ActorRef, maxBatchSize: Int) = {
     workInProgress -= workPullingJesPollingActor
     val beheaded = beheadWorkQueue(maxBatchSize)
     beheaded.workToDo match {
       case Some(work) =>
-        sendWork(workPullingJesPollingActor, JesPollingWorkBatch(work))
+        log.debug("Sending work to JES API query manager.")
+        val workBatch = JesPollingWorkBatch(work)
+        workPullingJesPollingActor ! workBatch
+        workInProgress += (workPullingJesPollingActor -> workBatch)
       case None =>
-        log.debug(s"No work for JES poller. Sad.")
+        log.debug("No work for JES poller")
         workPullingJesPollingActor ! NoWorkToDo
     }
 
     workQueue = beheaded.newWorkQueue
   }
 
-  private def sendWork(workPullingJesPollingActor: ActorRef, work: JesPollingWorkBatch) = {
-    log.debug(s"Sending work to JES poller.")
-    workPullingJesPollingActor ! work
-    workInProgress += (workPullingJesPollingActor -> work)
-  }
-
-  private final case class BeheadedWorkQueue(workToDo: Option[NonEmptyList[JesStatusPollQuery]], newWorkQueue: Queue[JesStatusPollQuery])
+  private final case class BeheadedWorkQueue(workToDo: Option[NonEmptyList[JesApiQuery]], newWorkQueue: Queue[JesApiQuery])
   private def beheadWorkQueue(maxBatchSize: Int): BeheadedWorkQueue = {
 
     val head = workQueue.take(maxBatchSize).toList
@@ -72,44 +93,77 @@ class JesApiQueryManager(val qps: Int) extends Actor with ActorLogging {
   }
 
   private def handleTerminated(terminee: ActorRef) = {
-    // Currently we can assume this is a polling actor. Might change in a future update:
+    val cause = getFailureCause(terminee).getOrElse(new RuntimeException("No failure reason recorded"))
+    // We assume this is a polling actor. Might change in a future update:
     workInProgress.get(terminee) match {
       case Some(work) =>
         // Most likely due to an unexpected HTTP error, push the work back on the queue and keep going
-        log.info(s"The JES polling actor $terminee unexpectedly terminated while conducting ${work.workBatch.tail.size + 1} polls. Making a new one...")
+        log.error(s"The JES API worker actor $terminee unexpectedly terminated while conducting ${work.workBatch.tail.size + 1} polls. Making a new one...")
         workInProgress -= terminee
-        workQueue = workQueue ++ work.workBatch.toList
+        work.workBatch.toList.foreach { toResubmit =>
+          self ! JesApiQueryFailed(toResubmit, new JesApiException(cause))
+        }
       case None =>
         // It managed to die while doing absolutely nothing...!?
         // Maybe it deserves an entry in https://en.wikipedia.org/wiki/List_of_unusual_deaths
         // Oh well, in the mean time don't do anything, just start a new one
-        log.error(s"The JES polling actor $terminee managed to unexpectedly terminate whilst doing absolutely nothing. This is probably a programming error. Making a new one...")
+        log.error(cause, s"The JES API worker actor managed to unexpectedly terminate whilst doing absolutely nothing (${cause.getMessage}). This is probably a programming error. Making a new one...")
     }
 
-    resetStatusPoller()
+    resetWorker()
   }
 
-  private def resetStatusPoller() = {
-    statusPoller = makeStatusPoller()
+  private def resetWorker() = {
+    statusPoller = makeWorkerActor()
     context.watch(statusPoller)
     log.info(s"watching $statusPoller")
   }
 
-  private[statuspolling] def makeStatusPoller(): ActorRef = context.actorOf(statusPollerProps)
+  // Separate method to allow overriding in tests:
+  private[statuspolling] def makeWorkerActor(): ActorRef = context.actorOf(statusPollerProps)
 }
 
 object JesApiQueryManager {
 
-  def props(qps: Int): Props = Props(new JesApiQueryManager(qps)).withDispatcher(BackendDispatcher)
+  def props(qps: Int Refined Positive): Props = Props(new JesApiQueryManager(qps)).withDispatcher(BackendDispatcher)
+
+  sealed trait JesApiQueryManagerRequest
 
   /**
     * Poll the job represented by the Run.
     */
-  final case class DoPoll(run: Run)
+  final case class DoPoll(run: Run) extends JesApiQueryManagerRequest
 
-  private[statuspolling] final case class JesStatusPollQuery(requester: ActorRef, run: Run)
-  private[statuspolling] final case class JesPollingWorkBatch(workBatch: NonEmptyList[JesStatusPollQuery])
+  /**
+    * Create an ephemeral pipeline and run it in JES.
+    */
+  final case class DoCreateRun(genomicsInterface: Genomics, rpr: RunPipelineRequest) extends JesApiQueryManagerRequest
+
+  private[statuspolling] trait JesApiQuery {
+    val failedAttempts: Int
+    def requester: ActorRef
+    def genomicsInterface: Genomics
+    def withFailedAttempt: JesApiQuery
+    def backoff: Backoff
+  }
+  private object JesApiQuery {
+    // This must be a def, we want a new one each time (they're mutable! Boo!)
+    def backoff: Backoff = SimpleExponentialBackoff(1.second, 1000.seconds, 1.5d)
+  }
+  private[statuspolling] final case class JesStatusPollQuery(requester: ActorRef, run: Run, failedAttempts: Int = 0, backoff: Backoff = JesApiQuery.backoff) extends JesApiQuery {
+    override val genomicsInterface = run.genomicsInterface
+    override def withFailedAttempt = this.copy(failedAttempts = failedAttempts + 1, backoff = backoff.next)
+  }
+  private[statuspolling] final case class JesRunCreationQuery(requester: ActorRef, genomicsInterface: Genomics, rpr: RunPipelineRequest, failedAttempts: Int = 0, backoff: Backoff = JesApiQuery.backoff) extends JesApiQuery {
+    override def withFailedAttempt = this.copy(failedAttempts = failedAttempts + 1, backoff = backoff.next)
+  }
+
+  private[statuspolling] final case class JesPollingWorkBatch(workBatch: NonEmptyList[JesApiQuery])
   private[statuspolling] case object NoWorkToDo
 
   private[statuspolling] final case class RequestJesPollingWork(maxBatchSize: Int)
+
+  final case class GoogleJsonException(e: GoogleJsonError, responseHeaders: HttpHeaders) extends IOException with CromwellFatalExceptionMarker
+  final class JesApiException(e: Throwable) extends RuntimeException(e) with CromwellFatalExceptionMarker
+  final case class JesApiQueryFailed(query: JesApiQuery, cause: JesApiException)
 }
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActor.scala
index 9a0d55182..4f0aff53d 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActor.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActor.scala
@@ -3,29 +3,29 @@ package cromwell.backend.impl.jes.statuspolling
 import akka.actor.{Actor, ActorLogging, ActorRef, Props}
 import cats.data.NonEmptyList
 import com.google.api.client.googleapis.batch.BatchRequest
-import com.google.api.client.googleapis.batch.json.JsonBatchCallback
 import com.google.api.client.googleapis.json.GoogleJsonError
-import com.google.api.client.http.HttpHeaders
-import com.google.api.services.genomics.model.Operation
-import cromwell.backend.impl.jes.{JesAttributes, Run}
-import cromwell.backend.impl.jes.statuspolling.JesApiQueryManager.{JesPollingWorkBatch, JesStatusPollQuery, NoWorkToDo}
+import com.google.api.services.genomics.Genomics
+import cromwell.backend.impl.jes.statuspolling.JesApiQueryManager._
 import cromwell.backend.impl.jes.statuspolling.JesPollingActor._
 import cromwell.core.Dispatcher.BackendDispatcher
+import eu.timepit.refined.api.Refined
+import eu.timepit.refined.numeric._
 
-import scala.collection.JavaConversions._
-import scala.concurrent.{ExecutionContext, Future, Promise}
+import scala.concurrent.{ExecutionContext, Future}
 import scala.util.{Failure, Success, Try}
 import scala.concurrent.duration._
+import scala.collection.JavaConversions._
 
 /**
-  * Polls JES for status. Pipes the results back (so expect either a RunStatus or a akka.actor.Status.Failure).
+  * Sends batched requests to JES as a worker to the JesApiQueryManager
   */
-class JesPollingActor(val pollingManager: ActorRef, val qps: Int) extends Actor with ActorLogging {
+class JesPollingActor(val pollingManager: ActorRef, val qps: Int Refined Positive) extends Actor with ActorLogging
+  with StatusPolling with RunCreation {
   // The interval to delay between submitting each batch
-  lazy val batchInterval = determineBatchInterval(determineEffectiveQps(qps))
+  lazy val batchInterval = determineBatchInterval(qps)
   log.debug("JES batch polling interval is {}", batchInterval)
 
-  self ! NoWorkToDo // Starts the check-for-work cycle
+  self ! NoWorkToDo // Starts the check-for-work cycle when the actor is fully initialized.
 
   implicit val ec: ExecutionContext = context.dispatcher
 
@@ -33,58 +33,35 @@ class JesPollingActor(val pollingManager: ActorRef, val qps: Int) extends Actor
 
     case JesPollingWorkBatch(workBatch) =>
       log.debug(s"Got a polling batch with ${workBatch.tail.size + 1} requests.")
-      val batchResultFutures = handleBatch(workBatch)
-      val overallFuture = Future.sequence(batchResultFutures.toList)
-      overallFuture.andThen(interstitialRecombobulation)
+      handleBatch(workBatch).andThen(interstitialRecombobulation)
       ()
     case NoWorkToDo =>
       scheduleCheckForWork()
   }
 
-  private def handleBatch(workBatch: NonEmptyList[JesStatusPollQuery]): NonEmptyList[Future[Try[Unit]]] = {
-
-    // Assume that the auth for the first element is also able to query the remaining Runs
-    val batch: BatchRequest = createBatch(workBatch.head.run)
+  private def handleBatch(workBatch: NonEmptyList[JesApiQuery]): Future[List[Try[Unit]]] = {
+    // Assume that the auth for the first element is also good enough for everything else:
+    val batch: BatchRequest = createBatch(workBatch.head.genomicsInterface)
 
     // Create the batch:
-    val batchFutures = workBatch map { pollingRequest =>
-      val completionPromise = Promise[Try[Unit]]()
-      val resultHandler = batchResultHandler(pollingRequest.requester, completionPromise)
-      enqueueStatusPollInBatch(pollingRequest.run, batch, resultHandler)
-      completionPromise.future
+    // WARNING: These call change 'batch' as a side effect. Things might go awry if map runs items in parallel?
+    val batchFutures = workBatch map {
+      case pollingRequest: JesStatusPollQuery => enqueueStatusPollInBatch(pollingRequest, batch)
+      case runCreationRequest: JesRunCreationQuery => enqueueRunCreationInBatch(runCreationRequest, batch)
+
+      // We do the "successful Failure" thing so that the Future.sequence doesn't short-out immediately when the first one fails.
+      case other => Future.successful(Failure(new RuntimeException(s"Cannot handle ${other.getClass.getSimpleName} requests")))
     }
 
     // Execute the batch and return the map:
     runBatch(batch)
-    batchFutures
+    Future.sequence(batchFutures.toList)
   }
 
   // These are separate functions so that the tests can hook in and replace the JES-side stuff
-  private[statuspolling] def createBatch(run: Run): BatchRequest = run.genomicsInterface.batch()
-  private[statuspolling] def enqueueStatusPollInBatch(run: Run, batch: BatchRequest, resultHandler: JsonBatchCallback[Operation]) = {
-    run.getOperationCommand.queue(batch, resultHandler)
-  }
+  private[statuspolling] def createBatch(genomicsInterface: Genomics): BatchRequest = genomicsInterface.batch()
   private[statuspolling] def runBatch(batch: BatchRequest) = batch.execute()
 
-  private def batchResultHandler(originalRequester: ActorRef, completionPromise: Promise[Try[Unit]]) = new JsonBatchCallback[Operation] {
-    override def onSuccess(operation: Operation, responseHeaders: HttpHeaders): Unit = {
-      log.debug(s"Batch result onSuccess callback triggered!")
-      originalRequester ! interpretOperationStatus(operation)
-      completionPromise.trySuccess(Success(()))
-      ()
-    }
-
-    override def onFailure(e: GoogleJsonError, responseHeaders: HttpHeaders): Unit = {
-      log.debug(s"Batch request onFailure callback triggered!")
-      originalRequester ! JesPollFailed(e, responseHeaders)
-      completionPromise.trySuccess(Failure(new Exception(mkErrorString(e))))
-      ()
-    }
-  }
-
-  private[statuspolling] def mkErrorString(e: GoogleJsonError) = e.getErrors.toList.mkString(", ")
-  private[statuspolling] def interpretOperationStatus(operation: Operation) = Run.interpretOperationStatus(operation)
-
   // TODO: FSMify this actor?
   private def interstitialRecombobulation: PartialFunction[Try[List[Try[Unit]]], Unit] = {
     case Success(allSuccesses) if allSuccesses.forall(_.isSuccess) =>
@@ -93,14 +70,15 @@ class JesPollingActor(val pollingManager: ActorRef, val qps: Int) extends Actor
     case Success(someFailures) =>
       val errors = someFailures collect { case Failure(t) => t.getMessage }
       if (log.isDebugEnabled) {
-        log.warning("{} failures fetching JES statuses", errors.size)
+        log.warning("{} failures (from {} requests) fetching JES statuses", errors.size, someFailures.size)
       } else {
-        log.warning("{} failures fetching JES statuses: {}", errors.size, errors.mkString(", "))
+        log.warning("{} failures (from {} requests) fetching JES statuses: {}", errors.size, someFailures.size, errors.mkString(", "))
       }
       scheduleCheckForWork()
     case Failure(t) =>
       // NB: Should be impossible since we only ever do completionPromise.trySuccess()
-      log.error("Completion promise unexpectedly set to Failure: {}", t.getMessage)
+      val msg = "Programmer Error: Completion promise unexpectedly set to Failure: {}. Don't do this, otherwise the Future.sequence is short-circuited on the first failure"
+      log.error(msg, t.getMessage)
       scheduleCheckForWork()
   }
 
@@ -113,22 +91,11 @@ class JesPollingActor(val pollingManager: ActorRef, val qps: Int) extends Actor
     ()
   }
 
-  /**
-    * We don't want to allow non-positive QPS values. Catch these instances and replace them with a sensible default.
-    * Here we're using the default value coming from JES itself
-    */
-  private def determineEffectiveQps(qps: Int): Int = {
-    if (qps > 0) qps
-    else {
-      val defaultQps = JesAttributes.GenomicsApiDefaultQps
-      log.warning("Supplied QPS for Google Genomics API was not positive, value was {} using {} instead", qps, defaultQps)
-      defaultQps
-    }
-  }
+  private[statuspolling] def mkErrorString(e: GoogleJsonError) = e.getErrors.toList.mkString(", ")
 }
 
 object JesPollingActor {
-  def props(pollingManager: ActorRef, qps: Int) = Props(new JesPollingActor(pollingManager, qps)).withDispatcher(BackendDispatcher)
+  def props(pollingManager: ActorRef, qps: Int Refined Positive) = Props(new JesPollingActor(pollingManager, qps)).withDispatcher(BackendDispatcher)
 
   // The Batch API limits us to 100 at a time
   val MaxBatchSize = 100
@@ -141,11 +108,9 @@ object JesPollingActor {
     * Forcing the minimum value to be 1 second, for now it seems unlikely to matter and it makes testing a bit
     * easier
     */
-  def determineBatchInterval(qps: Int): FiniteDuration = {
-    val maxInterval = MaxBatchSize / qps.toDouble // Force this to be floating point in case the value is < 1
-    val interval = Math.max(maxInterval * 0.9, 1)
+  def determineBatchInterval(qps: Int Refined Positive): FiniteDuration = {
+    val maxInterval = MaxBatchSize / qps.value
+    val interval = Math.max(maxInterval / 0.9, 1).toInt
     interval.seconds
   }
-
-  final case class JesPollFailed(e: GoogleJsonError, responseHeaders: HttpHeaders)
 }
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesRunCreationClient.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesRunCreationClient.scala
new file mode 100644
index 000000000..1114e108b
--- /dev/null
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesRunCreationClient.scala
@@ -0,0 +1,46 @@
+package cromwell.backend.impl.jes.statuspolling
+
+import akka.actor.{Actor, ActorLogging, ActorRef}
+import com.google.api.services.genomics.Genomics
+import com.google.api.services.genomics.model.RunPipelineRequest
+import cromwell.backend.impl.jes.statuspolling.JesApiQueryManager.JesApiQueryFailed
+import cromwell.backend.standard.StandardAsyncJob
+
+import scala.concurrent.{Future, Promise}
+import scala.util.{Failure, Success, Try}
+
+/**
+  * I'm putting this stuff in a mixin to avoid polluting the main class.
+  *
+  * Be sure to make the main class's receive look like:
+  *   override def receive = runCreationClientReceive orElse { ... }
+  */
+trait JesRunCreationClient { this: Actor with ActorLogging =>
+
+  private var runCreationClientPromise: Option[Promise[StandardAsyncJob]] = None
+
+  val pollingActor: ActorRef
+
+  def runCreationClientReceive: Actor.Receive = {
+    case job: StandardAsyncJob =>
+      completePromise(Success(job))
+    case JesApiQueryFailed(_, e) => completePromise(Failure(e))
+  }
+
+  private def completePromise(job: Try[StandardAsyncJob]) = {
+    runCreationClientPromise foreach { _.complete(job) }
+    runCreationClientPromise = None
+  }
+
+  def runPipeline(genomicsInterface: Genomics, rpr: RunPipelineRequest): Future[StandardAsyncJob] = {
+    runCreationClientPromise match {
+      case Some(p) =>
+        p.future
+      case None =>
+        pollingActor ! JesApiQueryManager.DoCreateRun(genomicsInterface, rpr)
+        val newPromise = Promise[StandardAsyncJob]()
+        runCreationClientPromise = Option(newPromise)
+        newPromise.future
+    }
+  }
+}
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActorClient.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesStatusRequestClient.scala
similarity index 77%
rename from supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActorClient.scala
rename to supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesStatusRequestClient.scala
index 9070378c3..6b7f8ee00 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActorClient.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesStatusRequestClient.scala
@@ -1,9 +1,7 @@
 package cromwell.backend.impl.jes.statuspolling
 
-import java.io.IOException
-
 import akka.actor.{Actor, ActorLogging, ActorRef}
-import cromwell.backend.impl.jes.statuspolling.JesPollingActor.JesPollFailed
+import cromwell.backend.impl.jes.statuspolling.JesApiQueryManager.JesApiQueryFailed
 import cromwell.backend.impl.jes.{Run, RunStatus}
 
 import scala.concurrent.{Future, Promise}
@@ -15,7 +13,7 @@ import scala.util.{Failure, Success, Try}
   * Be sure to make the main class's receive look like:
   *   override def receive = pollingActorClientReceive orElse { ... }
   */
-trait JesPollingActorClient { this: Actor with ActorLogging =>
+trait JesStatusRequestClient { this: Actor with ActorLogging =>
 
   private var pollingActorClientPromise: Option[Promise[RunStatus]] = None
 
@@ -25,9 +23,9 @@ trait JesPollingActorClient { this: Actor with ActorLogging =>
     case r: RunStatus =>
       log.debug(s"Polled status received: $r")
       completePromise(Success(r))
-    case JesPollFailed(e, responseHeaders) =>
-      log.debug("JES poll failed! Sad.")
-      completePromise(Failure(new IOException(s"Google request failed: ${e.toPrettyString}")))
+    case JesApiQueryFailed(_, e) =>
+      log.debug("JES poll failed!")
+      completePromise(Failure(e))
   }
 
   private def completePromise(runStatus: Try[RunStatus]) = {
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/RunCreation.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/RunCreation.scala
new file mode 100644
index 000000000..3e6536597
--- /dev/null
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/RunCreation.scala
@@ -0,0 +1,42 @@
+package cromwell.backend.impl.jes.statuspolling
+
+import com.google.api.client.googleapis.batch.BatchRequest
+import com.google.api.client.googleapis.batch.json.JsonBatchCallback
+import com.google.api.client.googleapis.json.GoogleJsonError
+import com.google.api.client.http.HttpHeaders
+import com.google.api.services.genomics.Genomics
+import com.google.api.services.genomics.model.{Operation, RunPipelineRequest}
+import cromwell.backend.impl.jes.statuspolling.JesApiQueryManager._
+import cromwell.backend.standard.StandardAsyncJob
+
+import scala.concurrent.{Future, Promise}
+import scala.util.{Failure, Success, Try}
+
+private[statuspolling] trait RunCreation { this: JesPollingActor =>
+
+  private def runCreationResultHandler(originalRequest: JesApiQuery, completionPromise: Promise[Try[Unit]]) = new JsonBatchCallback[Operation] {
+    override def onSuccess(operation: Operation, responseHeaders: HttpHeaders): Unit = {
+      originalRequest.requester ! getJob(operation)
+      completionPromise.trySuccess(Success(()))
+      ()
+    }
+
+    override def onFailure(e: GoogleJsonError, responseHeaders: HttpHeaders): Unit = {
+      pollingManager ! JesApiQueryFailed(originalRequest, new JesApiException(GoogleJsonException(e, responseHeaders)))
+      completionPromise.trySuccess(Failure(new Exception(mkErrorString(e))))
+      ()
+    }
+  }
+
+  def enqueueRunCreationInBatch(runCreationQuery: JesRunCreationQuery, batch: BatchRequest): Future[Try[Unit]] = {
+    val completionPromise = Promise[Try[Unit]]()
+    val resultHandler = runCreationResultHandler(runCreationQuery, completionPromise)
+    addRunCreationToBatch(runCreationQuery.rpr, runCreationQuery.genomicsInterface, batch, resultHandler)
+    completionPromise.future
+  }
+
+  private def addRunCreationToBatch(rpr: RunPipelineRequest, genomicsInterface: Genomics, batch: BatchRequest, resultHandler: JsonBatchCallback[Operation]) =
+    genomicsInterface.pipelines().run(rpr).queue(batch, resultHandler)
+
+  private def getJob(operation: Operation) = StandardAsyncJob(operation.getName)
+}
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/StatusPolling.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/StatusPolling.scala
new file mode 100644
index 000000000..afd4b0673
--- /dev/null
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/StatusPolling.scala
@@ -0,0 +1,130 @@
+package cromwell.backend.impl.jes.statuspolling
+
+import java.time.OffsetDateTime
+import java.util.{ArrayList => JArrayList}
+
+import com.google.api.client.util.{ArrayMap => GArrayMap}
+import com.google.api.client.googleapis.batch.BatchRequest
+import com.google.api.client.googleapis.batch.json.JsonBatchCallback
+import com.google.api.client.googleapis.json.GoogleJsonError
+import com.google.api.client.http.HttpHeaders
+import com.google.api.services.genomics.model.Operation
+import cromwell.backend.impl.jes.RunStatus.{Failed, Initializing, Running, Success}
+import cromwell.backend.impl.jes.{Run, RunStatus}
+import cromwell.backend.impl.jes.statuspolling.JesApiQueryManager.{GoogleJsonException, JesApiException, JesApiQueryFailed, JesStatusPollQuery}
+import cromwell.core.ExecutionEvent
+
+import scala.language.postfixOps
+import scala.collection.JavaConverters._
+import scala.concurrent.{Future, Promise}
+import scala.util.{Failure, Try, Success => TrySuccess}
+
+private[statuspolling] trait StatusPolling { this: JesPollingActor =>
+
+  private def statusPollResultHandler(originalRequest: JesStatusPollQuery, completionPromise: Promise[Try[Unit]]) = new JsonBatchCallback[Operation] {
+    override def onSuccess(operation: Operation, responseHeaders: HttpHeaders): Unit = {
+      originalRequest.requester ! interpretOperationStatus(operation)
+      completionPromise.trySuccess(TrySuccess(()))
+      ()
+    }
+
+    override def onFailure(e: GoogleJsonError, responseHeaders: HttpHeaders): Unit = {
+      pollingManager ! JesApiQueryFailed(originalRequest, new JesApiException(GoogleJsonException(e, responseHeaders)))
+      completionPromise.trySuccess(Failure(new Exception(mkErrorString(e))))
+      ()
+    }
+  }
+
+  def enqueueStatusPollInBatch(pollingRequest: JesStatusPollQuery, batch: BatchRequest): Future[Try[Unit]] = {
+    val completionPromise = Promise[Try[Unit]]()
+    val resultHandler = statusPollResultHandler(pollingRequest, completionPromise)
+    addStatusPollToBatch(pollingRequest.run, batch, resultHandler)
+    completionPromise.future
+  }
+
+  // Done so that the specs can override this behaviour
+  def interpretOperationStatus(op: Operation) = StatusPolling.interpretOperationStatus(op)
+
+  def addStatusPollToBatch(run: Run, batch: BatchRequest, resultHandler: JsonBatchCallback[Operation]) =
+    run.getOperationCommand.queue(batch, resultHandler)
+}
+
+private[statuspolling] object StatusPolling {
+
+  private val AcceptableEvents = Set("start", "pulling-image", "localizing-files", "running-docker", "delocalizing-files", "ok", "fail", "start-shutdown", "preempted")
+
+  implicit class RunOperationExtension(val operation: Operation) extends AnyVal {
+    def hasStarted = operation.getMetadata.asScala.get("startTime") isDefined
+  }
+
+  def interpretOperationStatus(op: Operation): RunStatus = {
+    require(op != null, "Operation must not be null.")
+    try {
+      if (op.getDone) {
+        lazy val eventList = getEventList(op)
+        lazy val computeEngineOption = for {
+          runtimeMetadata <- op.getMetadata.asScala.get("runtimeMetadata")
+          computeEngine <- runtimeMetadata.asInstanceOf[GArrayMap[String, Object]].asScala.get("computeEngine")
+        } yield computeEngine.asInstanceOf[GArrayMap[String, String]].asScala
+        lazy val machineType = computeEngineOption.flatMap(_.get("machineType"))
+        lazy val instanceName = computeEngineOption.flatMap(_.get("instanceName"))
+        lazy val zone = computeEngineOption.flatMap(_.get("zone"))
+
+        // If there's an error, generate a Failed status. Otherwise, we were successful!
+        Option(op.getError) match {
+          case None => Success(eventList, machineType, zone, instanceName)
+          case Some(error) =>
+            Failed(error.getCode, Option(error.getMessage), eventList, machineType, zone, instanceName)
+        }
+      } else if (op.hasStarted) {
+        Running
+      } else {
+        Initializing
+      }
+    } catch {
+      case npe: NullPointerException =>
+        throw new RuntimeException(s"Caught NPE while processing operation ${op.getName}: $op", npe)
+    }
+  }
+
+  def getEventList(op: Operation): Seq[ExecutionEvent] = {
+    val metadata = op.getMetadata.asScala.toMap
+
+    val starterEvents: Seq[ExecutionEvent] = Seq(
+      eventIfExists("createTime", metadata, "waiting for quota"),
+      eventIfExists("startTime", metadata, "initializing VM")).flatten
+
+    val eventsList = for {
+      events <- metadata.get("events").toSeq
+      entry <- events.asInstanceOf[JArrayList[GArrayMap[String, String]]].asScala
+    } yield ExecutionEvent(entry.get("description"), OffsetDateTime.parse(entry.get("startTime")))
+
+    val filteredEventsList: Seq[ExecutionEvent] = eventsList filter { i => AcceptableEvents.contains(i.name) }
+
+    // A little bit ugly... the endTime of the jes operation can actually be before the final "event" time, due to differences
+    // in the reported precision. As a result, we have to make sure it all lines up nicely:
+    val finalEvent = getCromwellPollIntervalEvent(metadata, filteredEventsList)
+
+    starterEvents ++ filteredEventsList :+ finalEvent
+  }
+
+  private def getCromwellPollIntervalEvent(metadata: Map[String, AnyRef], eventsList: Seq[ExecutionEvent]) = {
+    {
+      val jesReportedEndTime = eventIfExists("endTime", metadata, "cromwell poll interval")
+      val finalEventsListTime = if (eventsList.nonEmpty) Some(eventsList.last.offsetDateTime) else None
+
+      (jesReportedEndTime, finalEventsListTime) match {
+        case (Some(jesEndTime), Some(finalEventTime)) =>
+          if (jesEndTime.offsetDateTime isAfter finalEventTime) jesEndTime else jesEndTime.copy(offsetDateTime = finalEventTime)
+        case (Some(jesEndTime), None) => jesEndTime
+        case (None, Some(finalEventTime)) => ExecutionEvent("cromwell poll interval", finalEventTime)
+        case (None, None) =>
+          throw new IllegalArgumentException("Both jesReportedEndTime and finalEventsListTime were None.")
+      }
+    }
+  }
+
+  private def eventIfExists(name: String, metadata: Map[String, AnyRef], eventName: String): Option[ExecutionEvent] = {
+    metadata.get(name) map { time => ExecutionEvent(eventName, OffsetDateTime.parse(time.toString)) }
+  }
+}
diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala
index eb2082e69..eb5272cbd 100644
--- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala
+++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala
@@ -148,9 +148,9 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
                              promise: Promise[BackendJobExecutionResponse],
                              jesSingletonActor: ActorRef): ActorRef = {
 
-    // Mock/stub out the bits that would reach out to JES.
-    val run = mock[Run]
-    val handle = new JesPendingExecutionHandle(jobDescriptor, StandardAsyncJob(run.runId), Option(run), None)
+    val job = StandardAsyncJob(UUID.randomUUID().toString)
+    val run = Run(job, null)
+    val handle = new JesPendingExecutionHandle(jobDescriptor, run.job, Option(run), None)
 
     class ExecuteOrRecoverActor extends TestableJesJobExecutionActor(jobDescriptor, promise, jesConfiguration, jesSingletonActor = jesSingletonActor) {
       override def executeOrRecover(mode: ExecutionMode)(implicit ec: ExecutionContext): Future[ExecutionHandle] = Future.successful(handle)
@@ -220,7 +220,8 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
   it should "not restart 2 of 1 unexpected shutdowns without another preemptible VM" in {
     val actorRef = buildPreemptibleTestActorRef(2, 1)
     val jesBackend = actorRef.underlyingActor
-    val handle = mock[JesPendingExecutionHandle]
+    val runId = StandardAsyncJob(UUID.randomUUID().toString)
+    val handle = new JesPendingExecutionHandle(null, runId, None, None)
 
     val failedStatus = Failed(10, Option("14: VM XXX shut down unexpectedly."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance"))
     val executionResult = jesBackend.handleExecutionResult(failedStatus, handle)
@@ -232,7 +233,8 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
   it should "restart 1 of 1 unexpected shutdowns without another preemptible VM" in {
     val actorRef = buildPreemptibleTestActorRef(1, 1)
     val jesBackend = actorRef.underlyingActor
-    val handle = mock[JesPendingExecutionHandle]
+    val runId = StandardAsyncJob(UUID.randomUUID().toString)
+    val handle = new JesPendingExecutionHandle(null, runId, None, None)
 
     val failedStatus = Failed(10, Option("14: VM XXX shut down unexpectedly."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance"))
     val executionResult = jesBackend.handleExecutionResult(failedStatus, handle)
@@ -247,7 +249,8 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
   it should "restart 1 of 2 unexpected shutdowns with another preemptible VM" in {
     val actorRef = buildPreemptibleTestActorRef(1, 2)
     val jesBackend = actorRef.underlyingActor
-    val handle = mock[JesPendingExecutionHandle]
+    val runId = StandardAsyncJob(UUID.randomUUID().toString)
+    val handle = new JesPendingExecutionHandle(null, runId, None, None)
 
     val failedStatus = Failed(10, Option("14: VM XXX shut down unexpectedly."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance"))
     val executionResult = jesBackend.handleExecutionResult(failedStatus, handle)
@@ -262,7 +265,8 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
   it should "handle Failure Status for various errors" in {
     val actorRef = buildPreemptibleTestActorRef(1, 1)
     val jesBackend = actorRef.underlyingActor
-    val handle = mock[JesPendingExecutionHandle]
+    val runId = StandardAsyncJob(UUID.randomUUID().toString)
+    val handle = new JesPendingExecutionHandle(null, runId, None, None)
 
     def checkFailedResult(errorCode: Int, errorMessage: Option[String]): ExecutionHandle = {
       val failed =
diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/statuspolling/JesApiQueryManagerSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/statuspolling/JesApiQueryManagerSpec.scala
index c7434419a..2564dfdfd 100644
--- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/statuspolling/JesApiQueryManagerSpec.scala
+++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/statuspolling/JesApiQueryManagerSpec.scala
@@ -9,7 +9,11 @@ import org.scalatest.{FlatSpecLike, Matchers}
 import scala.concurrent.duration._
 import akka.testkit._
 import JesApiQueryManagerSpec._
+import cromwell.backend.impl.jes.statuspolling.JesApiQueryManager.JesStatusPollQuery
+import cromwell.backend.standard.StandardAsyncJob
 import cromwell.util.AkkaTestUtil
+import eu.timepit.refined.api.Refined
+import eu.timepit.refined.numeric._
 import org.scalatest.concurrent.Eventually
 
 import scala.collection.immutable.Queue
@@ -35,7 +39,7 @@ class JesApiQueryManagerSpec extends TestKitSuite("JesApiQueryManagerSpec") with
 
     // Send a few status poll requests:
     statusRequesters foreach { case (index, probe) =>
-      jaqmActor.tell(msg = JesApiQueryManager.DoPoll(Run(index.toString, null)), sender = probe.ref)
+      jaqmActor.tell(msg = JesApiQueryManager.DoPoll(Run(StandardAsyncJob(index.toString), null)), sender = probe.ref)
     }
 
     // Should have no messages to the actual statusPoller yet:
@@ -52,8 +56,9 @@ class JesApiQueryManagerSpec extends TestKitSuite("JesApiQueryManagerSpec") with
           val zippedWithRequesters = workBatch.toList.zip(requesters)
           zippedWithRequesters foreach { case (pollQuery, (index, testProbe)) =>
             pollQuery.requester should be(testProbe.ref)
-            pollQuery.run.runId should be(index.toString)
+            pollQuery.asInstanceOf[JesStatusPollQuery].run.job should be(StandardAsyncJob(index.toString))
           }
+        case other => fail(s"Unexpected message: $other")
       }
     }
 
@@ -82,7 +87,7 @@ class JesApiQueryManagerSpec extends TestKitSuite("JesApiQueryManagerSpec") with
 
        // Send a few status poll requests:
       BatchSize indexedTimes { index =>
-        jaqmActor.tell(msg = JesApiQueryManager.DoPoll(Run(index.toString, null)), sender = emptyActor)
+        jaqmActor.tell(msg = JesApiQueryManager.DoPoll(Run(StandardAsyncJob(index.toString), null)), sender = emptyActor)
       }
 
       jaqmActor.tell(msg = JesApiQueryManager.RequestJesPollingWork(BatchSize), sender = statusPoller1)
@@ -108,7 +113,7 @@ object JesApiQueryManagerSpec {
 /**
   * This test class allows us to hook into the JesApiQueryManager's makeStatusPoller and provide our own TestProbes instead
   */
-class TestJesApiQueryManager(qps: Int, statusPollerProbes: ActorRef*) extends JesApiQueryManager(qps) {
+class TestJesApiQueryManager(qps: Int Refined Positive, statusPollerProbes: ActorRef*) extends JesApiQueryManager(qps) {
   var testProbes: Queue[ActorRef] = _
   var testPollerCreations: Int = _
 
@@ -117,7 +122,7 @@ class TestJesApiQueryManager(qps: Int, statusPollerProbes: ActorRef*) extends Je
     testPollerCreations = 0
   }
 
-  override private[statuspolling] def makeStatusPoller(): ActorRef = {
+  override private[statuspolling] def makeWorkerActor(): ActorRef = {
     // Initialize the queue, if necessary:
     if (testProbes == null) {
       init()
diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActorSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActorSpec.scala
index 05b655c8b..2b13ea4bc 100644
--- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActorSpec.scala
+++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActorSpec.scala
@@ -12,11 +12,13 @@ import cats.data.NonEmptyList
 import com.google.api.client.googleapis.batch.BatchRequest
 import com.google.api.client.googleapis.batch.json.JsonBatchCallback
 import com.google.api.client.googleapis.json.GoogleJsonError
+import com.google.api.services.genomics.Genomics
 import com.google.api.services.genomics.model.Operation
 import cromwell.backend.impl.jes.{JesConfiguration, Run, RunStatus}
-import cromwell.backend.impl.jes.statuspolling.JesApiQueryManager.JesStatusPollQuery
-import cromwell.backend.impl.jes.statuspolling.JesPollingActor.JesPollFailed
+import cromwell.backend.impl.jes.statuspolling.JesApiQueryManager.{JesApiException, JesApiQueryFailed, JesStatusPollQuery, RequestJesPollingWork}
 import cromwell.backend.impl.jes.statuspolling.TestJesPollingActor.{CallbackFailure, CallbackSuccess, JesBatchCallbackResponse}
+import eu.timepit.refined.api.Refined
+import eu.timepit.refined.numeric.Positive
 import org.specs2.mock.Mockito
 
 import scala.collection.immutable.Queue
@@ -36,8 +38,9 @@ class JesPollingActorSpec extends TestKitSuite("JesPollingActor") with FlatSpecL
   var jpActor: TestActorRef[TestJesPollingActor] = _
 
   it should "correctly calculate batch intervals" in {
-    JesPollingActor.determineBatchInterval(10) shouldBe 9.seconds
-    JesPollingActor.determineBatchInterval(100) shouldBe 1.second
+    import eu.timepit.refined.auto._
+    JesPollingActor.determineBatchInterval(10) should be(11.seconds)
+    JesPollingActor.determineBatchInterval(100000) shouldBe 1.seconds
   }
 
   it should "query for work and wait for a reply" in {
@@ -45,17 +48,17 @@ class JesPollingActorSpec extends TestKitSuite("JesPollingActor") with FlatSpecL
     managerProbe.expectNoMsg(max = AwaitAlmostNothing)
   }
 
-  it should "respond directly to requesters with various run statuses" in {
+  it should "respond correctly with various run statuses" in {
     managerProbe.expectMsgClass(max = TestExecutionTimeout, c = classOf[JesApiQueryManager.RequestJesPollingWork])
 
     val requester1 = TestProbe()
-    val query1 = JesStatusPollQuery(requester1.ref, mock[Run])
+    val query1 = JesStatusPollQuery(requester1.ref, Run(null, null))
     val requester2 = TestProbe()
-    val query2 = JesStatusPollQuery(requester2.ref, mock[Run])
+    val query2 = JesStatusPollQuery(requester2.ref, Run(null, null))
     val requester3 = TestProbe()
-    val query3 = JesStatusPollQuery(requester3.ref, mock[Run])
+    val query3 = JesStatusPollQuery(requester3.ref, Run(null, null))
 
-    // For two requests the callback succeeds (first with RunStatus.Success, then RunStatus.Failed). The third callback fails:
+    // For two requests the callback succeeds (first with RunStatus.Success, then RunStatus.Failed). The third callback fails (simulating a network timeout, for example):
     jpActor.underlyingActor.callbackResponses :+= CallbackSuccess
     jpActor.underlyingActor.callbackResponses :+= CallbackSuccess
     jpActor.underlyingActor.callbackResponses :+= CallbackFailure
@@ -77,10 +80,16 @@ class JesPollingActorSpec extends TestKitSuite("JesPollingActor") with FlatSpecL
 
     requester1.expectMsg(successStatus)
     requester2.expectMsg(failureStatus)
-    requester3.expectMsgClass(classOf[JesPollFailed])
-
-    // And the poller is done! Now the manager should now have (only one) request for more work:
-    managerProbe.expectMsgClass(max = TestExecutionTimeout, c = classOf[JesApiQueryManager.RequestJesPollingWork])
+    requester3.expectNoMsg(max = AwaitAlmostNothing)
+
+    // Requester3 expected nothing... Instead, the manager expects an API failure notification and then a request for more work:
+    managerProbe.expectMsgPF(TestExecutionTimeout) {
+      case failure: JesApiQueryFailed =>
+        if (!failure.cause.isInstanceOf[JesApiException]) fail("Unexpected failure cause class: " + failure.cause.getClass.getSimpleName)
+        if (failure.query != query2 && failure.query != query3) fail("Unexpected query caused failure: " + failure.query)
+    }
+    managerProbe.expectMsg(RequestJesPollingWork(JesPollingActor.MaxBatchSize))
+    managerProbe.expectNoMsg(max = AwaitAlmostNothing)
   }
 
   before {
@@ -89,20 +98,12 @@ class JesPollingActorSpec extends TestKitSuite("JesPollingActor") with FlatSpecL
   }
 }
 
-object JesPollingActorSpec extends Mockito {
-  def mockRun(runId: String): Run = {
-    val run = mock[Run]
-    run.runId returns runId
-    run
-  }
-}
-
 /**
   * Testable JES polling actor.
   * - Mocks out the methods which actually call out to JES, and allows the callbacks to be triggered in a testable way
   * - Also waits a **lot** less time before polls!
   */
-class TestJesPollingActor(manager: ActorRef, qps: Int) extends JesPollingActor(manager, qps) with Mockito {
+class TestJesPollingActor(manager: ActorRef, qps: Int Refined Positive) extends JesPollingActor(manager, qps) with Mockito {
 
   override lazy val batchInterval = 10.milliseconds
 
@@ -111,7 +112,7 @@ class TestJesPollingActor(manager: ActorRef, qps: Int) extends JesPollingActor(m
   var callbackResponses: Queue[JesBatchCallbackResponse] = Queue.empty
   var runBatchRequested: Boolean = false
 
-  override private[statuspolling] def createBatch(run: Run): BatchRequest = null
+  override private[statuspolling] def createBatch(genomicsInterface: Genomics): BatchRequest = null
   override private[statuspolling] def runBatch(batch: BatchRequest): Unit = runBatchRequested = true
 
   def executeBatch(): Unit = {
@@ -122,13 +123,13 @@ class TestJesPollingActor(manager: ActorRef, qps: Int) extends JesPollingActor(m
         handler.onFailure(error, null)
     }}
   }
-  override private[statuspolling] def enqueueStatusPollInBatch(run: Run, batch: BatchRequest, resultHandler: JsonBatchCallback[Operation]): Unit = resultHandlers :+= resultHandler
-  override private[statuspolling] def interpretOperationStatus(operation: Operation): RunStatus = {
+  override def addStatusPollToBatch(run: Run, batch: BatchRequest, resultHandler: JsonBatchCallback[Operation]): Unit = resultHandlers :+= resultHandler
+  override def interpretOperationStatus(operation: Operation): RunStatus = {
     val (status, newQueue) = operationStatusResponses.dequeue
     operationStatusResponses = newQueue
     status
   }
-  override private[statuspolling] def mkErrorString(e: GoogleJsonError) = "NA"
+  override def mkErrorString(e: GoogleJsonError) = "NA"
 }
 
 object TestJesPollingActor {
diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/RunSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/statuspolling/StatusPollingSpec.scala
similarity index 87%
rename from supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/RunSpec.scala
rename to supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/statuspolling/StatusPollingSpec.scala
index 7deec7f84..ffd7baf04 100644
--- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/RunSpec.scala
+++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/statuspolling/StatusPollingSpec.scala
@@ -1,4 +1,4 @@
-package cromwell.backend.impl.jes
+package cromwell.backend.impl.jes.statuspolling
 
 import java.time.OffsetDateTime
 import java.util
@@ -12,26 +12,26 @@ import org.specs2.mock.{Mockito => MockitoTrait}
 
 import scala.collection.JavaConverters._
 
-class RunSpec extends FlatSpec with Matchers with MockitoTrait {
-  behavior of "JES Run"
+class StatusPollingSpec extends FlatSpec with Matchers with MockitoTrait {
+  behavior of "JES Status polling"
 
   it should "parse events from Operation metadata" in {
     val op: Operation = new Operation()
     op.setMetadata(eventsMetadata.asJava)
 
-    val list = Run.getEventList(op)
+    val list = StatusPolling.getEventList(op)
     list should contain theSameElementsAs eventsExpected
   }
 
   it should "require operation be non-null" in {
-    val exception = intercept[RuntimeException](Run.interpretOperationStatus(null))
+    val exception = intercept[RuntimeException](StatusPolling.interpretOperationStatus(null))
     exception.getMessage should be("requirement failed: Operation must not be null.")
   }
 
   it should "catch and wrap null pointer exceptions in an empty operation" in {
     val op = new Operation()
 
-    val exception = intercept[RuntimeException](Run.interpretOperationStatus(op))
+    val exception = intercept[RuntimeException](StatusPolling.interpretOperationStatus(op))
     exception.getMessage should be("Caught NPE while processing operation null: {}")
   }
 
@@ -39,7 +39,7 @@ class RunSpec extends FlatSpec with Matchers with MockitoTrait {
     val op = new Operation()
     op.setName("my/customName")
 
-    val exception = intercept[RuntimeException](Run.interpretOperationStatus(op))
+    val exception = intercept[RuntimeException](StatusPolling.interpretOperationStatus(op))
     exception.getMessage should be("Caught NPE while processing operation my/customName: {name=my/customName}")
   }
 
@@ -49,7 +49,7 @@ class RunSpec extends FlatSpec with Matchers with MockitoTrait {
     op.setDone(true)
     op.setMetadata(eventsMetadata.asJava)
 
-    val runStatus = Run.interpretOperationStatus(op)
+    val runStatus = StatusPolling.interpretOperationStatus(op)
 
     runStatus should be(a[Success])
 
@@ -70,7 +70,7 @@ class RunSpec extends FlatSpec with Matchers with MockitoTrait {
     val metadata = eventsMetadata ++ Map("runtimeMetadata" -> runtimeMetadata)
     op.setMetadata(metadata.asJava)
 
-    val runStatus = Run.interpretOperationStatus(op)
+    val runStatus = StatusPolling.interpretOperationStatus(op)
     runStatus should be(a[Success])
 
     val success = runStatus.asInstanceOf[Success]
@@ -93,7 +93,7 @@ class RunSpec extends FlatSpec with Matchers with MockitoTrait {
     val metadata = eventsMetadata ++ Map("runtimeMetadata" -> runtimeMetadata)
     op.setMetadata(metadata.asJava)
 
-    val runStatus = Run.interpretOperationStatus(op)
+    val runStatus = StatusPolling.interpretOperationStatus(op)
     runStatus should be(a[Success])
 
     val success = runStatus.asInstanceOf[Success]
@@ -117,7 +117,7 @@ class RunSpec extends FlatSpec with Matchers with MockitoTrait {
     val metadata = eventsMetadata ++ Map("runtimeMetadata" -> runtimeMetadata)
     op.setMetadata(metadata.asJava)
 
-    val runStatus = Run.interpretOperationStatus(op)
+    val runStatus = StatusPolling.interpretOperationStatus(op)
 
     runStatus should be(a[Success])
 
@@ -144,7 +144,7 @@ class RunSpec extends FlatSpec with Matchers with MockitoTrait {
     val metadata = eventsMetadata ++ Map("runtimeMetadata" -> runtimeMetadata)
     op.setMetadata(metadata.asJava)
 
-    val runStatus = Run.interpretOperationStatus(op)
+    val runStatus = StatusPolling.interpretOperationStatus(op)
 
     runStatus should be(a[Success])
 

From 006b386002c6bc6a438798135e9f7588fb35928a Mon Sep 17 00:00:00 2001
From: Khalid Shakir <kshakir@broadinstitute.org>
Date: Wed, 1 Mar 2017 18:40:52 -0500
Subject: [PATCH 80/84] Don't fail Travis JES for external users.

---
 .travis.yml                      |  2 +-
 src/bin/travis/testCentaurJes.sh | 11 +++++++++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 96e5ea4eb..1be54097a 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -16,7 +16,7 @@ before_cache:
   - find $HOME/.ivy2 -name "ivydata-*.properties" -delete
   - find $HOME/.sbt -name "*.lock" -delete
 before_install:
-  - openssl aes-256-cbc -K "$encrypted_5ebd3ff04788_key" -iv "$encrypted_5ebd3ff04788_iv" -in src/bin/travis/resources/jesConf.tar.enc -out jesConf.tar -d
+  - openssl aes-256-cbc -K "$encrypted_5ebd3ff04788_key" -iv "$encrypted_5ebd3ff04788_iv" -in src/bin/travis/resources/jesConf.tar.enc -out jesConf.tar -d || true
 env:
   global:
     - CENTAUR_BRANCH=develop
diff --git a/src/bin/travis/testCentaurJes.sh b/src/bin/travis/testCentaurJes.sh
index 711e69512..98cdbb33d 100755
--- a/src/bin/travis/testCentaurJes.sh
+++ b/src/bin/travis/testCentaurJes.sh
@@ -2,6 +2,17 @@
 
 set -e
 
+if [ "$TRAVIS_SECURE_ENV_VARS" = "false" ]; then
+    echo "************************************************************************************************"
+    echo "************************************************************************************************"
+    echo "**                                                                                            **"
+    echo "**  WARNING: Encrypted keys are unavailable to automatically test JES with centaur. Exiting.  **"
+    echo "**                                                                                            **"
+    echo "************************************************************************************************"
+    echo "************************************************************************************************"
+    exit 0
+fi
+
 removeCromwellJar() {
     gsutil rm "${JAR_GCS_PATH}" || true
 }

From 560b648f37a3c5ac26a126d9c6e6755dfb6ef546 Mon Sep 17 00:00:00 2001
From: Khalid Shakir <kshakir@broadinstitute.org>
Date: Sun, 26 Feb 2017 11:56:09 -0500
Subject: [PATCH 81/84] Config backend wasn't dockerizing script paths.

---
 core/src/main/resources/reference.conf             |  9 ++++++++-
 .../sfs/config/ConfigAsyncJobExecutionActor.scala  | 23 +++++++++++++---------
 2 files changed, 22 insertions(+), 10 deletions(-)

diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf
index 873380b0b..b3af0d63d 100644
--- a/core/src/main/resources/reference.conf
+++ b/core/src/main/resources/reference.conf
@@ -179,7 +179,14 @@ backend {
         String? docker_user
         """
         submit = "/bin/bash ${script}"
-        submit-docker = """docker run --rm ${"--user " + docker_user} -v ${cwd}:${docker_cwd} -i ${docker} /bin/bash ${docker_cwd}/execution/script"""
+        submit-docker = """
+        docker run \
+          --rm -i \
+          ${"--user " + docker_user} \
+          -v ${cwd}:${docker_cwd} \
+          ${docker} \
+          /bin/bash ${script}
+        """
 
         # Root directory where Cromwell writes job results.  This directory must be
         # visible and writeable by the Cromwell process as well as the jobs that Cromwell
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigAsyncJobExecutionActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigAsyncJobExecutionActor.scala
index e2eaeaee3..399532146 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigAsyncJobExecutionActor.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigAsyncJobExecutionActor.scala
@@ -29,7 +29,7 @@ sealed trait ConfigAsyncJobExecutionActor extends SharedFileSystemAsyncJobExecut
     */
   override lazy val processArgs: SharedFileSystemCommand = {
     val submitScript = jobPaths.script.plusExt("submit")
-    val submitInputs = standardInputs ++ dockerInputs ++ runtimeAttributeInputs
+    val submitInputs = standardInputs ++ dockerizedInputs ++ runtimeAttributeInputs
     val submitTaskName = if (isDockerRun) SubmitDockerTask else SubmitTask
     writeTaskScript(submitScript, submitTaskName, submitInputs)
     SharedFileSystemCommand("/bin/bash", submitScript)
@@ -64,23 +64,28 @@ sealed trait ConfigAsyncJobExecutionActor extends SharedFileSystemAsyncJobExecut
   private lazy val standardInputs: WorkflowCoercedInputs = {
     Map(
       JobNameInput -> WdlString(jobName),
-      CwdInput -> WdlString(jobPaths.callRoot.pathAsString),
-      StdoutInput -> WdlString(jobPaths.stdout.pathAsString),
-      StderrInput -> WdlString(jobPaths.stderr.pathAsString),
-      ScriptInput -> WdlString(jobPaths.script.pathAsString)
+      CwdInput -> WdlString(jobPaths.callRoot.pathAsString)
     )
   }
 
   /**
-    * Extra arguments if this is a submit-docker command, or Map.empty.
+    * The inputs that are not specified by the config, that will be passed into a command for either submit or
+    * submit-docker.
     */
-  private lazy val dockerInputs: WorkflowCoercedInputs = {
+  private lazy val dockerizedInputs: WorkflowCoercedInputs = {
     if (isDockerRun) {
       Map(
-        DockerCwdInput -> WdlString(jobPathsWithDocker.callDockerRoot.pathAsString)
+        DockerCwdInput -> WdlString(jobPathsWithDocker.callDockerRoot.pathAsString),
+        StdoutInput -> WdlString(jobPathsWithDocker.toDockerPath(jobPaths.stdout).pathAsString),
+        StderrInput -> WdlString(jobPathsWithDocker.toDockerPath(jobPaths.stderr).pathAsString),
+        ScriptInput -> WdlString(jobPathsWithDocker.toDockerPath(jobPaths.script).pathAsString)
       )
     } else {
-      Map.empty
+      Map(
+        StdoutInput -> WdlString(jobPaths.stdout.pathAsString),
+        StderrInput -> WdlString(jobPaths.stderr.pathAsString),
+        ScriptInput -> WdlString(jobPaths.script.pathAsString)
+      )
     }
   }
 

From fac784dd4078b8cc12fb4ca6c9abdbb05072990b Mon Sep 17 00:00:00 2001
From: Conrad <conrad.leonard@hotmail.com>
Date: Sat, 4 Mar 2017 03:28:08 +1000
Subject: [PATCH 82/84] Use mktemp to make temporary directory Fixes #2047
 (#2049)

* * use mktemp to make the temporary directory

* * more portable invocation of mktemp, works on OSX also
---
 .../cromwell/backend/standard/StandardAsyncExecutionActor.scala    | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala b/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala
index 0d5b73684..e6e257e2b 100644
--- a/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala
+++ b/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala
@@ -170,16 +170,15 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
     jobLogger.info(s"`$instantiatedCommand`")
 
     val cwd = commandDirectory
-    val tmpDir = cwd./("tmp")
     val rcPath = cwd./(jobPaths.returnCodeFilename)
     val rcTmpPath = rcPath.plusExt("tmp")
 
     val globFiles = backendEngineFunctions.findGlobOutputs(call, jobDescriptor)
 
     s"""|#!/bin/bash
-        |export _JAVA_OPTIONS=-Djava.io.tmpdir=$tmpDir
-        |export TMPDIR=$tmpDir
-        |mkdir -p $tmpDir
+        |tmpDir=$$(mktemp -d $cwd/tmp.XXXXXX) 
+        |export _JAVA_OPTIONS=-Djava.io.tmpdir=$$tmpDir
+        |export TMPDIR=$$tmpDir
         |$commandScriptPreamble
         |(
         |cd $cwd

From 2e53c6d0a14632b94845ecb0dec70363982194a5 Mon Sep 17 00:00:00 2001
From: Ruchi <rmunshi@broadinstitute.org>
Date: Fri, 3 Mar 2017 16:11:42 -0500
Subject: [PATCH 83/84] update readme examples and list external contributors
 (#2051)

---
 CHANGELOG.md | 4 +++-
 README.md    | 6 +++++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index a8791a696..f388b4139 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,8 @@
 
 ## 25
 
+### External Contributors
+* A special thank you to @adamstruck, @antonkulaga and @delocalizer for their contributions to Cromwell.
 ### Breaking Changes
 
 * Metadata keys for call caching are changed. All call caching keys are now in a `callCaching` stanza. `Call cache read result` has moved here and is now `result`. The `allowResultReuse` and `effectiveCallCachingMode` have moved here. The `hit` boolean is a simple indication of whether or not it was a hit, with no additional information. An example using the new format is:
@@ -44,7 +46,7 @@ resulted in strings prepended with `file:///path/to/file`. Now absolute file pat
   See https://github.com/broadinstitute/cromwell#call-caching-docker-tags for more details. 
 * Added docker hash lookup. Cromwell will try to lookup the hash for a docker image with a floating tag, and use that hash when executing the job.
   This will be reflected in the metadata where the docker runtime attribute will contains the hash that was used.
-  If Cromwell is unable to lookup the hash hash, the job will be run with the original user defined floating tag.
+  If Cromwell is unable to lookup the docker hash, the job will be run with the original user defined floating tag.
   Cromwell is currently able to lookup public and private docker hashes for images on Docker Hub and Google Container Engine for job running on the JES backend.
   For other backends, cromwell is able to lookup public docker hashes for Docker Hub and Google Container Engine.
   See https://github.com/broadinstitute/cromwell#call-caching-docker-tags for more details. 
diff --git a/README.md b/README.md
index c4e7672bb..1ac9cfd76 100644
--- a/README.md
+++ b/README.md
@@ -425,6 +425,7 @@ The configuration file is in [Hocon](https://github.com/typesafehub/config/blob/
 webservice {
   port = 8000
   interface = 0.0.0.0
+  binding-timeout = 5s
   instance.name = "reference"
 }
 ```
@@ -434,6 +435,7 @@ Or, alternatively, as dot-separated values:
 ```hocon
 webservice.port = 8000
 webservice.interface = 0.0.0.0
+webservice.binding-timeout = 5s
 webservice.instance.name = "reference"
 ```
 
@@ -2815,7 +2817,9 @@ refresh interval will default to 2 seconds.  To turn off metadata summary refres
 ```
 services {
   MetadataService {
-    metadata-summary-refresh-interval = "10 seconds"
+    config {
+      metadata-summary-refresh-interval = "10 seconds"
+    }
   }
 }
 ```

From 0e9b8bbe00fce3114861f41afcf3b0d18b3b1426 Mon Sep 17 00:00:00 2001
From: Ruchi Munshi <rmunshi@broadinstitute.org>
Date: Fri, 3 Mar 2017 17:03:30 -0500
Subject: [PATCH 84/84] Update Cromwell to use non-SNAP versions for
 Lenthall/WDL4s

---
 project/Dependencies.scala | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/project/Dependencies.scala b/project/Dependencies.scala
index 83fa0c1d1..f1e3b83e2 100644
--- a/project/Dependencies.scala
+++ b/project/Dependencies.scala
@@ -1,8 +1,8 @@
 import sbt._
 
 object Dependencies {
-  lazy val lenthallV = "0.21-e1b7822-SNAP"
-  lazy val wdl4sV = "0.10-785f16a-SNAP"
+  lazy val lenthallV = "0.21"
+  lazy val wdl4sV = "0.10"
   lazy val sprayV = "1.3.3"
   /*
   spray-json is an independent project from the "spray suite"