diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml
index b15d7f2ad..de31e99ed 100644
--- a/.idea/inspectionProfiles/Project_Default.xml
+++ b/.idea/inspectionProfiles/Project_Default.xml
@@ -3,4 +3,4 @@
     <option name="myName" value="Project Default" />
     <inspection_tool class="TypeAnnotation" enabled="false" level="WARNING" enabled_by_default="false" />
   </profile>
-</component>
\ No newline at end of file
+</component>
diff --git a/.pullapprove.yml b/.pullapprove.yml
index d9d70c225..c26c94ba2 100644
--- a/.pullapprove.yml
+++ b/.pullapprove.yml
@@ -1,18 +1,28 @@
-approve_by_comment: true
-approve_regex: ':\+1:'
-reset_on_push: false
-author_approval: ignored
-reviewers:
+# enabling version 2 turns github reviews on by default
+version: 2
+group_defaults:
+  approve_by_comment:
+    enabled: true
+    approve_regex: ':\+1:'
+  reset_on_push:
+    enabled: false
+groups:
+  reviewers:
     required: 2
-    members:
+    github_reviews:
+      enabled: true
+    author_approval:
+      ignored: true
+    users:
         - Horneth
         - cjllanwarne
         - francares
         - gauravs90
         - geoffjentry
-        - jainh
+        - jsotobroad
         - katevoss
         - kcibul
         - kshakir
         - mcovarr
         - ruchim
+        - danbills
diff --git a/.travis.yml b/.travis.yml
index 1be54097a..7b8dfaa2f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,8 +1,10 @@
 sudo: required
 dist: trusty
+services:
+  - docker
 language: scala
 scala:
-  - 2.11.8
+  - 2.12.2
 jdk:
   - oraclejdk8
 cache:
@@ -16,10 +18,12 @@ before_cache:
   - find $HOME/.ivy2 -name "ivydata-*.properties" -delete
   - find $HOME/.sbt -name "*.lock" -delete
 before_install:
-  - openssl aes-256-cbc -K "$encrypted_5ebd3ff04788_key" -iv "$encrypted_5ebd3ff04788_iv" -in src/bin/travis/resources/jesConf.tar.enc -out jesConf.tar -d || true
+  # https://github.com/travis-ci/travis-ci/issues/7940#issuecomment-310759657
+  - sudo rm -f /etc/boto.cfg
 env:
   global:
     - CENTAUR_BRANCH=develop
+    - INTEGRATION_TESTS_DIR=src/main/resources/integrationTestCases
   matrix:
     # Setting this variable twice will cause the 'script' section to run twice with the respective env var invoked
     - BUILD_TYPE=sbt
@@ -36,3 +40,12 @@ deploy:
   script: src/bin/travis/publishRelease.sh
   on:
     tags: true
+notifications:
+  slack:
+    rooms:
+      - secure: B5KYcnhk/ujAUWlHsjzP7ROLm6MtYhaGikdYf6JYINovhMbVKnZCTlZEy7rqT3L2T5uJ25iefD500VQGk1Gn7puQ1sNq50wqjzQaj20PWEiBwoWalcV/nKBcQx1TyFT13LJv8fbFnVPxFCkC3YXoHedx8qAhDs8GH/tT5J8XOC8=
+    template:
+      - "Build <%{build_url}|#%{build_number}> (<%{compare_url}|%{commit}>) of %{repository}@%{branch} by %{author} %{result} in %{duration}"
+    on_success: change
+    on_failure: change
+    on_pull_requests: false
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f388b4139..794a378b3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,289 @@
 # Cromwell Change Log
 
+## 29
+
+### Breaking Changes
+
+* Request timeouts for HTTP requests on the REST API now return a 503 status code instead of 500. The response for a request timeout is no longer in JSON format.
+* The metadata endpoint no longer returns gzipped responses by default. This now needs to be explicitly requested with an `Accept-Encoding: gzip` header 
+
+* Command line usage has been extensively revised for Cromwell 29.  Please see the
+[README](https://github.com/broadinstitute/cromwell#command-line-usage) for details.
+
+* The engine endpoints are now served under `/engine`. Previousely engine endpoints were available under
+`/api/engine`. Workflow endpoints are still served under `/api/workflows`. The setting `api.routeUnwrapped` has been
+retired at the same time.
+
+* The response format of the [callcaching/diff](https://github.com/broadinstitute/cromwell#get-apiworkflowsversioncallcachingdiff) endpoint has been updated.
+
+### Cromwell Server
+
+* Cromwell now attempts to gracefully shutdown when running in server mode and receiving a `SIGINT` (`Ctrl-C`) or `SIGTERM` (`kill`) signal. This includes waiting for all pending Database writes before exiting.
+A detailed explanation and information about how to configure this feature can be found in the [Cromwell Wiki](https://github.com/broadinstitute/cromwell/wiki/DevZone#graceful-server-shutdown).
+
+## 28
+
+### Bug Fixes
+
+#### WDL write_* functions add a final newline
+
+The following WDL functions now add a newline after the final line of output (the previous behavior of not adding this
+newline was inadvertent):
+- `write_lines`
+- `write_map`
+- `write_object`
+- `write_objects`
+- `write_tsv`
+
+For example:
+
+```
+task writer {
+  Array[String] a = ["foo", "bar"]
+  command {
+    # used to output: "foo\nbar"
+    # now outputs: "foo\nbar\n"
+    cat write_lines(a)
+  }
+}
+```
+
+#### `ContinueWhilePossible`
+
+A workflow utilizing the WorkflowFailureMode Workflow Option `ContinueWhilePossible` will now successfully reach a terminal state once all runnable jobs have completed.
+#### `FailOnStderr` 
+When `FailOnStderr` is set to false, Cromwell no longer checks for the existence of a stderr file for that task. 
+
+### WDL Functions
+
+#### New functions: floor, ceil and round:
+
+Enables the `floor`, `ceil` and `round` functions in WDL to convert floating point numbers to integers.
+
+For example we can now use the size of an input file to influence the amount of memory the task is given. In the example below a 500MB input file will result in a request for a VM with 2GB of memory:
+
+```
+task foo {
+    File in_file
+    command { ... }
+    runtime {
+      docker: "..."
+      memory: ceil(size(in_file)) * 4 
+    }
+}
+```
+
+### Call Caching
+
+* Hash values calculated by Cromwell for a call when call caching is enabled are now published to the metadata.
+It is published even if the call failed. However if the call is attempted multiple times (because it has been preempted for example),
+since hash values are strictly identical for all attempts, they will only be published in the last attempt section of the metadata for this call.
+If the hashes fail to be calculated, the reason is indicated in a `hashFailures` field in the `callCaching` section of the call metadata.
+*Important*: Hashes are not retroactively published to the metadata. Which means only workflows run on Cromwell 28+ will have hashes in their metadata.
+
+See the [README](https://github.com/broadinstitute/cromwell#get-apiworkflowsversionidmetadata) for an example metadata response.
+
+* New endpoint returning the hash differential for 2 calls. 
+
+`GET /api/workflows/:version/callcaching/diff`
+
+See the [README](https://github.com/broadinstitute/cromwell#get-apiworkflowsversioncallcachingdiff) for more details.
+
+### Workflow Submission
+
+* The workflow submission parameters `wdlSource` and `wdlDependencies` have been deprecated in favor of `workflowSource` and
+`workflowDependencies` respectively.  The older names are still supported in Cromwell 28 with deprecation warnings but will
+be removed in a future version of Cromwell.
+
+### Labels
+* A new `/labels` endpoint has been added to update labels for an existing workflow. See the [README](README.md#patch-apiworkflowsversionidlabels) for more information.
+* Label formatting requirements have been updated, please check the [README](README.md#label-format) for more detailed documentation.
+
+
+### JES Backend
+
+The JES backend now supports a `filesystems.gcs.caching.duplication-strategy` configuration entry.
+It can be set to specify the desired behavior of Cromwell regarding call outputs when a call finds a hit in the cache.
+The default value is `copy` which will copy all output files to the new call directory.
+A second value is allowed, `reference`, that will instead point to the original output files, without copying them.
+
+
+```hocon
+filesystems {
+  gcs {
+    auth = "application-default"
+    
+    caching {
+      duplication-strategy = "reference"
+    }
+  }
+}
+```
+
+A placeholder file will be placed in the execution folder of the cached call to explain the absence of output files and point to the location of the original ones.
+
+
+### Metadata Write Batching
+
+Metadata write batching works the same as in previous versions of Cromwell, but the default batch size has been changed from 1 to 200.  It's possible that 200 is too high in some environments, but 200 is more likely to be an appropriate value
+than the previous default.
+
+
+## 27
+
+### Migration
+
+* Call Caching has been improved in this version of Cromwell, specifically the time needed to determine whether or not a job can be cached
+ has drastically decreased. To achieve that the database schema has been modified and a migration is required in order to preserve the pre-existing cached jobs.
+ This migration is relatively fast compared to previous migrations. To get an idea of the time needed, look at the size of your `CALL_CACHING_HASH_ENTRY` table.
+ As a benchmark, it takes 1 minute for a table with 6 million rows.
+ The migration will only be executed on MySQL. Other databases will lose their previous cached jobs.
+ In order to run properly on MySQL, **the following flag needs to be adjusted**: https://dev.mysql.com/doc/refman/5.5/en/server-system-variables.html#sysvar_group_concat_max_len
+ The following query will give you a minimum to set the group_concat_max_len value to:
+ 
+ ```sql
+SELECT MAX(aggregated) as group_concat_max_len FROM
+      (
+            SELECT cche.CALL_CACHING_ENTRY_ID, SUM(LENGTH(CONCAT(cche.HASH_KEY, cche.HASH_VALUE))) AS aggregated
+            FROM CALL_CACHING_HASH_ENTRY cche
+            GROUP BY cche.CALL_CACHING_ENTRY_ID
+      ) aggregation
+ ```
+
+ Here is the SQL command to run to set the group_concat_max_len flag to the proper value:
+ 
+ ```sql
+SET GLOBAL group_concat_max_len = value
+ ```
+ 
+ Where `value` is replaced with the value you want to set it to.
+ 
+ Note that the migration will fail if the flag is not set properly.
+ 
+### Breaking Changes
+
+* The update to Slick 3.2 requires a database stanza to
+[switch](http://slick.lightbend.com/doc/3.2.0/upgrade.html#profiles-vs-drivers) from using `driver` to `profile`.
+
+```hocon
+database {
+  #driver = "slick.driver.MySQLDriver$" #old
+  profile = "slick.jdbc.MySQLProfile$"  #new
+  db {
+    driver = "com.mysql.jdbc.Driver"
+    url = "jdbc:mysql://host/cromwell?rewriteBatchedStatements=true"
+    user = "user"
+    password = "pass"
+    connectionTimeout = 5000
+  }
+}
+```
+
+### Call Caching
+
+Cromwell now supports call caching with floating Docker tags (e.g. `docker: "ubuntu:latest"`). Note it is still considered
+a best practice to specify Docker images as hashes where possible, especially for production usages.
+
+Within a single workflow Cromwell will attempt to resolve all floating tags to the same Docker hash, even if Cromwell is restarted
+during the execution of a workflow. In call metadata the `docker` runtime attribute is now the same as the
+value that actually appeared in the WDL:
+
+```
+   "runtimeAttributes": {
+     "docker": "ubuntu:latest",
+     "failOnStderr": "false",
+     "continueOnReturnCode": "0"
+   }
+```
+
+Previous versions of Cromwell rewrote the `docker` value to the hash of the Docker image.
+
+There is a new call-level metadata value `dockerImageUsed` which captures the hash of the Docker image actually used to
+run the call:
+
+```
+   "dockerImageUsed": "library/ubuntu@sha256:382452f82a8bbd34443b2c727650af46aced0f94a44463c62a9848133ecb1aa8"
+```
+
+### Docker
+
+* The Docker section of the configuration has been slightly reworked 
+An option to specify how a Docker hash should be looked up has been added. Two methods are available.
+    "local" will try to look for the image on the machine where cromwell is running. If it can't be found, Cromwell will try to `pull` the image and use the hash from the retrieved image.
+    "remote" will try to look up the image hash directly on the remote repository where the image is located (Docker Hub and GCR are supported)
+Note that the "local" option will require docker to be installed on the machine running cromwell, in order for it to call the docker CLI.
+* Adds hash lookup support for public [quay.io](https://quay.io/) images.
+
+### WDL Feature Support
+* Added support for the new WDL `basename` function. Allows WDL authors to get just the file name from a File (i.e. removing the directory path)
+* Allows coercion of `Map` objects into `Array`s of `Pair`s. This also allows WDL authors to directly scatter over WDL `Map`s.
+
+### Miscellaneous
+* Adds support for JSON file format for google service account credentials. As of Cromwell 27, PEM credentials for PAPI are deprecated and support might be removed in a future version.
+
+```
+google {
+
+  application-name = "cromwell"
+
+  auths = [
+    {
+      name = "service-account"
+      scheme = "service_account"
+      json-file = "/path/to/file.json"
+    }
+  ]
+}
+```
+
+### General Changes
+
+* The `/query` endpoint now supports querying by `label`. See the [README](README.md#get-apiworkflowsversionquery) for more information.
+* The `read_X` standard library functions limit accepted filesizes.  These differ by type, e.g. read_bool has a smaller limit than read_string.  See reference.conf for default settings.
+
+## 26
+
+### Breaking Changes
+
+* Failure metadata for calls and workflows was being displayed inconsistently, with different formats depending on the originating Cromwell version. Failures will now always present as an array of JSON objects each representing a failure. Each failure will have a message and a causedBy field. The causedBy field will be an array of similar failure objects. An example is given below:
+
+```
+failures: [{
+  message: "failure1",
+  causedBy: [{
+    message: "cause1",
+    causedBy: []
+   }, {
+    message: "cause2",
+    causedBy: []
+  }]
+ }, {
+  message: "failure2",
+  causedBy: []
+}]
+```
+
+### Additional Upgrade Time
+
+* Upgrading to Cromwell 26 will take additional time due to the migration of failure metadata. Cromwell will automatically run a database query during the upgrade which appears to be roughly linear to the number of rows in the METADATA_ENTRY table. You can estimate upgrade time using the following equation: `time to migrate (in seconds) ~= (rows in METADATA_ENTRY) / 65000` Note that due to differences in hardware and database speed, this is only a rough estimate.
+
+### Config Changes
+
+* Added a configuration option under `system.io` to throttle the number of I/O queries that Cromwell makes, as well as configure retry parameters.
+ This is mostly useful for the JES backend and should be updated to match the GCS quota available for the project.
+ 
+```
+system.io {
+  # Global Throttling - This is mostly useful for GCS and can be adjusted to match
+  # the quota availble on the GCS API
+  number-of-requests = 100000
+  per = 100 seconds
+  
+  # Number of times an I/O operation should be attempted before giving up and failing it.
+  number-of-attempts = 5
+}
+```
+
 ## 25
 
 ### External Contributors
diff --git a/NOTICE b/NOTICE
deleted file mode 100644
index 15d3c2b54..000000000
--- a/NOTICE
+++ /dev/null
@@ -1,4 +0,0 @@
-cromwell.webservice/PerRequest.scala (https://github.com/NET-A-PORTER/spray-actor-per-request)
-is distributed with this software under the Apache License, Version 2.0 (see the LICENSE-ASL file). In accordance
-with that license, that software comes with the following notices:
-    Copyright (C) 2011-2012 Ian Forsey
diff --git a/README.md b/README.md
index 1ac9cfd76..97040254e 100644
--- a/README.md
+++ b/README.md
@@ -18,9 +18,6 @@ A [Workflow Management System](https://en.wikipedia.org/wiki/Workflow_management
 * [Installing](#installing)
   * [Upgrading from 0.19 to 0.21](#upgrading-from-019-to-021)
 * [Command Line Usage](#command-line-usage)
-  * [run](#run)
-  * [server](#server)
-  * [version](#version)
 * [Getting Started with WDL](#getting-started-with-wdl)
   * [WDL Support](#wdl-support)
 * [Configuring Cromwell](#configuring-cromwell)
@@ -88,6 +85,7 @@ A [Workflow Management System](https://en.wikipedia.org/wiki/Workflow_management
   * [POST /api/workflows/:version/batch](#post-apiworkflowsversionbatch)
   * [GET /api/workflows/:version/query](#get-apiworkflowsversionquery)
   * [POST /api/workflows/:version/query](#post-apiworkflowsversionquery)
+  * [PATCH /api/workflows/:version/:id/labels](#patch-apiworkflowsversionidlabels)
   * [GET /api/workflows/:version/:id/status](#get-apiworkflowsversionidstatus)
   * [GET /api/workflows/:version/:id/outputs](#get-apiworkflowsversionidoutputs)
   * [GET /api/workflows/:version/:id/timing](#get-apiworkflowsversionidtiming)
@@ -95,8 +93,9 @@ A [Workflow Management System](https://en.wikipedia.org/wiki/Workflow_management
   * [GET /api/workflows/:version/:id/metadata](#get-apiworkflowsversionidmetadata)
   * [POST /api/workflows/:version/:id/abort](#post-apiworkflowsversionidabort)
   * [GET /api/workflows/:version/backends](#get-apiworkflowsversionbackends)
-  * [GET /api/engine/:version/stats](#get-apiengineversionstats)
-  * [GET /api/engine/:version/version](#get-apiengineversionversion)
+  * [GET /api/workflows/:version/callcaching/diff](#get-apiworkflowsversioncallcachingdiff)
+  * [GET /engine/:version/stats](#get-engineversionstats)
+  * [GET /engine/:version/version](#get-engineversionversion)
   * [Error handling](#error-handling)
 * [Developer](#developer)
   * [Generating table of contents on Markdown files](#generating-table-of-contents-on-markdown-files)
@@ -121,13 +120,13 @@ There is a [Cromwell gitter channel](https://gitter.im/broadinstitute/cromwell)
 
 The following is the toolchain used for development of Cromwell.  Other versions may work, but these are recommended.
 
-* [Scala 2.11.8](http://www.scala-lang.org/news/2.11.8/)
+* [Scala 2.12.2](http://www.scala-lang.org/news/2.12.1#scala-212-notes)
 * [SBT 0.13.12](https://github.com/sbt/sbt/releases/tag/v0.13.12)
 * [Java 8](http://www.oracle.com/technetwork/java/javase/overview/java8-2100321.html)
 
 # Building
 
-`sbt assembly` will build a runnable JAR in `target/scala-2.11/`
+`sbt assembly` will build a runnable JAR in `target/scala-2.12/`
 
 Tests are run via `sbt test`.  Note that the tests do require Docker to be running.  To test this out while downloading the Ubuntu image that is required for tests, run `docker pull ubuntu:latest` prior to running `sbt test`
 
@@ -141,170 +140,203 @@ See the [migration document](MIGRATION.md) for more details.
 
 # Command Line Usage
 
-Run the JAR file with no arguments to get the usage message:
+For built-in documentation of Cromwell command line usage, run the Cromwell JAR file with no arguments:
 
 ```
+$ java -jar cromwell-<versionNumber>.jar
+```
 
+For example, `$ java -jar cromwell-29.jar`. You will get a usage message like the following:
 
-$ java -jar cromwell.jar
-java -jar cromwell.jar <action> <parameters>
-
-Actions:
-run <WDL file> [<JSON inputs file>] [<JSON workflow options>]
-  [<OUTPUT workflow metadata>] [<Zip of WDL Files>] [<JSON labels file>]
+```
+cromwell 29
+Usage: java -jar /path/to/cromwell.jar [server|run] [options] <args>...
 
-  Given a WDL file and JSON file containing the value of the
-  workflow inputs, this will run the workflow locally and
-  print out the outputs in JSON format.  The workflow
-  options file specifies some runtime configuration for the
-  workflow (see README for details).  The workflow metadata
-  output is an optional file path to output the metadata. The
-  directory of WDL files is optional. However, it is required
-  if the primary workflow imports workflows that are outside
-  of the root directory of the Cromwell project.
+  --help                   Cromwell - Workflow Execution Engine
+  --version                
+Command: server
+Starts a web server on port 8000.  See the web server documentation for more details about the API endpoints.
+Command: run [options] workflow-source
+Run the workflow and print out the outputs in JSON format.
+  workflow-source          Workflow source file.
+  -i, --inputs <value>     Workflow inputs file.
+  -o, --options <value>    Workflow options file.
+  -t, --type <value>       Workflow type.
+  -v, --type-version <value>
+                           Workflow type version.
+  -l, --labels <value>     Workflow labels file.
+  -p, --imports <value>    A directory or zipfile to search for workflow imports.
+  -m, --metadata-output <value>
+                           An optional directory path to output metadata.
+```
 
-  Use a single dash ("-") to skip optional files. Ex:
-    run noinputs.wdl - - metadata.json -
+## --version
 
-  server
+The `--version` option prints the version of Cromwell and exits.
 
-  Starts a web server on port 8000.  See the web server
-  documentation for more details about the API endpoints.
+## --help
 
-  -version
+The `--help` option prints the full help text above and exits.
 
-  Returns the version of the Cromwell engine.
+## server
 
-```
+The `server` command runs Cromwell as a web server.  No arguments are accepted.
+See the documentation for Cromwell's REST endpoints [here](#rest-api).
 
 ## run
 
-Given a WDL file and a JSON inputs file (see `inputs` subcommand), Run the workflow and print the outputs:
+The `run` command executes a single workflow in Cromwell.
 
-```
-$ java -jar cromwell.jar run 3step.wdl inputs.json
-... play-by-play output ...
-{
-  "three_step.ps.procs": "/var/folders/kg/c7vgxnn902lc3qvc2z2g81s89xhzdz/T/stdout1272284837004786003.tmp",
-  "three_step.cgrep.count": 0,
-  "three_step.wc.count": 13
-}
-```
+### workflow-source
+The `run` command requires a single argument for the workflow source file.
+ 
+### --inputs
+An optional file of workflow inputs.  Although optional, it is a best practice to use an inputs file to satisfy workflow
+requirements rather than hardcoding inputs directly into a workflow source file.
 
-The JSON inputs can be left off if there's a file with the same name as the WDL file but with a `.inputs` extension.  For example, this will assume that `3step.inputs` exists:
+### --options
+An optional file of workflow options.  Some options are global (supported by all backends), while others are backend-specific.
+See the [workflow options](#workflow-options) documentation for more details.
 
-```
-$ java -jar cromwell.jar run 3step.wdl
-```
+### --type
+An optional parameter to specify the language for the workflow source.  Any value specified for this parameter is currently
+ignored and internally the value `WDL` is used.
 
-If your workflow has no inputs, you can specify `-` as the value for the inputs parameter:
+### --type-version
+An optional parameter to specify the version of the language for the workflow source.  Currently any specified value is ignored.
 
-```
-$ java -jar cromwell.jar run my_workflow.wdl -
-```
-
-The third, optional parameter to the 'run' subcommand is a JSON file of workflow options.  By default, the command line will look for a file with the same name as the WDL file but with the extension `.options`.  But one can also specify a value of `-` manually to specify that there are no workflow options.
+### --labels
+An optional parameter to specify a file of JSON key-value label pairs to associate with the workflow.
 
-See the section [workflow options](#workflow-options) for more details.
-
-```
-$ java -jar cromwell.jar run my_jes_wf.wdl my_jes_wf.json wf_options.json
-```
+### --imports
+You have the option of importing WDL workflows or tasks to use within your workflow, known as sub-workflows.
+If you use sub-workflows within your primary workflow then you must include a zip file with the WDL import files.
 
-The fourth, optional parameter to the 'run' subcommand is a path where the workflow metadata will be written.  By default, no workflow metadata will be written.
+For example, say you have a directory of WDL files:
 
 ```
-$ java -jar cromwell.jar run my_wf.wdl - - my_wf.metadata.json
-... play-by-play output ...
-$ cat my_wf.metadata.json
-{
-  "workflowName": "w",
-  "calls": {
-    "w.x": [{
-      "executionStatus": "Done",
-      "stdout": "/Users/jdoe/projects/cromwell/cromwell-executions/w/a349534f-137b-4809-9425-1893ac272084/call-x/stdout",
-      "shardIndex": -1,
-      "outputs": {
-        "o": "local\nremote"
-      },
-      "runtimeAttributes": {
-        "failOnStderr": "false",
-        "continueOnReturnCode": "0"
-      },
-      "cache": {
-        "allowResultReuse": true
-      },
-      "inputs": {
-        "remote": "/Users/jdoe/remote.txt",
-        "local": "local.txt"
-      },
-      "returnCode": 0,
-      "backend": "Local",
-      "end": "2016-07-11T10:27:56.074-04:00",
-      "stderr": "/Users/jdoe/projects/cromwell/cromwell-executions/w/a349534f-137b-4809-9425-1893ac272084/call-x/stderr",
-      "callRoot": "cromwell-executions/w/a349534f-137b-4809-9425-1893ac272084/call-x",
-      "attempt": 1,
-      "start": "2016-07-11T10:27:55.992-04:00"
-    }]
-  },
-  "outputs": {
-    "w.x.o": "local\nremote"
-  },
-  "workflowRoot": "cromwell-executions/w/a349534f-137b-4809-9425-1893ac272084",
-  "id": "a349534f-137b-4809-9425-1893ac272084",
-  "inputs": {
-    "w.x.remote": "/Users/jdoe/remote.txt",
-    "w.x.local": "local.txt"
-  },
-  "submission": "2016-07-11T10:27:54.907-04:00",
-  "status": "Succeeded",
-  "end": "2016-07-11T10:27:56.108-04:00",
-  "start": "2016-07-11T10:27:54.919-04:00"
-}
-```
-
-The fifth, optional parameter to the 'run' subcommand is a zip file which contains WDL source files. This zip file can be passed
-and your primary workflow can import any WDL's from that collection and re-use those tasks.
-
-For example, consider you have a directory of WDL files:
-```
-my_WDLs
+wdl_library
 └──cgrep.wdl
 └──ps.wdl
 └──wc.wdl
 ```
 
-If you zip that directory to my_WDLs.zip, you have the option to pass it in as the last parameter in your run command
-and be able to reference these WDLs as imports in your primary WDL. For example, your primary WDL can look like this:
+If you zip that directory into `wdl_library.zip`, then you can reference and use these WDLs within your primary WDL.
+
+This could be your primary WDL:
+
 ```
 import "ps.wdl" as ps
 import "cgrep.wdl"
 import "wc.wdl" as wordCount
 
-workflow threestep {
+workflow my_wf {
 
 call ps.ps as getStatus
 call cgrep.cgrep { input: str = getStatus.x }
 call wordCount { input: str = ... }
 
 }
-
 ```
-The command to run this WDL, without needing any inputs, workflow options or metadata files would look like:
 
-```
-$ java -jar cromwell.jar run threestep.wdl - - - /path/to/my_WDLs.zip
-```
+Then to run this WDL without any inputs, workflow options, or metadata files, you would enter:
 
-The sixth optional parameter is a path to a labels file. See [Labels](#labels) for information and the expected format.
+`$ java -jar cromwell-<versionNumber>.jar run my_wf.wdl --imports /path/to/wdl_library.zip`
 
-## server
+### --metadata-output
+
+You can include a path where Cromwell will write the workflow metadata JSON, such as start/end timestamps, status, inputs, and outputs. By default, Cromwell does not write workflow metadata.
 
-Start a server on port 8000, the API for the server is described in the [REST API](#rest-api) section.
+This example includes a metadata path called `/path/to/my_wf.metadata`:
+
+```
+$ java -jar cromwell-<versionNumber>.jar run my_wf.wdl --metadata-output /path/to/my_wf.metadata
+```
 
-## version
+Again, Cromwell is very verbose. Here is the metadata output in my_wf.metadata:
 
-Returns the version of Cromwell engine.
+```
+{
+  "workflowName": "my_wf",
+  "submittedFiles": {
+    "inputs": "{\"my_wf.hello.addressee\":\"m'Lord\"}",
+    "workflow": "\ntask hello {\n  String addressee\n  command {\n    echo \"Hello ${addressee}!\"\n  }\n  output {\n    String salutation = read_string(stdout())\n  }\n  runtime {\n   
+\n  }\n}\n\nworkflow my_wf {\n  call hello\n  output {\n     hello.salutation\n  }\n}\n",
+    "options": "{\n\n}"
+  },
+  "calls": {
+    "my_wf.hello": [
+      {
+        "executionStatus": "Done",
+        "stdout": "/Users/jdoe/Documents/cromwell-executions/my_wf/cd0fe94a-984e-4a19-ab4c-8f7f07038068/call-hello/execution/stdout",
+        "backendStatus": "Done",
+        "shardIndex": -1,
+        "outputs": {
+          "salutation": "Hello m'Lord!"
+        },
+        "runtimeAttributes": {
+          "continueOnReturnCode": "0",
+          "failOnStderr": "false"
+        },
+        "callCaching": {
+          "allowResultReuse": false,
+          "effectiveCallCachingMode": "CallCachingOff"
+        },
+        "inputs": {
+          "addressee": "m'Lord"
+        },
+        "returnCode": 0,
+        "jobId": "28955",
+        "backend": "Local",
+        "end": "2017-04-19T10:53:25.045-04:00",
+        "stderr": "/Users/jdoe/Documents/cromwell-executions/my_wf/cd0fe94a-984e-4a19-ab4c-8f7f07038068/call-hello/execution/stderr",
+        "callRoot": "/Users/jdoe/Documents/cromwell-executions/my_wf/cd0fe94a-984e-4a19-ab4c-8f7f07038068/call-hello",
+        "attempt": 1,
+        "executionEvents": [
+          {
+            "startTime": "2017-04-19T10:53:23.570-04:00",
+            "description": "PreparingJob",
+            "endTime": "2017-04-19T10:53:23.573-04:00"
+          },
+          {
+            "startTime": "2017-04-19T10:53:23.569-04:00",
+            "description": "Pending",
+            "endTime": "2017-04-19T10:53:23.570-04:00"
+          },
+          {
+            "startTime": "2017-04-19T10:53:25.040-04:00",
+            "description": "UpdatingJobStore",
+            "endTime": "2017-04-19T10:53:25.045-04:00"
+          },
+          {
+            "startTime": "2017-04-19T10:53:23.570-04:00",
+            "description": "RequestingExecutionToken",
+            "endTime": "2017-04-19T10:53:23.570-04:00"
+          },
+          {
+            "startTime": "2017-04-19T10:53:23.573-04:00",
+            "description": "RunningJob",
+            "endTime": "2017-04-19T10:53:25.040-04:00"
+          }
+        ],
+        "start": "2017-04-19T10:53:23.569-04:00"
+      }
+    ]
+  },
+  "outputs": {
+    "my_wf.hello.salutation": "Hello m'Lord!"
+  },
+  "workflowRoot": "/Users/jdoe/Documents/cromwell-executions/my_wf/cd0fe94a-984e-4a19-ab4c-8f7f07038068",
+  "id": "cd0fe94a-984e-4a19-ab4c-8f7f07038068",
+  "inputs": {
+    "my_wf.hello.addressee": "m'Lord"
+  },
+  "submission": "2017-04-19T10:53:19.565-04:00",
+  "status": "Succeeded",
+  "end": "2017-04-19T10:53:25.063-04:00",
+  "start": "2017-04-19T10:53:23.535-04:00"
+}
+```
 
 # Getting Started with WDL
 
@@ -368,14 +400,23 @@ For many examples on how to use WDL see [the WDL site](https://github.com/broadi
   * [File stdout()](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#file-stdout)
   * [File stderr()](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#file-stderr)
   * [Array\[String\] read_lines(String|File)](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#arraystring-read_linesstringfile)
+    * File reads are limited to 128 KB. Configurable via conf file.
   * [Array\[Array\[String\]\] read_tsv(String|File)](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#arrayarraystring-read_tsvstringfile)
+    * File reads are limited to 128 KB. Configurable via conf file.
   * [Map\[String, String\] read_map(String|File)](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#mapstring-string-read_mapstringfile)
+    * File reads are limited to 128 KB. Configurable via conf file.
   * [Object read_object(String|File)](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#object-read_objectstringfile)
+    * File reads are limited to 128 KB. Configurable via conf file.
   * [Array\[Object\] read_objects(String|File)](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#arrayobject-read_objectsstringfile)
+    * File reads are limited to 128 KB. Configurable via conf file.
   * [Int read_int(String|File)](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#int-read_intstringfile)
+    * File reads are limited to 19 B. Configurable via conf file.
   * [String read_string(String|File)](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#string-read_stringstringfile)
+    * File reads are limited to 128 KB. Configurable via conf file.
   * [Float read_float(String|File)](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#float-read_floatstringfile)
+    * File reads are limited to 50 B. Configurable via conf file.
   * [Boolean read_boolean(String|File)](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#boolean-read_booleanstringfile)
+    * File reads are limited to 7 B. Configurable via conf file.
   * [File write_lines(Array\[String\])](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#file-write_linesarraystring)
   * [File write_tsv(Array\[Array\[String\]\])](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#file-write_tsvarrayarraystring)
   * [File write_map(Map\[String, String\])](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#file-write_mapmapstring-string)
@@ -385,8 +426,8 @@ For many examples on how to use WDL see [the WDL site](https://github.com/broadi
   * [String sub(String, String, String)](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#string-substring-string-string)
   * [Array\[Int\] range(Int)](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#arrayint-rangeint)
   * [Array\[Array\[X\]\] transpose(Array\[Array\[X\]\])](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#arrayarrayx-transposearrayarrayx)
-  * [Pair(X,Y) zip(X,Y)](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#pairxy-zipxy)
-  * [Pair(X,Y) cross(X,Y)](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#pairxy-crossxy)
+  * [Array\[Pair\[X,Y\]\] zip(Array\[X\], Array\[Y\])](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#arraypairxy-ziparrayx-arrayy)
+  * [Array\[Pair\[X,Y\]\] cross(Array\[X\], Array\[Y\])](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#arraypairxy-crossarrayx-arrayy)
   * [Integer length(Array\[X\])](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#integer-lengtharrayx)
   * [Array\[String\] prefix(String, Array\[X\])](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#arraystring-prefixstring-arrayx)
 * [Data Types & Serialization](https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#data-types--serialization)
@@ -417,7 +458,7 @@ For many examples on how to use WDL see [the WDL site](https://github.com/broadi
 
 # Configuring Cromwell
 
-Cromwell's default configuration file is located at `src/main/resources/application.conf`.
+Cromwell's default configuration file is located at `core/src/main/resources/reference.conf`.
 
 The configuration file is in [Hocon](https://github.com/typesafehub/config/blob/master/HOCON.md#hocon-human-optimized-config-object-notation) which means the configuration file can specify configuration as JSON-like stanzas like:
 
@@ -445,12 +486,40 @@ This allows any value to be overridden on the command line:
 java -Dwebservice.port=8080 cromwell.jar ...
 ```
 
-It is recommended that one copies `src/main/resources/application.conf`, modify it, then link to it via:
+
+To customize configuration it is recommended that one copies relevant stanzas from `core/src/main/resources/reference.conf` into a new file, modify it as appropriate, then pass it to Cromwell via:
 
 ```
-java -Dconfig.file=/path/to/application.conf cromwell.jar ...
+java -Dconfig.file=/path/to/yourOverrides.conf cromwell.jar ...
 ```
 
+## I/O
+
+Cromwell centralizes as many of its I/O operations as possible through a unique entry point. This allows users to effectively control and throttle the number of requests and resources allocated to those operations throughout the entire system.
+It is possible to configure this throttling behavior in the configuration:
+
+```
+system.io {
+  number-of-requests = 100000
+  per = 100 seconds
+}
+```
+
+This is particularly useful when running Cromwell on a JES backend for example, as Google imposes a quota on the number of GCS queries that can be made.
+
+### Resilience
+
+I/O operations can fail for a number of reason from network failures to server errors. Some of those errors are not fatal and can be retried.
+Cromwell will retry I/O operations on such retryable errors, up to a number of times. This number (more precisely the number of attempts that will be made) can be set using the following configuration option:
+
+```
+system.io {
+  # Number of times an I/O operation should be attempted before giving up and failing it.
+  number-of-attempts = 5
+}
+```
+
+
 ## Workflow Submission
 
 Cromwell has a configurable cap on the number of workflows running at a time. To set this value provide an integer value to the `system.max-concurrent-workflows` config value.
@@ -469,8 +538,7 @@ Then, edit the configuration file `database` stanza, as follows:
 
 ```
 database {
-
-  driver = "slick.driver.MySQLDriver$"
+  profile = "slick.jdbc.MySQLProfile$"
   db {
     driver = "com.mysql.jdbc.Driver"
     url = "jdbc:mysql://host/cromwell?rewriteBatchedStatements=true"
@@ -478,10 +546,6 @@ database {
     password = "pass"
     connectionTimeout = 5000
   }
-
-  test {
-    ...
-  }
 }
 ```
 
@@ -499,7 +563,7 @@ system {
 
 Or, via `-Dsystem.abort-jobs-on-terminate=true` command line option.
 
-By default, this value is false when running `java -jar cromwell.jar server`, and true when running `java -jar cromwell.jar run <wdl> <inputs>`.
+By default, this value is false when running `java -jar cromwell.jar server`, and true when running `java -jar cromwell.jar run <workflow source> <inputs>`.
 
 # Security
 
@@ -615,7 +679,7 @@ When Cromwell runs a workflow, it first creates a directory `<cromwell_root>/<wo
 
 Each `call` has its own subdirectory located at `<workflow_root>/call-<call_name>`.  This is the `<call_dir>`.  For example, having a `stdout` and `stderr` file is common among both backends and they both write a shell script file to the `<call_dir>` as well.  See the descriptions below for details about backend-specific files that are written to these directories.
 
-An example of a workflow output directory for a three-step WDL file might look like this:
+An example of a workflow output directory for a three-step workflow might look like this:
 
 ```
 cromwell-executions/
@@ -979,7 +1043,7 @@ backend {
     TES {
       actor-factory = "cromwell.backend.impl.tes.TesBackendLifecycleActorFactory"
       config {
-        endpoint = "https://<some-url>/v1/jobs"
+        endpoint = "https://<some-url>/v1/tasks"
         root = "cromwell-executions"
         dockerRoot = "/cromwell-executions"
         concurrent-job-limit = 1000
@@ -997,16 +1061,13 @@ This backend supports the following optional runtime attributes / workflow optio
 * docker: Docker image to use such as "Ubuntu".
 * dockerWorkingDir: defines the working directory in the container.
 
-Outputs:
-It will use `dockerOutputDir` runtime attribute / workflow option to resolve the folder in which the execution results will placed. If there is no `dockerWorkingDir` defined it will use `/cromwell-executions/<workflow_uuid>/call-<call_name>/execution`.
-
 ### CPU, Memory and Disk
 This backend supports CPU, memory and disk size configuration through the use of the following runtime attributes / workflow options:
-* cpu: defines the amount of CPU to use. Default value: 1. Type: Integer. Ex: 4.
-* memory: defines the amount of memory to use. Default value: "2 GB". Type: String. Ex: "4 GB" or "4096 MB"
-* disk: defines the amount of disk to use. Default value: "2 GB". Type: String. Ex: "1 GB" or "1024 MB"
+* cpu: defines the amount of CPU to use. Type: Integer. Ex: 4.
+* memory: defines the amount of memory to use. Type: String. Ex: "4 GB" or "4096 MB"
+* disk: defines the amount of disk to use. Type: String. Ex: "1 GB" or "1024 MB"
 
-It they are not set, the TES backend will use default values.
+If they are not set, the TES backend may use default values.
 
 ## Sun GridEngine Backend
 
@@ -1239,78 +1300,117 @@ The `job-id-regex` should contain one capture group while matching against the w
 
 Allows to execute jobs using HTCondor which is a specialized workload management system for compute-intensive jobs created by the Center for High Throughput Computing in the Department of Computer Sciences at the University of Wisconsin-Madison (UW-Madison).
 
-This backend creates six files in the `<call_dir>` (see previous section):
+The backend is specified via the actor factory `ConfigBackendLifecycleActorFactory`:
 
-* `script` - A shell script of the job to be run.  This contains the user's command from the `command` section of the WDL code.
-* `stdout` - The standard output of the process
-* `stderr` - The standard error of the process
-* `submitfile` - A submit file that HtCondor understands in order to submit a job
-* `submitfile.stdout` - The standard output of the submit file
-* `submitfile.stderr` - The standard error of the submit file
+```
+backend {
+  providers {
+    HtCondor {
+      config {
+        actor-factory = "cromwell.backend.impl.sfs.config.ConfigBackendLifecycleActorFactory"
+        # ... other configuration
+      }
+    }
+  }
+}
+```
 
-The `script` file contains:
+This backend makes the same assumption about the filesystem that the local backend does: the Cromwell process and the jobs both have read/write access to the CWD of the job.
+
+The CWD will contain a `script.sh` file which will contain the same contents as the Local backend:
 
 ```
+#!/bin/sh
 cd <container_call_root>
 <user_command>
 echo $? > rc
 ```
 
-The `submitfile` file contains:
+The job is launched with a configurable script command such as:
 
 ```
-executable=cromwell-executions/test/e950e07d-4132-4fe0-8d86-ab6925dd94ad/call-merge_files/script
-output=cromwell-executions/test/e950e07d-4132-4fe0-8d86-ab6925dd94ad/call-merge_files/stdout
-error=cromwell-executions/test/e950e07d-4132-4fe0-8d86-ab6925dd94ad/call-merge_files/stderr
-log=cromwell-executions/test/e950e07d-4132-4fe0-8d86-ab6925dd94ad/call-merge_files/merge_files.log
+chmod 755 ${script}
+cat > ${cwd}/execution/submitFile <<EOF
+Iwd=${cwd}/execution
+requirements=${nativeSpecs}
+leave_in_queue=true
+request_memory=${memory_mb}
+request_disk=${disk_kb}
+error=${err}
+output=${out}
+log_xml=true
+request_cpus=${cpu}
+executable=${script}
+log=${cwd}/execution/execution.log
 queue
-
+EOF
+condor_submit ${cwd}/execution/submitFile
 ```
 
-### Caching configuration
-This implementation also add basic caching support. It relies in a cache provider to store successful job results.
-By default a MongoDB based cache implementation is provided but there is the option of implementing a new provider based on CacheActorFactory and CacheActor interfaces.
+The HtCondor backend gets the job ID from parsing the `submit.stdout` text file.
 
-From application.conf file:
-```
-cache {
-  provider = "cromwell.backend.impl.htcondor.caching.provider.mongodb.MongoCacheActorFactory"
-  enabled = true
-  forceRewrite = false
-  db {
-    host = "127.0.0.1"
-    port = 27017
-    name = "htcondor"
-    collection = "cache"
-  }
-}
+Since the `script.sh` ends with `echo $? > rc`, the backend will wait for the existence of this file, parse out the return code and determine success or failure and then subsequently post-process.
 
-```
+The command used to submit the job is specified under the configuration key `backend.providers.HtCondor.config.submit`. It uses the same syntax as a command in WDL, and will be provided the variables:
 
-* provider: it defines the provider to use based on CacheActorFactory and CacheActor interfaces.
-* enabled: enables or disables cache.
-* forceRewrite: it allows to invalidate the cache entry and store result again.
-* db section: configuration related to MongoDB provider. It may not exist for other implementations.
+* `script` - A shell script of the job to be run.  This contains the user's command from the `command` section of the WDL code.
+* `cwd` - The path where the script should be run.
+* `out` - The path to the stdout.
+* `err` - The path to the stderr.
+* `job_name` - A unique name for the job.
 
-### Docker
-This backend supports the following optional runtime attributes / workflow options for working with Docker:
-* docker: Docker image to use such as "Ubuntu".
-* dockerWorkingDir: defines the working directory in the container.
-* dockerOutputDir: defiles the output directory in the container when there is the need to define a volume for outputs within the container. By default if this attribute is not set, dockerOutputDir will be the job working directory.
+This backend also supports docker as optional feature. Configuration key `backend.providers.HtCondor.config.submit-docker` is specified for this end. When the WDL contains a docker runtime attribute, this command will be provided with two additional variables:
+
+* `docker` - The docker image name.
+* `docker_cwd` - The path where `cwd` should be mounted within the docker container.
 
-Inputs:
-HtCondor backend analyzes all inputs and do a distinct of the folders in order to mount input folders into the container.
+```
+chmod 755 ${script}
+cat > ${cwd}/execution/dockerScript <<EOF
+#!/bin/bash
+docker run --rm -i -v ${cwd}:${docker_cwd} ${docker} /bin/bash ${script}
+EOF
+chmod 755 ${cwd}/execution/dockerScript
+cat > ${cwd}/execution/submitFile <<EOF
+Iwd=${cwd}/execution
+requirements=${nativeSpecs}
+leave_in_queue=true
+request_memory=${memory_mb}
+request_disk=${disk_kb}
+error=${cwd}/execution/stderr
+output=${cwd}/execution/stdout
+log_xml=true
+request_cpus=${cpu}
+executable=${cwd}/execution/dockerScript
+log=${cwd}/execution/execution.log
+queue
+EOF
+condor_submit ${cwd}/execution/submitFile
+```
 
-Outputs:
-It will use dockerOutputDir runtime attribute / workflow option to resolve the folder in which the execution results will placed. If there is no dockerOutputDir defined it will use the current working directory.
+This backend support additional runtime attributes that are specified in the configuration key `backend.providers.HtCondor.config.runtime-attributes`. It uses the same syntax as specifying runtime attributes in a task in WDL.
 
-### CPU, Memory and Disk
-This backend supports CPU, memory and disk size configuration through the use of the following runtime attributes / workflow options:
-* cpu: defines the amount of CPU to use. Default value: 1. Type: Integer. Ex: 4.
-* memory: defines the amount of memory to use. Default value: "512 MB". Type: String. Ex: "4 GB" or "4096 MB"
-* disk: defines the amount of disk to use. Default value: "1024 MB". Type: String. Ex: "1 GB" or "1024 MB"
+There are five special runtime attribute configurations, `cpu`, `memory_mb`, `disk_kb`, `nativeSpecs`, `docker`.
+Optional values are defined with the prefix `?` attached to the type.
 
-It they are not set, HtCondor backend will use default values.
+```
+backend {
+  providers {
+    HtCondor {
+      config {
+        # ... other configuration
+	    runtime-attributes = """
+	       Int cpu = 1
+	       Float memory_mb = 512.0
+	       Float disk_kb = 256000.0
+	       String? nativeSpecs
+	       String? docker
+	    """
+      }
+    }
+  }
+}
+```
 
 ### Native Specifications
 The use of runtime attribute 'nativeSpecs' allows to the user to attach custom HtCondor configuration to tasks.
@@ -1318,15 +1418,18 @@ An example of this is when there is a need to work with 'requirements' or 'rank'
 
 ```
 "runtimeAttributes": {
-    "nativeSpecs": ["requirements = Arch == \"INTEL\"", "rank = Memory >= 64"]
+    cpu = 2
+    memory = "1GB"
+    disk = "1GB"
+    nativeSpecs: "TARGET.Arch == \"INTEL\" && TARGET.Memory >= 64"
 }
 ```
 
-nativeSpecs attribute needs to be specified as an array of strings to work.
+nativeSpecs attribute needs to be specified as String.
 
 ## Spark Backend
 
-This backend adds support for execution of spark jobs in a workflow using the existing wdl format. 
+This backend adds support for execution of spark jobs in a workflow.
 
 It supports the following Spark deploy modes:
 
@@ -1337,7 +1440,13 @@ It supports the following Spark deploy modes:
 
 ### Configuring Spark Project
 
-When using Spark backend uncomment the following Spark configuration in the application.conf file
+Cromwell's default configuration file is located at `core/src/main/resources/reference.conf`
+
+To customize configuration it is recommended that one copies relevant stanzas from `core/src/main/resources/reference.conf` into a new file, modify it as appropriate, then pass it to Cromwell via:
+
+java -Dconfig.file=/path/to/yourOverrides.conf cromwell.jar
+
+Spark configuration stanza is as follows: 
 
 ```conf
 Spark {
@@ -1394,7 +1503,7 @@ Supported runtime attributes for a Spark Job is as follows:
 * appMainClass ( Spark app/job entry point)
 * numberOfExecutors ( Specific to cluster deploy mode)
 
-Sample usage :
+Sample usage:
 
 ```wdl
 task sparkjob_with_yarn_cluster {
@@ -1420,8 +1529,8 @@ Supported File Systems as follows:
 * Network File System
 * Distributed file system
 
-### Sample Wdl
-Next, create a Wdl, and it's json input like so:
+### Sample WDL
+Next, create a WDL, and its json input like so:
 
 ```wdl
 task sparkjob_with_yarn_cluster {
@@ -1775,16 +1884,15 @@ Valid keys and their meanings:
     * **google_project** - (JES backend only) Specifies which google project to execute this workflow.
     * **refresh_token** - (JES backend only) Only used if `localizeWithRefreshToken` is specified in the [configuration file](#configuring-cromwell).
     * **auth_bucket** - (JES backend only) defaults to the the value in **jes_gcs_root**.  This should represent a GCS URL that only Cromwell can write to.  The Cromwell account is determined by the `google.authScheme` (and the corresponding `google.userAuth` and `google.serviceAuth`)
-    * **monitoring_script** - (JES backend only) Specifies a GCS URL to a script that will be invoked prior to the WDL command being run.  For example, if the value for monitoring_script is "gs://bucket/script.sh", it will be invoked as `./script.sh > monitoring.log &`.  The value `monitoring.log` file will be automatically de-localized.
+    * **monitoring_script** - (JES backend only) Specifies a GCS URL to a script that will be invoked prior to the user command being run.  For example, if the value for monitoring_script is "gs://bucket/script.sh", it will be invoked as `./script.sh > monitoring.log &`.  The value `monitoring.log` file will be automatically de-localized.
 
 # Labels
 
-Every call in Cromwell is labelled by Cromwell so that it can be queried about later. The current label set automatically applied is:
+Every call run on the JES backend is given certain labels by default, so that Google resources can be queried by these labels later. The current default label set automatically applied is:
 
 | Key | Value | Example | Notes |
 |-----|-------|---------|-------|
 | cromwell-workflow-id | The Cromwell ID given to the root workflow (i.e. the ID returned by Cromwell on submission) | cromwell-d4b412c5-bf3d-4169-91b0-1b635ce47a26 | To fit the required [format](#label-format), we prefix with 'cromwell-' |
-| cromwell-workflow-name | The name of the root workflow | my-root-workflow | |
 | cromwell-sub-workflow-name | The name of this job's sub-workflow | my-sub-workflow | Only present if the task is called in a subworkflow. |
 | wdl-task-name | The name of the WDL task | my-task | |
 | wdl-call-alias | The alias of the WDL call that created this job | my-task-1 | Only present if the task was called with an alias. |
@@ -1802,10 +1910,15 @@ Custom labels can also be applied to every call in a workflow by specifying a cu
 
 ## Label Format
 
-To fit in with the Google schema for labels, label key and value strings must match the regex `[a-z]([-a-z0-9]*[a-z0-9])?` and be between 1 and 63 characters in length. 
-
-For custom labels, Cromwell will reject any request which is made containing invalid label strings. For automatically applied labels, Cromwell will modify workflow/task/call names to fit the schema, according to the following rules:
+When labels are supplied to Cromwell, it will fail any request containing invalid label strings. Below are the requirements for a valid label key/value pair in Cromwell:
+- Label keys and values can't contain characters other than `[a-z]`, `[0-9]` or `-`.
+- Label keys must start with `[a-z]` and end with `[a-z]` or `[0-9]`.
+- Label values must start and end with `[a-z]` or `[0-9]`.
+- Label keys may not be empty but label values may be empty.
+- Label key and values have a max char limit of 63.
 
+Google has a different schema for labels, where label key and value strings must match the regex `[a-z]([-a-z0-9]*[a-z0-9])?` and be no more than 63 characters in length.
+For automatically applied labels, Cromwell will modify workflow/task/call names to fit the schema, according to the following rules:
 - Any capital letters are lowercased.
 - Any character which is not one of `[a-z]`, `[0-9]` or `-` will be replaced with `-`.
 - If the start character does not match `[a-z]` then prefix with `x--`
@@ -1844,18 +1957,18 @@ Cromwell also accepts two [workflow option](#workflow-options) related to call c
 * If call caching is enabled, but one wishes to run a workflow but not add any of the calls into the call cache when they finish, the `write_to_cache` option can be set to `false`.  This value defaults to `true`.
 * If call caching is enabled, but you don't want to check the cache for any `call` invocations, set the option `read_from_cache` to `false`.  This value also defaults to `true`
 
-> **Note:** If call caching is disabled, the to workflow options `read_from_cache` and `write_to_cache` will be ignored and the options will be treated as though they were 'false'.
+> **Note:** If call caching is disabled, the workflow options `read_from_cache` and `write_to_cache` will be ignored and the options will be treated as though they were 'false'.
 
 ## Docker Tags
 
 Docker tags are a convenient way to point to a version of an image (ubuntu:14.04), or even the latest version (ubuntu:latest).
 For that purpose, tags are mutable, meaning that the image they point to can change, while the tag name stays the same.
-While this is very convenient in some cases, using mutable, or "floating" tags in WDL affects the reproducibility of the WDL file: the same WDL using "ubuntu:latest" run now, and a year, or even a month from now will actually run with different docker images.
+While this is very convenient in some cases, using mutable, or "floating" tags in tasks affects the reproducibility of a workflow: the same workflow using "ubuntu:latest" run now, and a year, or even a month from now will actually run with different docker images.
 This has an even bigger impact when Call Caching is turned on in Cromwell, and could lead to unpredictable behaviors if a tag is updated in the middle of a workflow or even a scatter for example.
 Docker provides another way of identifying an image version, using the specific digest of the image. The digest is guaranteed to be different if 2 images have different byte content. For more information see https://docs.docker.com/registry/spec/api/#/content-digests
 A docker image with digest can be referenced as follows : **ubuntu@sha256:71cd81252a3563a03ad8daee81047b62ab5d892ebbfbf71cf53415f29c130950**
 The above image refers to a specific image of ubuntu, that does not depend on a floating tag.
-A WDL containing this Docker image run now and a year from now will run in the exact same container.
+A workflow containing this Docker image run now and a year from now will run in the exact same container.
 
 In order to remove unpredictable behaviors, Cromwell takes the following approach regarding floating docker tags.
 
@@ -1865,22 +1978,33 @@ When Cromwell finds a job ready to be run, it will first look at its docker runt
 * The job does specify a docker runtime attribute:
     * The docker image uses a hash: All call caching settings apply normally
     * The docker image uses a floating tag:
-        Call caching `reading` will be disabled for this job. Specifically, Cromwell will *not* attempt to find an entry in the cache for this job.
-        Additionally, cromwell will attempt to look up the hash of the image. Upon success, it will replace the user's docker value with the hash.
-        This mechanism ensures that as long as Cromwell is able to lookup the hash, the job is guaranteed to have run on the container with that hash.
-        The docker value with the hash used for the job will be reported in the runtime attributes section of the metadata.
-        If Cromwell fails to lookup the hash (unsupported registry, wrong credentials, ...) it will run the job with the user provided floating tag.
-        If call caching writing is turned on, Cromwell will still write the job in the cache database, using:
-         * the hash if the lookup succeeded
-         * the floating tag otherwise.
-         
-Docker registry and access levels supported by Cromwell for docker hash lookup:
-
-|       |       DockerHub    ||       GCR       ||
-|:-----:|:---------:|:-------:|:------:|:-------:|
-|       |   Public  | Private | Public | Private |
-|  JES  |     X     |    X    |    X   |    X    |
-| Other |     X     |         |    X   |         |
+        * Cromwell will attempt to look up the hash of the image. Upon success it will pass both the floating tag and this hash value to the backend.
+        * All backends currently included with Cromwell will utilize this hash value to run the job.
+        * Within a single workflow all floating tags will resolve to the same hash value even if Cromwell is restarted when the workflow is running.
+        * If Cromwell fails to lookup the hash (unsupported registry, wrong credentials, ...) it will run the job with the user provided floating tag.
+        * The actual Docker image (floating tag or hash) used for the job will be reported in the `dockerImageUsed` attribute of the call metadata.
+
+### Docker Lookup
+
+Cromwell provides 2 methods to lookup a docker hash from a docker tag:
+
+* Local
+    In this mode, cromwell will first attempt to find the image on the local machine where it's running using the docker CLI. If the image is present, then its hash will be used.
+    If it's not present, cromwell will execute a `docker pull` to try and retrieve it. If this succeeds, the newly retrieved hash will be used. Otherwise the lookup will be considered failed.
+    Note that cromwell runs the `docker` CLI the same way a human would. This means two things:
+     * The machine Cromwell is running on needs to have docker installed and a docker daemon running.
+     * Whichever credentials (and only those) are available on that machine will be available to pull the image.
+    
+* Remote
+    In this mode, cromwell will attempt to retrieve the hash by contacting the remote docker registry where the image is stored. This currently supports Docker Hub and GCR.
+    
+    Docker registry and access levels supported by Cromwell for docker hash lookup in "remote" mode:
+    
+    |       |       DockerHub    ||       GCR       ||
+    |:-----:|:---------:|:-------:|:------:|:-------:|
+    |       |   Public  | Private | Public | Private |
+    |  JES  |     X     |    X    |    X   |    X    |
+    | Other |     X     |         |    X   |         |
 
 ## Local Filesystem Options
 When running a job on the Config (Shared Filesystem) backend, Cromwell provides some additional options in the backend's config section:
@@ -1916,7 +2040,7 @@ When running a job on the Config (Shared Filesystem) backend, Cromwell provides
 ```
 # Imports
 
-Import statements inside of a WDL file are supported by Cromwell when running in Server mode as well as Single Workflow Runner Mode.
+Import statements inside of a workflow file are supported by Cromwell when running in Server mode as well as Single Workflow Runner Mode.
 
 In Single Workflow Runner Mode, you pass in a zip file which includes the WDL files referenced by the import statements. Cromwell requires the zip file to be passed in as a command line argument, as explained by the section [run](#run).
 
@@ -1925,7 +2049,7 @@ For example, given a workflow `wf.wdl` and an imports directory `WdlImports.zip`
 java -jar cromwell.jar wf.wdl wf.inputs - - WdlImports.zip
 ```
 
-In Server Mode, you pass in a zip file using the parameter `wdlDependencies` via the [POST /api/workflows/:version](#post-apiworkflowsversion) endpoint.
+In Server Mode, you pass in a zip file using the parameter `workflowDependencies` via the [POST /api/workflows/:version](#post-apiworkflowsversion) endpoint.
 
 
 # Sub Workflows
@@ -2386,7 +2510,7 @@ It's also possible to set the URL query parameter `expandSubWorkflows` to `true`
 
 # REST API
 
-The `server` subcommand on the executable JAR will start an HTTP server which can accept WDL files to run as well as check status and output of existing workflows.
+The `server` subcommand on the executable JAR will start an HTTP server which can accept workflow files to run as well as check status and output of existing workflows.
 
 The following sub-sections define which HTTP Requests the web server can accept and what they will return.  Example HTTP requests are given in [HTTPie](https://github.com/jkbrzt/httpie) and [cURL](https://curl.haxx.se/)
 
@@ -2398,12 +2522,12 @@ All web server requests include an API version in the url. The current version i
 
 This endpoint accepts a POST request with a `multipart/form-data` encoded body.  The form fields that may be included are:
 
-* `wdlSource` - *Required* Contains the WDL file to submit for execution.
-* `workflowInputs` - *Optional* JSON file containing the inputs.  A skeleton file can be generated from [wdltool](https://github.com/broadinstitute/wdltool) using the "inputs" subcommand.
+* `workflowSource` - *Required* Contains the workflow source file to submit for execution.
+* `workflowInputs` - *Optional* JSON file containing the inputs.  For WDL workflows a skeleton file can be generated from [wdltool](https://github.com/broadinstitute/wdltool) using the "inputs" subcommand.
 * `workflowInputs_n` - *Optional* Where `n` is an integer. JSON file containing the 'n'th set of auxiliary inputs.
 * `workflowOptions` - *Optional* JSON file containing options for this workflow execution.  See the [run](#run) CLI sub-command for some more information about this.
 * `customLabels` - *Optional* JSON file containing a set of custom labels to apply to this workflow. See [Labels](#labels) for the expected format.
-* `wdlDependencies` - *Optional* ZIP file containing WDL files that are used to resolve import statements.
+* `workflowDependencies` - *Optional* ZIP file containing workflow source files that are used to resolve import statements.
 
 Regarding the workflowInputs parameter, in case of key conflicts between multiple input JSON files, higher values of x in workflowInputs_x override lower values. For example, an input specified in workflowInputs_3 will override an input with the same name in workflowInputs or workflowInputs_2.
 Similarly, an input key specified in workflowInputs_5 will override an identical input key in any other input file.
@@ -2414,13 +2538,13 @@ Additionally, although Swagger has a limit of 5 JSON input files, the REST endpo
 cURL:
 
 ```
-$ curl -v "localhost:8000/api/workflows/v1" -F wdlSource=@src/main/resources/3step.wdl -F workflowInputs=@test.json
+$ curl -v "localhost:8000/api/workflows/v1" -F workflowSource=@src/main/resources/3step.wdl -F workflowInputs=@test.json
 ```
 
 HTTPie:
 
 ```
-$ http --print=hbHB --form POST localhost:8000/api/workflows/v1 wdlSource=@src/main/resources/3step.wdl workflowInputs@inputs.json
+$ http --print=hbHB --form POST localhost:8000/api/workflows/v1 workflowSource=@src/main/resources/3step.wdl workflowInputs@inputs.json
 ```
 
 Request:
@@ -2436,7 +2560,7 @@ Host: localhost:8000
 User-Agent: HTTPie/0.9.2
 
 --64128d499e9e4616adea7d281f695dca
-Content-Disposition: form-data; name="wdlSource"
+Content-Disposition: form-data; name="workflowSource"
 
 task ps {
   command {
@@ -2506,13 +2630,13 @@ To specify workflow options as well:
 cURL:
 
 ```
-$ curl -v "localhost:8000/api/workflows/v1" -F wdlSource=@wdl/jes0.wdl -F workflowInputs=@wdl/jes0.json -F workflowOptions=@options.json
+$ curl -v "localhost:8000/api/workflows/v1" -F workflowSource=@wdl/jes0.wdl -F workflowInputs=@wdl/jes0.json -F workflowOptions=@options.json
 ```
 
 HTTPie:
 
 ```
-http --print=HBhb --form POST http://localhost:8000/api/workflows/v1 wdlSource=@wdl/jes0.wdl workflowInputs@wdl/jes0.json workflowOptions@options.json
+http --print=HBhb --form POST http://localhost:8000/api/workflows/v1 workflowSource=@wdl/jes0.wdl workflowInputs@wdl/jes0.json workflowOptions@options.json
 ```
 
 Request (some parts truncated for brevity):
@@ -2528,7 +2652,7 @@ Host: localhost:8000
 User-Agent: HTTPie/0.9.2
 
 --f3fd038395644de596c460257626edd7
-Content-Disposition: form-data; name="wdlSource"
+Content-Disposition: form-data; name="workflowSource"
 
 task x { ... }
 task y { ... }
@@ -2564,28 +2688,28 @@ Content-Disposition: form-data; name="workflowOptions"; filename="options.json"
 This endpoint accepts a POST request with a `multipart/form-data`
 encoded body.  The form fields that may be included are:
 
-* `wdlSource` - *Required* Contains the WDL file to submit for
+* `workflowSource` - *Required* Contains the workflow source file to submit for
 execution.
 * `workflowInputs` - *Required* JSON file containing the inputs in a
-JSON array. A skeleton file for a single inputs json element can be
+JSON array. For WDL workflows a skeleton file for a single inputs json element can be
 generated from [wdltool](https://github.com/broadinstitute/wdltool)
 using the "inputs" subcommand. The orderded endpoint responses will
 contain one workflow submission response for each input, respectively.
 * `workflowOptions` - *Optional* JSON file containing options for this
 workflow execution.  See the [run](#run) CLI sub-command for some more
 information about this.
-* `wdlDependencies` - *Optional* ZIP file containing WDL files that are used to resolve import statements. Applied equally to all workflowInput sets.
+* `workflowDependencies` - *Optional* ZIP file containing workflow source files that are used to resolve import statements. Applied equally to all workflowInput sets.
 
 cURL:
 
 ```
-$ curl -v "localhost:8000/api/workflows/v1/batch" -F wdlSource=@src/main/resources/3step.wdl -F workflowInputs=@test_array.json
+$ curl -v "localhost:8000/api/workflows/v1/batch" -F workflowSource=@src/main/resources/3step.wdl -F workflowInputs=@test_array.json
 ```
 
 HTTPie:
 
 ```
-$ http --print=hbHB --form POST localhost:8000/api/workflows/v1/batch wdlSource=@src/main/resources/3step.wdl workflowInputs@inputs_array.json
+$ http --print=hbHB --form POST localhost:8000/api/workflows/v1/batch workflowSource=@src/main/resources/3step.wdl workflowInputs@inputs_array.json
 ```
 
 Request:
@@ -2601,7 +2725,7 @@ Host: localhost:8000
 User-Agent: HTTPie/0.9.2
 
 --64128d499e9e4616adea7d281f695dcb
-Content-Disposition: form-data; name="wdlSource"
+Content-Disposition: form-data; name="workflowSource"
 
 task ps {
   command {
@@ -2682,13 +2806,13 @@ To specify workflow options as well:
 cURL:
 
 ```
-$ curl -v "localhost:8000/api/workflows/v1/batch" -F wdlSource=@wdl/jes0.wdl -F workflowInputs=@wdl/jes0_array.json -F workflowOptions=@options.json
+$ curl -v "localhost:8000/api/workflows/v1/batch" -F workflowSource=@wdl/jes0.wdl -F workflowInputs=@wdl/jes0_array.json -F workflowOptions=@options.json
 ```
 
 HTTPie:
 
 ```
-http --print=HBhb --form POST http://localhost:8000/api/workflows/v1/batch wdlSource=@wdl/jes0.wdl workflowInputs@wdl/jes0_array.json workflowOptions@options.json
+http --print=HBhb --form POST http://localhost:8000/api/workflows/v1/batch workflowSource=@wdl/jes0.wdl workflowInputs@wdl/jes0_array.json workflowOptions@options.json
 ```
 
 Request (some parts truncated for brevity):
@@ -2704,7 +2828,7 @@ Host: localhost:8000
 User-Agent: HTTPie/0.9.2
 
 --f3fd038395644de596c460257626edd8
-Content-Disposition: form-data; name="wdlSource"
+Content-Disposition: form-data; name="workflowSource"
 
 task x { ... }
 task y { ... }
@@ -2747,6 +2871,7 @@ This endpoint allows for querying workflows based on the following criteria:
 * `name`
 * `id`
 * `status`
+* `label`
 * `start` (start datetime with mandatory offset)
 * `end` (end datetime with mandatory offset)
 * `page` (page of results)
@@ -2756,9 +2881,13 @@ Names, ids, and statuses can be given multiple times to include
 workflows with any of the specified names, ids, or statuses. When
 multiple names are specified, any workflow matching one of the names
 will be returned. The same is true for multiple ids or statuses. When
-different types of criteria are specified, for example names and
-statuses, the results must match both the one of the specified names and
-one of the statuses. Using page and pagesize will enable server side pagination.
+more than one label is specified, only workflows associated to all of
+the given labels will be returned. 
+
+When a combination of criteria are specified, for example querying by 
+names and statuses, the results must return workflows that match one of 
+the specified names and one of the statuses. Using page and pagesize will
+enable server side pagination.
 
 Valid statuses are `Submitted`, `Running`, `Aborting`, `Aborted`, `Failed`, and `Succeeded`.  `start` and `end` should
 be in [ISO8601 datetime](https://en.wikipedia.org/wiki/ISO_8601) format with *mandatory offset* and `start` cannot be after `end`.
@@ -2810,6 +2939,54 @@ Server: spray-can/1.3.3
 }
 ```
 
+Labels have to be queried in key and value pairs separated by a colon, i.e. `label-key:label-value`. For example, if a batch of workflows was submitted with the following labels JSON:
+```
+{
+  "label-key-1" : "label-value-1",
+  "label-key-2" : "label-value-2"
+}
+```
+
+A request to query for succeeded workflows with both labels would be:
+
+cURL:
+```
+$ curl "http://localhost:8000/api/workflows/v1/query?status=Succeeded&label=label-key-1:label-value-1&label=label-key-2:label-value-2
+```
+
+HTTPie:
+```
+$ http "http://localhost:8000/api/workflows/v1/query?status=Succeeded&label=label-key-1:label-value-1&label=label-key-2:label-value-2
+```
+
+Response:
+```
+HTTP/1.1 200 OK
+Content-Length: 608
+Content-Type: application/json; charset=UTF-8
+Date: Tue, 9 May 2017 20:24:33 GMT
+Server: spray-can/1.3.3
+
+{
+    "results": [
+        {
+            "end": "2017-05-09T16:07:30.515-04:00", 
+            "id": "83fc23d5-48d1-456e-997a-087e55cd2e06", 
+            "name": "wf_hello", 
+            "start": "2017-05-09T16:01:51.940-04:00", 
+            "status": "Succeeded"
+        }, 
+        {
+            "end": "2017-05-09T16:07:13.174-04:00", 
+            "id": "7620a5c6-a5c6-466c-994b-dd8dca917b9b", 
+            "name": "wf_goodbye", 
+            "start": "2017-05-09T16:01:51.939-04:00", 
+            "status": "Succeeded"
+        }
+    ]
+}
+```
+
 Query data is refreshed from raw data periodically according to the configuration value `services.MetadataService.metadata-summary-refresh-interval`.
 This interval represents the duration between the end of one summary refresh sweep and the beginning of the next sweep.  If not specified the
 refresh interval will default to 2 seconds.  To turn off metadata summary refresh, specify an infinite refresh interval value with "Inf".
@@ -2881,6 +3058,37 @@ Server: spray-can/1.3.3
 }
 ```
 
+## PATCH /api/workflows/:version/:id/labels
+
+This endpoint is used to update multiple labels for an existing workflow. When supplying a label with a key unique to the workflow submission, a new label key/value entry is appended to that workflow's metadata. When supplying a label with a key that is already associated to the workflow submission, the original label value is updated with the new value for that workflow's metadata.
+
+The [labels](#labels) must be a mapping of key/value pairs in JSON format that are sent via the PATCH body. The request content type must be
+`application/json`.
+
+cURL:
+
+```
+$ curl -X PATCH --header "Content-Type: application/json" -d "{\"label-key-1\":\"label-value-1\", \"label-key-2\": \"label-value-2\"}" "http://localhost:8000/api/workflows/v1/c4c6339c-8cc9-47fb-acc5-b5cb8d2809f5/labels"
+```
+
+HTTPie:
+
+```
+$ echo '{"label-key-1":"label-value-1", "label-key-2": "label-value-2"}' | http PATCH "http://localhost:8000/api/workflows/v1/c4c6339c-8cc9-47fb-acc5-b5cb8d2809f5/labels"
+```
+
+Response:
+```
+{ "id": "c4c6339c-8cc9-47fb-acc5-b5cb8d2809f5",
+  "labels":
+    {
+      "label-key-1": "label-value-1",
+      "label-key-2": "label-value-2"
+    }
+}
+```
+
+
 ## GET /api/workflows/:version/:id/status
 
 cURL:
@@ -3043,11 +3251,18 @@ Content-Type: application/json; charset=UTF-8
 Content-Length: 7286
 {
   "workflowName": "sc_test",
+  "submittedFiles": {
+      "inputs": "{}",
+      "workflow": "task do_prepare {\n    File input_file\n    command {\n        split -l 1 ${input_file} temp_ && ls -1 temp_?? > files.list\n    }\n    output {\n        Array[File] split_files = read_lines(\"files.list\")\n    }\n}\n# count the number of words in the input file, writing the count to an output file overkill in this case, but simulates a real scatter-gather that would just return an Int (map)\ntask do_scatter {\n    File input_file\n    command {\n        wc -w ${input_file} > output.txt\n    }\n    output {\n        File count_file = \"output.txt\"\n    }\n}\n# aggregate the results back together (reduce)\ntask do_gather {\n    Array[File] input_files\n    command <<<\n        cat ${sep = ' ' input_files} | awk '{s+=$$1} END {print s}'\n    >>>\n    output {\n        Int sum = read_int(stdout())\n    }\n}\nworkflow sc_test {\n    call do_prepare\n    scatter(f in do_prepare.split_files) {\n        call do_scatter {\n            input: input_file = f\n        }\n    }\n    call do_gather {\n        input: input_files = do_scatter.count_file\n    }\n}",
+      "options": "{\n\n}",
+      "workflowType": "WDL"
+  },
   "calls": {
     "sc_test.do_prepare": [
       {
         "executionStatus": "Done",
         "stdout": "/home/jdoe/cromwell/cromwell-executions/sc_test/8e592ed8-ebe5-4be0-8dcb-4073a41fe180/call-do_prepare/stdout",
+        "backendStatus": "Done",
         "shardIndex": -1,
         "outputs": {
           "split_files": [
@@ -3062,6 +3277,30 @@ Content-Length: 7286
             "failOnStderr": "true",
             "continueOnReturnCode": "0"
         },
+        "callCaching": {
+            "allowResultReuse": true,
+            "hit": false,
+            "result": "Cache Miss",
+            "hashes": {
+              "output count": "C4CA4238A0B923820DCC509A6F75849B",
+              "runtime attribute": {
+                "docker": "N/A",
+                "continueOnReturnCode": "CFCD208495D565EF66E7DFF9F98764DA",
+                "failOnStderr": "68934A3E9455FA72420237EB05902327"
+              },
+              "output expression": {
+                "Array": "D856082E6599CF6EC9F7F42013A2EC4C"
+              },
+              "input count": "C4CA4238A0B923820DCC509A6F75849B",
+              "backend name": "509820290D57F333403F490DDE7316F4",
+              "command template": "9F5F1F24810FACDF917906BA4EBA807D",
+              "input": {
+                "File input_file": "11fa6d7ed15b42f2f73a455bf5864b49"
+              }
+            },
+            "effectiveCallCachingMode": "ReadAndWriteCache"
+        },
+        "jobId": "34479",
         "returnCode": 0,
         "backend": "Local",
         "end": "2016-02-04T13:47:56.000-05:00",
@@ -3075,15 +3314,40 @@ Content-Length: 7286
       {
         "executionStatus": "Preempted",
         "stdout": "/home/jdoe/cromwell/cromwell-executions/sc_test/8e592ed8-ebe5-4be0-8dcb-4073a41fe180/call-do_scatter/shard-0/stdout",
+        "backendStatus": "Preempted",
         "shardIndex": 0,
         "outputs": {},
         "runtimeAttributes": {
            "failOnStderr": "true",
            "continueOnReturnCode": "0"
         },
+        "callCaching": {
+          "allowResultReuse": true,
+          "hit": false,
+          "result": "Cache Miss",
+          "hashes": {
+            "output count": "C4CA4238A0B923820DCC509A6F75849B",
+            "runtime attribute": {
+              "docker": "N/A",
+              "continueOnReturnCode": "CFCD208495D565EF66E7DFF9F98764DA",
+              "failOnStderr": "68934A3E9455FA72420237EB05902327"
+            },
+            "output expression": {
+              "File count_file": "EF1B47FFA9990E8D058D177073939DF7"
+            },
+            "input count": "C4CA4238A0B923820DCC509A6F75849B",
+            "backend name": "509820290D57F333403F490DDE7316F4",
+            "command template": "FD00A1B0AB6A0C97B0737C83F179DDE7",
+            "input": {
+              "File input_file": "a53794d214dc5dedbcecdf827bf683a2"
+            }
+          },
+         "effectiveCallCachingMode": "ReadAndWriteCache"
+        },
         "inputs": {
           "input_file": "f"
         },
+        "jobId": "34496",
         "backend": "Local",
         "end": "2016-02-04T13:47:56.000-05:00",
         "stderr": "/home/jdoe/cromwell/cromwell-executions/sc_test/8e592ed8-ebe5-4be0-8dcb-4073a41fe180/call-do_scatter/shard-0/stderr",
@@ -3094,6 +3358,7 @@ Content-Length: 7286
       {
         "executionStatus": "Done",
         "stdout": "/home/jdoe/cromwell/cromwell-executions/sc_test/8e592ed8-ebe5-4be0-8dcb-4073a41fe180/call-do_scatter/shard-0/attempt-2/stdout",
+        "backendStatus": "Done",
         "shardIndex": 0,
         "outputs": {
           "count_file": "/home/jdoe/cromwell/cromwell-test-executions/sc_test/8e592ed8-ebe5-4be0-8dcb-4073a41fe180/call-do_scatter/shard-0/attempt-2/output.txt"
@@ -3102,10 +3367,34 @@ Content-Length: 7286
            "failOnStderr": "true",
            "continueOnReturnCode": "0"
         },
+        "callCaching": {
+          "allowResultReuse": true,
+          "hit": false,
+          "result": "Cache Miss",
+          "hashes": {
+            "output count": "C4CA4238A0B923820DCC509A6F75849B",
+            "runtime attribute": {
+              "docker": "N/A",
+              "continueOnReturnCode": "CFCD208495D565EF66E7DFF9F98764DA",
+              "failOnStderr": "68934A3E9455FA72420237EB05902327"
+            },
+            "output expression": {
+              "File count_file": "EF1B47FFA9990E8D058D177073939DF7"
+            },
+            "input count": "C4CA4238A0B923820DCC509A6F75849B",
+            "backend name": "509820290D57F333403F490DDE7316F4",
+            "command template": "FD00A1B0AB6A0C97B0737C83F179DDE7",
+            "input": {
+              "File input_file": "a53794d214dc5dedbcecdf827bf683a2"
+            }
+          },
+         "effectiveCallCachingMode": "ReadAndWriteCache"
+        },
         "inputs": {
           "input_file": "f"
         },
         "returnCode": 0,
+        "jobId": "34965",
         "end": "2016-02-04T13:47:56.000-05:00",
         "stderr": "/home/jdoe/cromwell/cromwell-executions/sc_test/8e592ed8-ebe5-4be0-8dcb-4073a41fe180/call-do_scatter/shard-0/attempt-2/stderr",
         "attempt": 2,
@@ -3115,6 +3404,7 @@ Content-Length: 7286
       {
         "executionStatus": "Done",
         "stdout": "/home/jdoe/cromwell/cromwell-executions/sc_test/8e592ed8-ebe5-4be0-8dcb-4073a41fe180/call-do_scatter/shard-1/stdout",
+        "backendStatus": "Done",
         "shardIndex": 1,
         "outputs": {
           "count_file": "/home/jdoe/cromwell/cromwell-test-executions/sc_test/8e592ed8-ebe5-4be0-8dcb-4073a41fe180/call-do_scatter/shard-1/output.txt"
@@ -3123,10 +3413,34 @@ Content-Length: 7286
            "failOnStderr": "true",
            "continueOnReturnCode": "0"
         },
+        "callCaching": {
+          "allowResultReuse": true,
+          "hit": false,
+          "result": "Cache Miss",
+          "hashes": {
+            "output count": "C4CA4238A0B923820DCC509A6F75849B",
+            "runtime attribute": {
+              "docker": "N/A",
+              "continueOnReturnCode": "CFCD208495D565EF66E7DFF9F98764DA",
+              "failOnStderr": "68934A3E9455FA72420237EB05902327"
+            },
+            "output expression": {
+              "File count_file": "EF1B47FFA9990E8D058D177073939DF7"
+            },
+            "input count": "C4CA4238A0B923820DCC509A6F75849B",
+            "backend name": "509820290D57F333403F490DDE7316F4",
+            "command template": "FD00A1B0AB6A0C97B0737C83F179DDE7",
+            "input": {
+              "File input_file": "d3410ade53df34c78488544285cf743c"
+            }
+          },
+         "effectiveCallCachingMode": "ReadAndWriteCache"
+        },
         "inputs": {
           "input_file": "f"
         },
         "returnCode": 0,
+        "jobId": "34495",
         "backend": "Local",
         "end": "2016-02-04T13:47:56.000-05:00",
         "stderr": "/home/jdoe/cromwell/cromwell-executions/sc_test/8e592ed8-ebe5-4be0-8dcb-4073a41fe180/call-do_scatter/shard-1/stderr",
@@ -3139,6 +3453,7 @@ Content-Length: 7286
       {
         "executionStatus": "Done",
         "stdout": "/home/jdoe/cromwell/cromwell-executions/sc_test/8e592ed8-ebe5-4be0-8dcb-4073a41fe180/call-do_gather/stdout",
+        "backendStatus": "Done",
         "shardIndex": -1,
         "outputs": {
           "sum": 12
@@ -3147,6 +3462,29 @@ Content-Length: 7286
            "failOnStderr": "true",
            "continueOnReturnCode": "0"
         },
+        "callCaching": {
+          "allowResultReuse": true,
+          "hit": false,
+          "result": "Cache Miss",
+          "hashes": {
+            "output count": "C4CA4238A0B923820DCC509A6F75849B",
+            "runtime attribute": {
+              "docker": "N/A",
+              "continueOnReturnCode": "CFCD208495D565EF66E7DFF9F98764DA",
+              "failOnStderr": "68934A3E9455FA72420237EB05902327"
+            },
+            "output expression": {
+              "File count_file": "EF1B47FFA9990E8D058D177073939DF7"
+            },
+            "input count": "C4CA4238A0B923820DCC509A6F75849B",
+            "backend name": "509820290D57F333403F490DDE7316F4",
+            "command template": "FD00A1B0AB6A0C97B0737C83F179DDE7",
+            "input": {
+              "File input_file": "e0ef752ab4824939d7947f6012b7c141"
+            }
+          },
+          "effectiveCallCachingMode": "ReadAndWriteCache"
+        },
         "inputs": {
           "input_files": [
             "/home/jdoe/cromwell/cromwell-test-executions/sc_test/8e592ed8-ebe5-4be0-8dcb-4073a41fe180/call-do_scatter/shard-0/attempt-2/output.txt",
@@ -3154,6 +3492,7 @@ Content-Length: 7286
           ]
         },
         "returnCode": 0,
+        "jobId": "34494",
         "backend": "Local",
         "end": "2016-02-04T13:47:57.000-05:00",
         "stderr": "/home/jdoe/cromwell/cromwell-executions/sc_test/8e592ed8-ebe5-4be0-8dcb-4073a41fe180/call-do_gather/stderr",
@@ -3178,6 +3517,10 @@ Content-Length: 7286
   "inputs": {
     "sc_test.do_prepare.input_file": "/home/jdoe/cromwell/11.txt"
   },
+  "labels": {
+    "cromwell-workflow-name": "sc_test",
+    "cromwell-workflow-id": "cromwell-17633f21-11a9-414f-a95b-2e21431bd67d"
+  },
   "submission": "2016-02-04T13:47:55.000-05:00",
   "status": "Succeeded",
   "end": "2016-02-04T13:47:57.000-05:00",
@@ -3296,6 +3639,16 @@ The `call` and `workflow` may optionally contain failures shaped like this:
 ]
 ```
 
+### Compressing the metadata response
+
+The response from the metadata endpoint can be quite large depending on the workflow. To help with this Cromwell supports gzip encoding the metadata prior to sending it back to the client. In order to enable this, make sure your client is sending the `Accept-Encoding: gzip` header.
+
+For instance, with cURL:
+                   
+```
+$ curl -H "Accept-Encoding: gzip" http://localhost:8000/api/workflows/v1/b3e45584-9450-4e73-9523-fc3ccf749848/metadata
+```
+
 ## POST /api/workflows/:version/:id/abort
 
 cURL:
@@ -3352,18 +3705,186 @@ Server: spray-can/1.3.3
 }
 ```
 
-## GET /api/engine/:version/stats
+## GET /api/workflows/:version/callcaching/diff
+
+**Disclaimer**: This endpoint depends on hash values being published to the metadata, which only happens as of Cromwell 28.
+Workflows run with prior versions of Cromwell cannot be used with this endpoint.
+A `404 NotFound` will be returned when trying to use this endpoint if either workflow has been run on a prior version.
+
+This endpoint returns the hash differences between 2 *completed* (successfully or not) calls.
+The following query parameters are supported:
+
+| Parameter |                                        Description                                        | Required |
+|:---------:|:-----------------------------------------------------------------------------------------:|:--------:|
+| workflowA | Workflow ID of the first call                                                             | yes      |
+| callA     | Fully qualified name of the first call. **Including workflow name**. (see example below)  | yes      |
+| indexA    | Shard index of the first call                                                             | depends  |
+| workflowB | Workflow ID of the second call                                                            | yes      |
+| callB     | Fully qualified name of the second call. **Including workflow name**. (see example below) | yes      |
+| indexB    | Shard index of the second call                                                            | depends  |
+
+About the `indexX` parameters: It is required if the call was in a scatter. Otherwise it should *not* be specified.
+If an index parameter is wrongly specified, the call will not be found and the request will result in a 404 response.
+
+cURL:
+
+```
+$ curl "http://localhost:8000/api/workflows/v1/callcaching/diff?workflowA=85174842-4a44-4355-a3a9-3a711ce556f1&callA=wf_hello.hello&workflowB=7479f8a8-efa4-46e4-af0d-802addc66e5d&callB=wf_hello.hello"
+```
+
+HTTPie:
+
+```
+$ http "http://localhost:8000/api/workflows/v1/callcaching/diff?workflowA=85174842-4a44-4355-a3a9-3a711ce556f1&callA=wf_hello.hello&workflowB=7479f8a8-efa4-46e4-af0d-802addc66e5d&callB=wf_hello.hello"
+```
+
+Response:
+```
+HTTP/1.1 200 OK
+Content-Length: 1274
+Content-Type: application/json; charset=UTF-8
+Date: Tue, 06 Jun 2017 16:44:33 GMT
+Server: spray-can/1.3.3
+
+{
+  "callA": {
+    "executionStatus": "Done",
+    "workflowId": "85174842-4a44-4355-a3a9-3a711ce556f1",
+    "callFqn": "wf_hello.hello",
+    "jobIndex": -1,
+    "allowResultReuse": true
+  },
+  "callB": {
+    "executionStatus": "Done",
+    "workflowId": "7479f8a8-efa4-46e4-af0d-802addc66e5d",
+    "callFqn": "wf_hello.hello",
+    "jobIndex": -1,
+    "allowResultReuse": true
+  },
+  "hashDifferential": [
+    {
+      "hashKey": "command template",
+      "callA": "4EAADE3CD5D558C5A6CFA4FD101A1486",
+      "callB": "3C7A0CA3D7A863A486DBF3F7005D4C95"
+    },
+    {
+      "hashKey": "input count",
+      "callA": "C4CA4238A0B923820DCC509A6F75849B",
+      "callB": "C81E728D9D4C2F636F067F89CC14862C"
+    },
+    {
+      "hashKey": "input: String addressee",
+      "callA": "D4CC65CB9B5F22D8A762532CED87FE8D",
+      "callB": "7235E005510D99CB4D5988B21AC97B6D"
+    },
+    {
+      "hashKey": "input: String addressee2",
+      "callA": "116C7E36B4AE3EAFD07FA4C536CE092F",
+      "callB": null
+    }
+  ]
+}
+```
+
+The response is a JSON object with 3 fields:
+
+- `callA` reports information about the first call, including its `allowResultReuse` value that will be used to determine whether or not this call can be cached to.
+- `callB` reports information about the second call, including its `allowResultReuse` value that will be used to determine whether or not this call can be cached to.
+- `hashDifferential` is an array in which each element represents a difference between the hashes of `callA` and `callB`.
+
+*If this array is empty, `callA` and `callB` have the same hashes*.
+
+Differences can be of 3 kinds:
+
+- `callA` and `callB` both have the same hash key but their values are different.
+For instance, in the example above, 
+
+```json
+{
+  "hashKey": "input: String addressee",
+  "callA": "D4CC65CB9B5F22D8A762532CED87FE8D",
+  "callB": "7235E005510D99CB4D5988B21AC97B6D"
+}
+```
+
+indicates that both `callA` and `callB` have a `String` input called `addressee`, but different values were used at runtime, resulting in different MD5 hashes.
+
+- `callA` has a hash key that `callB` doesn't have
+For instance, in the example above, 
+
+```json
+{
+  "hashKey": "input: String addressee2",
+  "callA": "116C7E36B4AE3EAFD07FA4C536CE092F",
+  "callB": null
+}
+```
+
+indicates that `callA` has a `String` input called `addressee2` that doesn't exist in `callB`. For that reason the value of the second field is `null`.
+
+- `callB` has a hash key that `callA` doesn't have. This is the same case as above but reversed.
+
+If no cache entry for `callA` or `callB` can be found, the response will be in the following format:
+
+```
+HTTP/1.1 404 NotFound
+Content-Length: 178
+Content-Type: application/json; charset=UTF-8
+Date: Tue, 06 Jun 2017 17:02:15 GMT
+Server: spray-can/1.3.3
+
+{
+  "status": "error",
+  "message": "Cannot find call 479f8a8-efa4-46e4-af0d-802addc66e5d:wf_hello.hello:-1"
+}
+```
+
+If neither `callA` nor `callB` can be found, the response will be in the following format:
+
+
+```
+HTTP/1.1 404 NotFound
+Content-Length: 178
+Content-Type: application/json; charset=UTF-8
+Date: Tue, 06 Jun 2017 17:02:15 GMT
+Server: spray-can/1.3.3
+
+{
+  "status": "error",
+  "message": "Cannot find calls 5174842-4a44-4355-a3a9-3a711ce556f1:wf_hello.hello:-1, 479f8a8-efa4-46e4-af0d-802addc66e5d:wf_hello.hello:-1"
+}
+```
+
+If the query is malformed and required parameters are missing, the response will be in the following format:
+
+```
+HTTP/1.1 400 BadRequest
+Content-Length: 178
+Content-Type: application/json; charset=UTF-8
+Date: Tue, 06 Jun 2017 17:02:15 GMT
+Server: spray-can/1.3.3
+{
+  "status": "fail",
+  "message": "Wrong parameters for call cache diff query:\nmissing workflowA query parameter\nmissing callB query parameter",
+  "errors": [
+    "missing workflowA query parameter",
+    "missing callB query parameter"
+  ]
+}
+```
+
+## GET /engine/:version/stats
 
 This endpoint returns some basic statistics on the current state of the engine. At the moment that includes the number of running workflows and the number of active jobs. 
 
 cURL:
 ```
-$ curl http://localhost:8000/api/engine/v1/stats
+$ curl http://localhost:8000/engine/v1/stats
 ```
 
 HTTPie:
 ```
-$ http http://localhost:8000/api/engine/v1/stats
+$ http http://localhost:8000/engine/v1/stats
 ```
 
 Response:
@@ -3379,18 +3900,18 @@ Response:
 }
 ```
 
-## GET /api/engine/:version/version
+## GET /engine/:version/version
 
 This endpoint returns the version of the Cromwell engine.
 
 cURL:
 ```
-$ curl http://localhost:8000/api/engine/v1/version
+$ curl http://localhost:8000/engine/v1/version
 ```
 
 HTTPie:
 ```
-$ http http://localhost:8000/api/engine/v1/version
+$ http http://localhost:8000/engine/v1/version
 ```
 
 Response:
@@ -3405,8 +3926,6 @@ Response:
 }
 ```
 
-
-
 ## Error handling
 Requests that Cromwell can't process return a failure in the form of a JSON response respecting the following JSON schema:
 
@@ -3460,25 +3979,3 @@ e.g.
 The `message` field contains a short description of the error.
 
 The `errors` field is optional and may contain additional information about why the request failed.
-
-# Developer
-
-## Generating table of contents on Markdown files
-
-```
-$ pip install mdtoc
-$ mdtoc --check-links README.md
-```
-
-## Generating and Hosting ScalaDoc
-
-Essentially run `sbt doc` then commit the generated code into the `gh-pages` branch on this repository
-
-```
-$ sbt doc
-$ git co gh-pages
-$ mv target/scala-2.11/api scaladoc
-$ git add scaladoc
-$ git commit -m "API Docs"
-$ git push origin gh-pages
-```
diff --git a/backend/CHANGELOG.MD b/backend/CHANGELOG.MD
deleted file mode 100644
index 6e6cfdff0..000000000
--- a/backend/CHANGELOG.MD
+++ /dev/null
@@ -1,5 +0,0 @@
-# Cromwell-Backend Change Log
-
-1.0:
-=====
-
diff --git a/backend/src/main/scala/cromwell/backend/BackendCacheHitCopyingActor.scala b/backend/src/main/scala/cromwell/backend/BackendCacheHitCopyingActor.scala
index f1bf38386..b832e8400 100644
--- a/backend/src/main/scala/cromwell/backend/BackendCacheHitCopyingActor.scala
+++ b/backend/src/main/scala/cromwell/backend/BackendCacheHitCopyingActor.scala
@@ -1,34 +1,7 @@
 package cromwell.backend
 
-import akka.actor.{Actor, ActorLogging}
-import akka.event.LoggingReceive
-import cromwell.backend.BackendCacheHitCopyingActor.CopyOutputsCommand
-import cromwell.backend.BackendJobExecutionActor.{AbortedResponse, BackendJobExecutionResponse, JobFailedNonRetryableResponse}
-import cromwell.backend.BackendLifecycleActor._
 import cromwell.core.simpleton.WdlValueSimpleton
 
-import scala.concurrent.Future
-
 object BackendCacheHitCopyingActor {
   final case class CopyOutputsCommand(wdlValueSimpletons: Seq[WdlValueSimpleton], jobDetritusFiles: Map[String, String], returnCode: Option[Int])
 }
-
-trait BackendCacheHitCopyingActor extends Actor with ActorLogging with BackendJobLifecycleActor {
-
-  def copyCachedOutputs(wdlValueSimpletons: Seq[WdlValueSimpleton], jobDetritusFiles: Map[String, String], returnCode: Option[Int]): BackendJobExecutionResponse
-
-  def receive: Receive = LoggingReceive {
-    case CopyOutputsCommand(simpletons, jobDetritus, returnCode) =>
-      performActionThenRespond(Future(copyCachedOutputs(simpletons, jobDetritus, returnCode)), onFailure = cachingFailed, andThen = context stop self)
-    case AbortJobCommand =>
-      abort()
-      context.parent ! AbortedResponse(jobDescriptor.key)
-      context stop self
-  }
-
-  def abort(): Unit = log.warning("{}: Abort not supported during cache hit copying", jobTag)
-
-  private def cachingFailed(t: Throwable) = {
-    JobFailedNonRetryableResponse(jobKey = jobDescriptor.key, throwable = t, returnCode = None)
-  }
-}
diff --git a/backend/src/main/scala/cromwell/backend/BackendJobBreadCrumb.scala b/backend/src/main/scala/cromwell/backend/BackendJobBreadCrumb.scala
index 3fa6e9c19..c6ddcbcc4 100644
--- a/backend/src/main/scala/cromwell/backend/BackendJobBreadCrumb.scala
+++ b/backend/src/main/scala/cromwell/backend/BackendJobBreadCrumb.scala
@@ -3,9 +3,9 @@ package cromwell.backend
 import cromwell.backend.io.JobPaths
 import cromwell.core.path.Path
 import cromwell.core.{JobKey, WorkflowId}
-import wdl4s.Workflow
+import wdl4s.wdl.WdlWorkflow
 
-case class BackendJobBreadCrumb(workflow: Workflow, id: WorkflowId, jobKey: JobKey) {
+case class BackendJobBreadCrumb(workflow: WdlWorkflow, id: WorkflowId, jobKey: JobKey) {
   def toPath(root: Path): Path = {
     val workflowPart = root.resolve(workflow.unqualifiedName).resolve(id.toString)
     JobPaths.callPathBuilder(workflowPart, jobKey)
diff --git a/backend/src/main/scala/cromwell/backend/BackendJobExecutionActor.scala b/backend/src/main/scala/cromwell/backend/BackendJobExecutionActor.scala
index ffeccd7cd..068859172 100644
--- a/backend/src/main/scala/cromwell/backend/BackendJobExecutionActor.scala
+++ b/backend/src/main/scala/cromwell/backend/BackendJobExecutionActor.scala
@@ -7,8 +7,8 @@ import cromwell.backend.BackendLifecycleActor._
 import cromwell.backend.wdl.OutputEvaluator
 import cromwell.core.path.Path
 import cromwell.core.{CallOutputs, ExecutionEvent, JobKey}
-import wdl4s.expression.WdlStandardLibraryFunctions
-import wdl4s.values.WdlValue
+import wdl4s.wdl.expression.WdlStandardLibraryFunctions
+import wdl4s.wdl.values.WdlValue
 
 import scala.concurrent.Future
 import scala.util.{Success, Try}
@@ -24,7 +24,7 @@ object BackendJobExecutionActor {
   sealed trait BackendJobExecutionActorResponse extends BackendWorkflowLifecycleActorResponse
 
   sealed trait BackendJobExecutionResponse extends BackendJobExecutionActorResponse { def jobKey: JobKey }
-  case class JobSucceededResponse(jobKey: BackendJobDescriptorKey, returnCode: Option[Int], jobOutputs: CallOutputs, jobDetritusFiles: Option[Map[String, Path]], executionEvents: Seq[ExecutionEvent]) extends BackendJobExecutionResponse
+  case class JobSucceededResponse(jobKey: BackendJobDescriptorKey, returnCode: Option[Int], jobOutputs: CallOutputs, jobDetritusFiles: Option[Map[String, Path]], executionEvents: Seq[ExecutionEvent], dockerImageUsed: Option[String]) extends BackendJobExecutionResponse
   case class AbortedResponse(jobKey: BackendJobDescriptorKey) extends BackendJobExecutionResponse
   sealed trait BackendJobFailedResponse extends BackendJobExecutionResponse {  def throwable: Throwable; def returnCode: Option[Int] }
   case class JobFailedNonRetryableResponse(jobKey: JobKey, throwable: Throwable, returnCode: Option[Int]) extends BackendJobFailedResponse
diff --git a/backend/src/main/scala/cromwell/backend/BackendLifecycleActor.scala b/backend/src/main/scala/cromwell/backend/BackendLifecycleActor.scala
index a6a09cff4..5c1031726 100644
--- a/backend/src/main/scala/cromwell/backend/BackendLifecycleActor.scala
+++ b/backend/src/main/scala/cromwell/backend/BackendLifecycleActor.scala
@@ -3,7 +3,7 @@ package cromwell.backend
 import akka.actor.{Actor, ActorRef}
 import cromwell.backend.BackendLifecycleActor._
 import cromwell.core.logging.{JobLogging, WorkflowLogging}
-import wdl4s.TaskCall
+import wdl4s.wdl.WdlTaskCall
 
 import scala.concurrent.{ExecutionContext, Future}
 import scala.util.{Failure, Success}
@@ -65,7 +65,7 @@ trait BackendWorkflowLifecycleActor extends BackendLifecycleActor with WorkflowL
   /**
     * The subset of calls which this backend will be expected to run
     */
-  protected def calls: Set[TaskCall]
+  protected def calls: Set[WdlTaskCall]
 }
 
 trait BackendJobLifecycleActor extends BackendLifecycleActor with JobLogging {
diff --git a/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala b/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala
index e6aa055b1..c255fdc12 100644
--- a/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala
+++ b/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala
@@ -2,44 +2,71 @@ package cromwell.backend
 
 import akka.actor.{ActorRef, Props}
 import com.typesafe.config.Config
-import cromwell.backend.callcaching.FileHashingActor
-import cromwell.backend.callcaching.FileHashingActor.FileHashingFunction
 import cromwell.backend.io.WorkflowPathsWithDocker
 import cromwell.core.CallOutputs
 import cromwell.core.JobExecutionToken.JobExecutionTokenType
 import cromwell.core.path.Path
-import wdl4s.TaskCall
-import wdl4s.expression.{PureStandardLibraryFunctions, WdlStandardLibraryFunctions}
+import wdl4s.wdl.WdlTaskCall
+import wdl4s.wdl.expression.{PureStandardLibraryFunctions, WdlStandardLibraryFunctions}
 
 
 trait BackendLifecycleActorFactory {
+
+  /* ****************************** */
+  /*     Workflow Initialization    */
+  /* ****************************** */
+
   def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor,
-                                       calls: Set[TaskCall],
-                                       serviceRegistryActor: ActorRef): Option[Props] = None
+                                       ioActor: ActorRef,
+                                       calls: Set[WdlTaskCall],
+                                       serviceRegistryActor: ActorRef,
+                                       restarting: Boolean): Option[Props] = None
+
+  /* ****************************** */
+  /*          Job Execution         */
+  /* ****************************** */
 
   def jobExecutionActorProps(jobDescriptor: BackendJobDescriptor,
                              initializationData: Option[BackendInitializationData],
                              serviceRegistryActor: ActorRef,
+                             ioActor: ActorRef,
                              backendSingletonActor: Option[ActorRef]): Props
 
+  def jobExecutionTokenType: JobExecutionTokenType = JobExecutionTokenType("Default", None)
+
+  /* ****************************** */
+  /*      Workflow Finalization     */
+  /* ****************************** */
+
+  def workflowFinalizationActorProps(workflowDescriptor: BackendWorkflowDescriptor,
+                                     ioActor: ActorRef,
+                                     calls: Set[WdlTaskCall],
+                                     jobExecutionMap: JobExecutionMap,
+                                     workflowOutputs: CallOutputs,
+                                     initializationData: Option[BackendInitializationData]): Option[Props] = None
+
+  /* ****************************** */
+  /*           Call Caching         */
+  /* ****************************** */
+
+  def fileHashingActorProps: Option[(BackendJobDescriptor, Option[BackendInitializationData], ActorRef, ActorRef) => Props] = None
+
   /**
     * Providing this method to generate Props for a cache hit copying actor is optional.
     * To implement it, add a function:
-    * def cacheHitCopyingActorInner(jobDescriptor: BackendJobDescriptor,initializationData: Option[BackendInitializationData], serviceRegistryActor: ActorRef): Props
+    * def cacheHitCopyingActorInner(jobDescriptor: BackendJobDescriptor,initializationData: Option[BackendInitializationData], serviceRegistryActor: ActorRef, ioActor: ActorRef): Props
     * And then override this method to point to it:
     * override def cacheHitCopyingActorProps = Option(cacheHitCopyingActorInner _)
     *
     * Simples!
     */
-  def cacheHitCopyingActorProps: Option[(BackendJobDescriptor, Option[BackendInitializationData], ActorRef) => Props] = None
+  def cacheHitCopyingActorProps: Option[(BackendJobDescriptor, Option[BackendInitializationData], ActorRef, ActorRef) => Props] = None
 
-  def backendSingletonActorProps: Option[Props] = None
+  /* ****************************** */
+  /*              Misc.             */
+  /* ****************************** */
 
-  def workflowFinalizationActorProps(workflowDescriptor: BackendWorkflowDescriptor,
-                                     calls: Set[TaskCall],
-                                     jobExecutionMap: JobExecutionMap,
-                                     workflowOutputs: CallOutputs,
-                                     initializationData: Option[BackendInitializationData]): Option[Props] = None
+  def backendSingletonActorProps: Option[Props] = None
 
   def expressionLanguageFunctions(workflowDescriptor: BackendWorkflowDescriptor,
                                   jobKey: BackendJobDescriptorKey,
@@ -55,16 +82,14 @@ trait BackendLifecycleActorFactory {
 
   def runtimeAttributeDefinitions(initializationDataOption: Option[BackendInitializationData]): Set[RuntimeAttributeDefinition] = Set.empty
 
-  lazy val fileHashingFunction: Option[FileHashingFunction] = None
-  lazy val fileHashingActorCount: Int = 50
+  /**
+    * A set of KV store keys that this backend requests that the engine lookup before running each job.
+    */
+  def requestedKeyValueStoreKeys: Seq[String] = Seq.empty
 
-  def fileHashingActorProps: Props = FileHashingActor.props(fileHashingFunction)
-  
   /*
    * Returns credentials that can be used to authenticate to a docker registry server
    * in order to obtain a docker hash.
    */
   def dockerHashCredentials(initializationDataOption: Option[BackendInitializationData]): List[Any] = List.empty
-
-  def jobExecutionTokenType: JobExecutionTokenType = JobExecutionTokenType("Default", None)
 }
diff --git a/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala b/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala
index 5bf1d05fd..18e341ad7 100644
--- a/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala
+++ b/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala
@@ -9,10 +9,10 @@ import cromwell.backend.validation.ContinueOnReturnCodeValidation
 import cromwell.core.{WorkflowMetadataKeys, WorkflowOptions}
 import cromwell.services.metadata.MetadataService.PutMetadataAction
 import cromwell.services.metadata.{MetadataEvent, MetadataKey, MetadataValue}
-import wdl4s._
-import wdl4s.expression.PureStandardLibraryFunctions
-import wdl4s.types._
-import wdl4s.values.WdlValue
+import wdl4s.wdl._
+import wdl4s.wdl.expression.PureStandardLibraryFunctions
+import wdl4s.wdl.types._
+import wdl4s.wdl.values.WdlValue
 
 import scala.concurrent.Future
 import scala.util.{Failure, Success, Try}
@@ -38,7 +38,7 @@ object BackendWorkflowInitializationActor {
 trait BackendWorkflowInitializationActor extends BackendWorkflowLifecycleActor with ActorLogging {
   def serviceRegistryActor: ActorRef
 
-  def calls: Set[TaskCall]
+  def calls: Set[WdlTaskCall]
 
   /**
     * This method is meant only as a "pre-flight check" validation of runtime attribute expressions during workflow
@@ -69,7 +69,7 @@ trait BackendWorkflowInitializationActor extends BackendWorkflowLifecycleActor w
     * return `true` in both cases.
     */
   protected def continueOnReturnCodePredicate(valueRequired: Boolean)(wdlExpressionMaybe: Option[WdlValue]): Boolean = {
-    ContinueOnReturnCodeValidation.default.validateOptionalExpression(wdlExpressionMaybe)
+    ContinueOnReturnCodeValidation.default(configurationDescriptor.backendRuntimeConfig).validateOptionalExpression(wdlExpressionMaybe)
   }
 
   protected def runtimeAttributeValidators: Map[String, Option[WdlValue] => Boolean]
@@ -125,7 +125,7 @@ trait BackendWorkflowInitializationActor extends BackendWorkflowLifecycleActor w
           defaultRuntimeAttributes.get(name)
         }
 
-        def badRuntimeAttrsForTask(task: Task) = {
+        def badRuntimeAttrsForTask(task: WdlTask) = {
           runtimeAttributeValidators map { case (attributeName, validator) =>
             val value = task.runtimeAttributes.attrs.get(attributeName) orElse defaultRuntimeAttribute(attributeName)
             attributeName -> ((value, validator(value)))
diff --git a/backend/src/main/scala/cromwell/backend/MemorySize.scala b/backend/src/main/scala/cromwell/backend/MemorySize.scala
index bf5438764..0c1eddc9b 100644
--- a/backend/src/main/scala/cromwell/backend/MemorySize.scala
+++ b/backend/src/main/scala/cromwell/backend/MemorySize.scala
@@ -3,14 +3,16 @@ package cromwell.backend
 
 import cats.data.Validated._
 import cats.syntax.cartesian._
+import cats.syntax.either._
 import cats.syntax.validated._
 import lenthall.validation.ErrorOr._
 import mouse.string._
 
-import scala.language.postfixOps
 import scala.util.{Failure, Success, Try}
 import wdl4s.parser.MemoryUnit
 
+import scala.language.postfixOps
+
 
 object MemorySize {
   val memoryPattern = """(\d+(?:\.\d+)?)\s*(\w+)""".r
diff --git a/backend/src/main/scala/cromwell/backend/RuntimeAttributeDefinition.scala b/backend/src/main/scala/cromwell/backend/RuntimeAttributeDefinition.scala
index 0c9bf8727..7e548df2f 100644
--- a/backend/src/main/scala/cromwell/backend/RuntimeAttributeDefinition.scala
+++ b/backend/src/main/scala/cromwell/backend/RuntimeAttributeDefinition.scala
@@ -3,9 +3,9 @@ package cromwell.backend
 import cromwell.core.WorkflowOptions
 import cromwell.util.JsonFormatting.WdlValueJsonFormatter
 import lenthall.util.TryUtil
-import wdl4s.{WdlExpressionException, _}
-import wdl4s.expression.WdlStandardLibraryFunctions
-import wdl4s.values.WdlValue
+import wdl4s.wdl.{WdlExpressionException, _}
+import wdl4s.wdl.expression.WdlStandardLibraryFunctions
+import wdl4s.wdl.values.WdlValue
 
 import scala.util.{Success, Try}
 
@@ -47,7 +47,6 @@ object RuntimeAttributeDefinition {
       case (runtimeAttributeDefinition, Success(jsValue)) => runtimeAttributeDefinition.name -> jsValue.convertTo[WdlValue]
       case (RuntimeAttributeDefinition(name, Some(factoryDefault), _), _) => name -> factoryDefault
     }
-
     specifiedAttributes ++ defaults
   }
 }
diff --git a/backend/src/main/scala/cromwell/backend/async/AsyncBackendJobExecutionActor.scala b/backend/src/main/scala/cromwell/backend/async/AsyncBackendJobExecutionActor.scala
index fdb860945..509f30286 100644
--- a/backend/src/main/scala/cromwell/backend/async/AsyncBackendJobExecutionActor.scala
+++ b/backend/src/main/scala/cromwell/backend/async/AsyncBackendJobExecutionActor.scala
@@ -34,7 +34,7 @@ object AsyncBackendJobExecutionActor {
 
 trait AsyncBackendJobExecutionActor { this: Actor with ActorLogging =>
 
-  def retryable: Boolean
+  def dockerImageUsed: Option[String]
 
   // The scala package object (scala/package.scala) contains a neat list of runtime errors that are always going to be fatal.
   // We also consider any Error as fatal, and include the CromwellFatalExceptionMarker so we can mark our own fatal exceptions.
@@ -71,12 +71,11 @@ trait AsyncBackendJobExecutionActor { this: Actor with ActorLogging =>
   }
 
   private def failAndStop(t: Throwable) = {
-    val responseBuilder = if (retryable) JobFailedRetryableResponse else JobFailedNonRetryableResponse
-    completionPromise.success(responseBuilder.apply(jobDescriptor.key, t, None))
+    completionPromise.success(JobFailedNonRetryableResponse(jobDescriptor.key, t, None))
     context.stop(self)
   }
 
-  def receive: Receive = {
+  override def receive: Receive = {
     case mode: ExecutionMode => robustExecuteOrRecover(mode)
     case IssuePollRequest(handle) => robustPoll(handle)
     case PollResponseReceived(handle) if handle.isDone => self ! Finish(handle)
@@ -86,7 +85,7 @@ trait AsyncBackendJobExecutionActor { this: Actor with ActorLogging =>
       context.system.scheduler.scheduleOnce(pollBackOff.backoffMillis.millis, self, IssuePollRequest(handle))
       ()
     case Finish(SuccessfulExecutionHandle(outputs, returnCode, jobDetritusFiles, executionEvents, _)) =>
-      completionPromise.success(JobSucceededResponse(jobDescriptor.key, Some(returnCode), outputs, Option(jobDetritusFiles), executionEvents))
+      completionPromise.success(JobSucceededResponse(jobDescriptor.key, Some(returnCode), outputs, Option(jobDetritusFiles), executionEvents, dockerImageUsed))
       context.stop(self)
     case Finish(FailedNonRetryableExecutionHandle(throwable, returnCode)) =>
       completionPromise.success(JobFailedNonRetryableResponse(jobDescriptor.key, throwable, returnCode))
diff --git a/backend/src/main/scala/cromwell/backend/async/KnownJobFailureException.scala b/backend/src/main/scala/cromwell/backend/async/KnownJobFailureException.scala
index 79503c927..dd6091f42 100644
--- a/backend/src/main/scala/cromwell/backend/async/KnownJobFailureException.scala
+++ b/backend/src/main/scala/cromwell/backend/async/KnownJobFailureException.scala
@@ -2,7 +2,7 @@ package cromwell.backend.async
 
 import cromwell.core.path.Path
 import lenthall.exception.ThrowableAggregation
-import wdl4s.values.WdlValue
+import wdl4s.wdl.values.WdlValue
 
 abstract class KnownJobFailureException extends Exception {
   def stderrPath: Option[Path]
diff --git a/backend/src/main/scala/cromwell/backend/backend.scala b/backend/src/main/scala/cromwell/backend/backend.scala
index 7e5020d22..527a514f9 100644
--- a/backend/src/main/scala/cromwell/backend/backend.scala
+++ b/backend/src/main/scala/cromwell/backend/backend.scala
@@ -2,18 +2,19 @@ package cromwell.backend
 
 import com.typesafe.config.Config
 import cromwell.core.WorkflowOptions.WorkflowOption
-import cromwell.core.callcaching.CallCachingEligibility
+import cromwell.core.callcaching.MaybeCallCachingEligible
 import cromwell.core.labels.Labels
 import cromwell.core.{CallKey, WorkflowId, WorkflowOptions}
-import wdl4s._
-import wdl4s.values.WdlValue
+import cromwell.services.keyvalue.KeyValueServiceActor.KvResponse
+import wdl4s.wdl._
+import wdl4s.wdl.values.WdlValue
 
 import scala.util.Try
 
 /**
   * For uniquely identifying a job which has been or will be sent to the backend.
   */
-case class BackendJobDescriptorKey(call: TaskCall, index: Option[Int], attempt: Int) extends CallKey {
+case class BackendJobDescriptorKey(call: WdlTaskCall, index: Option[Int], attempt: Int) extends CallKey {
   def scope = call
   private val indexString = index map { _.toString } getOrElse "NA"
   val tag = s"${call.fullyQualifiedName}:$indexString:$attempt"
@@ -27,7 +28,8 @@ case class BackendJobDescriptor(workflowDescriptor: BackendWorkflowDescriptor,
                                 key: BackendJobDescriptorKey,
                                 runtimeAttributes: Map[LocallyQualifiedName, WdlValue],
                                 inputDeclarations: EvaluatedTaskInputs,
-                                callCachingEligibility: CallCachingEligibility) {
+                                maybeCallCachingEligible: MaybeCallCachingEligible,
+                                prefetchedKvStoreEntries: Map[String, KvResponse]) {
   val fullyQualifiedInputs = inputDeclarations map { case (declaration, value) => declaration.fullyQualifiedName -> value }
   val call = key.call
   override val toString = s"${key.mkTag(workflowDescriptor.id)}"
@@ -35,7 +37,7 @@ case class BackendJobDescriptor(workflowDescriptor: BackendWorkflowDescriptor,
 
 object BackendWorkflowDescriptor {
   def apply(id: WorkflowId,
-            workflow: Workflow,
+            workflow: WdlWorkflow,
             knownValues: Map[FullyQualifiedName, WdlValue],
             workflowOptions: WorkflowOptions,
             customLabels: Labels) = {
@@ -47,7 +49,7 @@ object BackendWorkflowDescriptor {
   * For passing to a BackendActor construction time
   */
 case class BackendWorkflowDescriptor(id: WorkflowId,
-                                     workflow: Workflow,
+                                     workflow: WdlWorkflow,
                                      knownValues: Map[FullyQualifiedName, WdlValue],
                                      workflowOptions: WorkflowOptions,
                                      customLabels: Labels,
@@ -63,10 +65,12 @@ case class BackendWorkflowDescriptor(id: WorkflowId,
 /**
   * For passing to a BackendActor construction time
   */
-case class BackendConfigurationDescriptor(backendConfig: Config, globalConfig: Config)
+case class BackendConfigurationDescriptor(backendConfig: Config, globalConfig: Config) {
+
+  lazy val backendRuntimeConfig = if (backendConfig.hasPath("default-runtime-attributes"))
+    Option(backendConfig.getConfig("default-runtime-attributes")) else None
+}
 
 final case class AttemptedLookupResult(name: String, value: Try[WdlValue]) {
   def toPair = name -> value
 }
-
-case class PreemptedException(msg: String) extends Exception(msg)
diff --git a/backend/src/main/scala/cromwell/backend/callcaching/CacheHitDuplicating.scala b/backend/src/main/scala/cromwell/backend/callcaching/CacheHitDuplicating.scala
deleted file mode 100644
index 979166f08..000000000
--- a/backend/src/main/scala/cromwell/backend/callcaching/CacheHitDuplicating.scala
+++ /dev/null
@@ -1,103 +0,0 @@
-package cromwell.backend.callcaching
-
-import akka.actor.ActorRef
-import cromwell.backend.BackendCacheHitCopyingActor
-import cromwell.backend.BackendJobExecutionActor.{BackendJobExecutionResponse, JobSucceededResponse}
-import cromwell.backend.io.JobPaths
-import cromwell.core.path.{Path, PathCopier}
-import cromwell.core.simpleton.{WdlValueBuilder, WdlValueSimpleton}
-import wdl4s.values.WdlFile
-
-import scala.language.postfixOps
-import scala.util.Try
-
-/**
-  * Mixin implementing common functionality for a BackendCacheHitCopyingActor.
-  *
-  * Implements copyCachedOutputs, with abstract methods for converting a string to a path, duplicating a path, returning
-  * a reference to the service registry actor, and obtaining various metadata/outputs for the job.
-  */
-trait CacheHitDuplicating {
-  this: BackendCacheHitCopyingActor =>
-
-  /**
-    * Duplicates two paths returned by getPath.
-    *
-    * @param source      Source path.
-    * @param destination Destination path.
-    */
-  protected def duplicate(source: Path, destination: Path): Unit
-
-  /**
-    * Returns an absolute path to the file.
-    *
-    * NOTE: If necessary for separate credentialing, we may split this method into getSourcePath and getDestinationPath.
-    *
-    * @param file the string version of the path
-    * @return an absolute path to the file with potential credentials embedded within.
-    */
-  protected def getPath(file: String): Try[Path]
-
-  protected def destinationCallRootPath: Path
-
-  protected def serviceRegistryActor: ActorRef
-
-  protected def destinationJobDetritusPaths: Map[String, Path]
-
-  // Usually implemented by a subclass of JobCachingActorHelper
-  protected def startMetadataKeyValues: Map[String, Any]
-
-  private def lookupSourceCallRootPath(sourceJobDetritusFiles: Map[String, String]): Path = {
-    sourceJobDetritusFiles.get(JobPaths.CallRootPathKey).map(getPath).get recover {
-      case failure =>
-        throw new RuntimeException(s"${JobPaths.CallRootPathKey} wasn't found for call ${jobDescriptor.call.fullyQualifiedName}", failure)
-    } get
-  }
-
-  /**
-    * After copying files, return the simpletons substituting the destination file paths.
-    */
-  private def copySimpletons(wdlValueSimpletons: Seq[WdlValueSimpleton],
-                             sourceCallRootPath: Path): Seq[WdlValueSimpleton] = {
-    wdlValueSimpletons map {
-      case WdlValueSimpleton(key, wdlFile: WdlFile) =>
-        val sourcePath = getPath(wdlFile.value).get
-        val destinationPath = PathCopier.getDestinationFilePath(sourceCallRootPath, sourcePath, destinationCallRootPath)
-        duplicate(sourcePath, destinationPath)
-        WdlValueSimpleton(key, WdlFile(destinationPath.pathAsString))
-      case wdlValueSimpleton => wdlValueSimpleton
-    }
-  }
-
-  private def copyDetritus(sourceJobDetritusFiles: Map[String, String]): Map[String, Path] = {
-    val sourceKeys = sourceJobDetritusFiles.keySet
-    val destinationKeys = destinationJobDetritusPaths.keySet
-    val fileKeys = sourceKeys.intersect(destinationKeys).filterNot(_ == JobPaths.CallRootPathKey)
-
-    val destinationJobDetritusFiles = fileKeys map { fileKey =>
-      val sourcePath = getPath(sourceJobDetritusFiles(fileKey)).get
-      val destinationPath = destinationJobDetritusPaths(fileKey)
-      duplicate(sourcePath, destinationPath)
-      (fileKey, destinationPath)
-    }
-
-    destinationJobDetritusFiles.toMap + (JobPaths.CallRootPathKey -> destinationCallRootPath)
-  }
-
-  override def copyCachedOutputs(wdlValueSimpletons: Seq[WdlValueSimpleton],
-                                 sourceJobDetritusFiles: Map[String, String],
-                                 returnCodeOption: Option[Int]): BackendJobExecutionResponse = {
-    val sourceCallRootPath = lookupSourceCallRootPath(sourceJobDetritusFiles)
-
-    val destinationSimpletons = copySimpletons(wdlValueSimpletons, sourceCallRootPath)
-    val destinationJobDetritusFiles = copyDetritus(sourceJobDetritusFiles)
-
-    val destinationJobOutputs = WdlValueBuilder.toJobOutputs(jobDescriptor.call.task.outputs, destinationSimpletons)
-
-    import cromwell.services.metadata.MetadataService.implicits.MetadataAutoPutter
-    serviceRegistryActor.putMetadata(
-      jobDescriptor.workflowDescriptor.id, Option(jobDescriptor.key), startMetadataKeyValues)
-
-    JobSucceededResponse(jobDescriptor.key, returnCodeOption, destinationJobOutputs, Option(destinationJobDetritusFiles), Seq.empty)
-  }
-}
diff --git a/backend/src/main/scala/cromwell/backend/callcaching/FileHashingActor.scala b/backend/src/main/scala/cromwell/backend/callcaching/FileHashingActor.scala
deleted file mode 100644
index ee1ecea4c..000000000
--- a/backend/src/main/scala/cromwell/backend/callcaching/FileHashingActor.scala
+++ /dev/null
@@ -1,42 +0,0 @@
-package cromwell.backend.callcaching
-
-import akka.actor.{Actor, ActorLogging, Props}
-import akka.event.LoggingAdapter
-import cromwell.backend.BackendInitializationData
-import cromwell.core.Dispatcher.BackendDispatcher
-import cromwell.core.JobKey
-import cromwell.core.callcaching._
-import wdl4s.values.WdlFile
-import FileHashingActor._
-
-import scala.util.{Failure, Success, Try}
-
-/**
-  * Blocking worker. Warning! If this actor dies then its mailbox of hash requests will be lost
-  */
-class FileHashingActor(workerFunction: Option[FileHashingFunction]) extends Actor with ActorLogging {
-  override def receive = {
-    case x: SingleFileHashRequest =>
-
-      // Create the path with the filesystem in the initialization data:
-      workerFunction.map(_.work(x, log)) match {
-        case Some(Success(hashSuccess)) => sender ! FileHashResponse(HashResult(x.hashKey, HashValue(hashSuccess)))
-        case Some(Failure(t)) => sender ! HashingFailedMessage(x.hashKey, t)
-        case None => sender ! HashingFailedMessage(x.hashKey, new NotImplementedError("Backend has no file hashing function"))
-      }
-    case x => log.error(s"Unexpected message to ${self.path.name}: $x")
-  }
-}
-
-object FileHashingActor {
-  def props(workerFunction: Option[FileHashingFunction]): Props = Props(new FileHashingActor(workerFunction)).withDispatcher(BackendDispatcher)
-
-  case class FileHashingFunction(work: (SingleFileHashRequest, LoggingAdapter) => Try[String])
-
-  sealed trait BackendSpecificHasherCommand { def jobKey: JobKey }
-  case class SingleFileHashRequest(jobKey: JobKey, hashKey: HashKey, file: WdlFile, initializationData: Option[BackendInitializationData]) extends BackendSpecificHasherCommand
-  case class HashesNoLongerRequired(jobKey: JobKey) extends BackendSpecificHasherCommand
-
-  sealed trait BackendSpecificHasherResponse extends SuccessfulHashResultMessage
-  case class FileHashResponse(hashResult: HashResult) extends BackendSpecificHasherResponse { override def hashes = Set(hashResult) }
-}
diff --git a/backend/src/main/scala/cromwell/backend/io/GlobFunctions.scala b/backend/src/main/scala/cromwell/backend/io/GlobFunctions.scala
index 9ece33cdd..7af985153 100644
--- a/backend/src/main/scala/cromwell/backend/io/GlobFunctions.scala
+++ b/backend/src/main/scala/cromwell/backend/io/GlobFunctions.scala
@@ -2,15 +2,15 @@ package cromwell.backend.io
 
 import cromwell.backend.BackendJobDescriptor
 import cromwell.core.CallContext
-import wdl4s.TaskCall
-import wdl4s.expression.{NoFunctions, PureStandardLibraryFunctionsLike}
-import wdl4s.values._
+import wdl4s.wdl.WdlTaskCall
+import wdl4s.wdl.expression.{NoFunctions, PureStandardLibraryFunctionsLike}
+import wdl4s.wdl.values._
 
 trait GlobFunctions extends PureStandardLibraryFunctionsLike {
 
   def callContext: CallContext
 
-  def findGlobOutputs(call: TaskCall, jobDescriptor: BackendJobDescriptor): Set[WdlGlobFile] = {
+  def findGlobOutputs(call: WdlTaskCall, jobDescriptor: BackendJobDescriptor): Set[WdlGlobFile] = {
     val globOutputs = call.task.findOutputFiles(jobDescriptor.fullyQualifiedInputs, NoFunctions) collect {
       case glob: WdlGlobFile => glob
     }
diff --git a/backend/src/main/scala/cromwell/backend/io/JobPaths.scala b/backend/src/main/scala/cromwell/backend/io/JobPaths.scala
index 2e0dfab43..72057c202 100644
--- a/backend/src/main/scala/cromwell/backend/io/JobPaths.scala
+++ b/backend/src/main/scala/cromwell/backend/io/JobPaths.scala
@@ -24,16 +24,17 @@ object JobPaths {
   }
 }
 
-trait JobPaths { this: WorkflowPaths =>
+trait JobPaths {
   import JobPaths._
 
+  def workflowPaths: WorkflowPaths
   def returnCodeFilename: String = "rc"
   def stdoutFilename: String = "stdout"
   def stderrFilename: String = "stderr"
   def scriptFilename: String = "script"
   
   def jobKey: JobKey
-  lazy val callRoot = callPathBuilder(workflowRoot, jobKey)
+  lazy val callRoot = callPathBuilder(workflowPaths.workflowRoot, jobKey)
   lazy val callExecutionRoot = callRoot
   lazy val stdout = callExecutionRoot.resolve(stdoutFilename)
   lazy val stderr = callExecutionRoot.resolve(stderrFilename)
diff --git a/backend/src/main/scala/cromwell/backend/io/JobPathsWithDocker.scala b/backend/src/main/scala/cromwell/backend/io/JobPathsWithDocker.scala
index 9524e8231..2febef209 100644
--- a/backend/src/main/scala/cromwell/backend/io/JobPathsWithDocker.scala
+++ b/backend/src/main/scala/cromwell/backend/io/JobPathsWithDocker.scala
@@ -4,15 +4,21 @@ import com.typesafe.config.Config
 import cromwell.backend.{BackendJobDescriptorKey, BackendWorkflowDescriptor}
 import cromwell.core.path.{Path, PathBuilder}
 
-class JobPathsWithDocker(val jobKey: BackendJobDescriptorKey,
-                         workflowDescriptor: BackendWorkflowDescriptor,
-                         config: Config,
-                         pathBuilders: List[PathBuilder] = WorkflowPaths.DefaultPathBuilders) extends WorkflowPathsWithDocker(
-  workflowDescriptor, config, pathBuilders) with JobPaths {
+object JobPathsWithDocker {
+  def apply(jobKey: BackendJobDescriptorKey,
+            workflowDescriptor: BackendWorkflowDescriptor,
+            config: Config,
+            pathBuilders: List[PathBuilder] = WorkflowPaths.DefaultPathBuilders) = {
+    val workflowPaths = new WorkflowPathsWithDocker(workflowDescriptor, config, pathBuilders)
+    new JobPathsWithDocker(workflowPaths, jobKey)
+  }
+}
+
+case class JobPathsWithDocker private[io] (override val workflowPaths: WorkflowPathsWithDocker, jobKey: BackendJobDescriptorKey) extends JobPaths {
   import JobPaths._
-  
+
   override lazy val callExecutionRoot = { callRoot.resolve("execution") }
-  val callDockerRoot = callPathBuilder(dockerWorkflowRoot, jobKey)
+  val callDockerRoot = callPathBuilder(workflowPaths.dockerWorkflowRoot, jobKey)
   val callExecutionDockerRoot = callDockerRoot.resolve("execution")
   val callInputsRoot = callRoot.resolve("inputs")
 
@@ -30,7 +36,7 @@ class JobPathsWithDocker(val jobKey: BackendJobDescriptorKey,
           *
           * TODO: this assumes that p.startsWith(localExecutionRoot)
           */
-        val subpath = p.subpath(executionRoot.getNameCount, p.getNameCount)
+        val subpath = p.subpath(workflowPaths.executionRoot.getNameCount, p.getNameCount)
         WorkflowPathsWithDocker.DockerRoot.resolve(subpath)
     }
   }
diff --git a/backend/src/main/scala/cromwell/backend/io/WorkflowPaths.scala b/backend/src/main/scala/cromwell/backend/io/WorkflowPaths.scala
index 8a784a1ae..fc7c1a960 100644
--- a/backend/src/main/scala/cromwell/backend/io/WorkflowPaths.scala
+++ b/backend/src/main/scala/cromwell/backend/io/WorkflowPaths.scala
@@ -45,5 +45,13 @@ trait WorkflowPaths extends PathFactory {
     * @param jobWorkflowDescriptor The workflow descriptor for the job.
     * @return The paths for the job.
     */
-  def toJobPaths(jobKey: BackendJobDescriptorKey, jobWorkflowDescriptor: BackendWorkflowDescriptor): JobPaths
+  def toJobPaths(jobKey: BackendJobDescriptorKey, jobWorkflowDescriptor: BackendWorkflowDescriptor): JobPaths = {
+    // If the descriptors are the same, no need to create a new WorkflowPaths
+    if (workflowDescriptor == jobWorkflowDescriptor) toJobPaths(this, jobKey)
+    else toJobPaths(withDescriptor(jobWorkflowDescriptor), jobKey)
+  }
+  
+  protected def toJobPaths(workflowPaths: WorkflowPaths, jobKey: BackendJobDescriptorKey): JobPaths
+  
+  protected def withDescriptor(workflowDescriptor: BackendWorkflowDescriptor): WorkflowPaths
 }
diff --git a/backend/src/main/scala/cromwell/backend/io/WorkflowPathsWithDocker.scala b/backend/src/main/scala/cromwell/backend/io/WorkflowPathsWithDocker.scala
index 4e7811feb..1809283b4 100644
--- a/backend/src/main/scala/cromwell/backend/io/WorkflowPathsWithDocker.scala
+++ b/backend/src/main/scala/cromwell/backend/io/WorkflowPathsWithDocker.scala
@@ -8,11 +8,12 @@ object WorkflowPathsWithDocker {
   val DockerRoot: Path = DefaultPathBuilder.get("/cromwell-executions")
 }
 
-class WorkflowPathsWithDocker(val workflowDescriptor: BackendWorkflowDescriptor, val config: Config, val pathBuilders: List[PathBuilder] = WorkflowPaths.DefaultPathBuilders) extends WorkflowPaths {
+final case class WorkflowPathsWithDocker(workflowDescriptor: BackendWorkflowDescriptor, config: Config, pathBuilders: List[PathBuilder] = WorkflowPaths.DefaultPathBuilders) extends WorkflowPaths {
   val dockerWorkflowRoot: Path = workflowPathBuilder(WorkflowPathsWithDocker.DockerRoot)
 
-  override def toJobPaths(jobKey: BackendJobDescriptorKey,
-                          jobWorkflowDescriptor: BackendWorkflowDescriptor): JobPathsWithDocker = {
-    new JobPathsWithDocker(jobKey, jobWorkflowDescriptor, config, pathBuilders)
+  override def toJobPaths(workflowPaths: WorkflowPaths, jobKey: BackendJobDescriptorKey): JobPathsWithDocker = {
+    new JobPathsWithDocker(workflowPaths.asInstanceOf[WorkflowPathsWithDocker], jobKey)
   }
+
+  override protected def withDescriptor(workflowDescriptor: BackendWorkflowDescriptor): WorkflowPaths = this.copy(workflowDescriptor = workflowDescriptor)
 }
diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala b/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala
index e6e257e2b..0d572de10 100644
--- a/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala
+++ b/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala
@@ -7,17 +7,20 @@ import akka.event.LoggingReceive
 import cromwell.backend.BackendJobExecutionActor.{AbortedResponse, BackendJobExecutionResponse}
 import cromwell.backend.BackendLifecycleActor.AbortJobCommand
 import cromwell.backend.async.AsyncBackendJobExecutionActor.{ExecutionMode, JobId, Recover}
-import cromwell.backend.async.{AbortedExecutionHandle, AsyncBackendJobExecutionActor, ExecutionHandle, FailedNonRetryableExecutionHandle, PendingExecutionHandle, ReturnCodeIsNotAnInt, StderrNonEmpty, SuccessfulExecutionHandle, WrongReturnCode}
+import cromwell.backend.async.{AbortedExecutionHandle, AsyncBackendJobExecutionActor, ExecutionHandle, FailedNonRetryableExecutionHandle, FailedRetryableExecutionHandle, PendingExecutionHandle, ReturnCodeIsNotAnInt, StderrNonEmpty, SuccessfulExecutionHandle, WrongReturnCode}
 import cromwell.backend.validation._
 import cromwell.backend.wdl.{Command, OutputEvaluator, WdlFileMapper}
 import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendJobDescriptor, BackendJobLifecycleActor}
+import cromwell.core.io.{AsyncIo, DefaultIoCommandBuilder}
 import cromwell.core.path.Path
 import cromwell.core.{CallOutputs, CromwellAggregatedException, CromwellFatalExceptionMarker, ExecutionEvent}
 import cromwell.services.keyvalue.KeyValueServiceActor._
+import cromwell.services.keyvalue.KvClient
 import cromwell.services.metadata.CallMetadataKeys
 import lenthall.util.TryUtil
-import wdl4s._
-import wdl4s.values.{WdlFile, WdlGlobFile, WdlSingleFile, WdlValue}
+import net.ceedubs.ficus.Ficus._
+import wdl4s.wdl._
+import wdl4s.wdl.values._
 
 import scala.concurrent.{ExecutionContext, ExecutionContextExecutor, Future, Promise}
 import scala.util.{Failure, Success, Try}
@@ -31,6 +34,7 @@ case class DefaultStandardAsyncExecutionActorParams
 (
   override val jobIdKey: String,
   override val serviceRegistryActor: ActorRef,
+  override val ioActor: ActorRef,
   override val jobDescriptor: BackendJobDescriptor,
   override val configurationDescriptor: BackendConfigurationDescriptor,
   override val backendInitializationDataOption: Option[BackendInitializationData],
@@ -48,7 +52,7 @@ case class DefaultStandardAsyncExecutionActorParams
   * NOTE: Unlike the parent trait `AsyncBackendJobExecutionActor`, this trait is subject to even more frequent updates
   * as the common behavior among the backends adjusts in unison.
   */
-trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with StandardCachingActorHelper {
+trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with StandardCachingActorHelper with AsyncIo with DefaultIoCommandBuilder with KvClient {
   this: Actor with ActorLogging with BackendJobLifecycleActor =>
 
   val SIGTERM = 143
@@ -61,8 +65,7 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
   type StandardAsyncRunStatus
 
   /** The pending execution handle for each poll. */
-  type StandardAsyncPendingExecutionHandle =
-    PendingExecutionHandle[StandardAsyncJob, StandardAsyncRunInfo, StandardAsyncRunStatus]
+  type StandardAsyncPendingExecutionHandle = PendingExecutionHandle[StandardAsyncJob, StandardAsyncRunInfo, StandardAsyncRunStatus]
 
   /** Standard set of parameters passed to the backend. */
   def standardParams: StandardAsyncExecutionActorParams
@@ -70,6 +73,8 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
   override lazy val configurationDescriptor: BackendConfigurationDescriptor = standardParams.configurationDescriptor
 
   override lazy val completionPromise: Promise[BackendJobExecutionResponse] = standardParams.completionPromise
+  
+  override lazy val ioActor = standardParams.ioActor
 
   /** Backend initialization data created by the a factory initializer. */
   override lazy val backendInitializationDataOption: Option[BackendInitializationData] =
@@ -88,6 +93,8 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
   lazy val backendEngineFunctions: StandardExpressionFunctions =
     standardInitializationData.expressionFunctions(jobPaths)
 
+  lazy val scriptEpilogue = configurationDescriptor.backendConfig.as[Option[String]]("script-epilogue").getOrElse("sync")
+
   /**
     * Maps WdlFile objects for use in the commandLinePreProcessor.
     *
@@ -176,7 +183,8 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
     val globFiles = backendEngineFunctions.findGlobOutputs(call, jobDescriptor)
 
     s"""|#!/bin/bash
-        |tmpDir=$$(mktemp -d $cwd/tmp.XXXXXX) 
+        |tmpDir=$$(mktemp -d $cwd/tmp.XXXXXX)
+        |chmod 777 $$tmpDir
         |export _JAVA_OPTIONS=-Djava.io.tmpdir=$$tmpDir
         |export TMPDIR=$$tmpDir
         |$commandScriptPreamble
@@ -189,9 +197,9 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
         |cd $cwd
         |${globManipulations(globFiles)}
         |)
-        |sync
+        |SCRIPT_EPILOGUE
         |mv $rcTmpPath $rcPath
-        |""".stripMargin.replace("INSTANTIATED_COMMAND", instantiatedCommand)
+        |""".stripMargin.replace("INSTANTIATED_COMMAND", instantiatedCommand).replace("SCRIPT_EPILOGUE", scriptEpilogue)
   }
 
   /** The instantiated command. */
@@ -243,9 +251,7 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
     *
     * @return the execution handle for the job.
     */
-  def executeAsync()(implicit ec: ExecutionContext): Future[ExecutionHandle] = {
-    Future.fromTry(Try(execute()))
-  }
+  def executeAsync(): Future[ExecutionHandle] = Future.fromTry(Try(execute()))
 
   /**
     * Recovers the specified job id, or starts a new job. The default implementation simply calls execute().
@@ -261,9 +267,7 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
     * @param jobId The previously recorded job id.
     * @return the execution handle for the job.
     */
-  def recoverAsync(jobId: StandardAsyncJob)(implicit ec: ExecutionContext): Future[ExecutionHandle] = {
-    Future.fromTry(Try(recover(jobId)))
-  }
+  def recoverAsync(jobId: StandardAsyncJob): Future[ExecutionHandle] = Future.fromTry(Try(recover(jobId)))
 
   /**
     * Returns the run status for the job.
@@ -281,10 +285,7 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
     * @param handle The handle of the running job.
     * @return The status of the job.
     */
-  def pollStatusAsync(handle: StandardAsyncPendingExecutionHandle)
-                     (implicit ec: ExecutionContext): Future[StandardAsyncRunStatus] = {
-    Future.fromTry(Try(pollStatus(handle)))
-  }
+  def pollStatusAsync(handle: StandardAsyncPendingExecutionHandle): Future[StandardAsyncRunStatus] = Future.fromTry(Try(pollStatus(handle)))
 
   /**
     * Adds custom behavior invoked when polling fails due to some exception. By default adds nothing.
@@ -473,15 +474,15 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
     */
   def handleExecutionFailure(runStatus: StandardAsyncRunStatus,
                              handle: StandardAsyncPendingExecutionHandle,
-                             returnCode: Option[Int]): ExecutionHandle = {
-    FailedNonRetryableExecutionHandle(new Exception(s"Task failed for unknown reason: $runStatus"), returnCode)
+                             returnCode: Option[Int]): Future[ExecutionHandle] = {
+    Future.successful(FailedNonRetryableExecutionHandle(new Exception(s"Task failed for unknown reason: $runStatus"), returnCode))
   }
 
   // See executeOrRecoverSuccess
   private var missedAbort = false
   private case class CheckMissedAbort(jobId: StandardAsyncJob)
 
-  context.become(standardReceiveBehavior(None) orElse receive)
+  context.become(kvClientReceive orElse ioReceive orElse standardReceiveBehavior(None) orElse receive)
 
   def standardReceiveBehavior(jobIdOption: Option[StandardAsyncJob]): Receive = LoggingReceive {
     case AbortJobCommand =>
@@ -495,14 +496,11 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
       }
       postAbort()
     case CheckMissedAbort(jobId: StandardAsyncJob) =>
-      context.become(standardReceiveBehavior(Option(jobId)) orElse receive)
+      context.become(kvClientReceive orElse ioReceive orElse standardReceiveBehavior(Option(jobId)) orElse receive)
       if (missedAbort)
         self ! AbortJobCommand
-    case KvPutSuccess(_) => // expected after the KvPut for the operation ID
   }
 
-  override def retryable = false
-
   override def executeOrRecover(mode: ExecutionMode)(implicit ec: ExecutionContext): Future[ExecutionHandle] = {
     val executeOrRecoverFuture = {
       mode match {
@@ -513,7 +511,7 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
       }
     }
 
-    executeOrRecoverFuture map executeOrRecoverSuccess recoverWith {
+    executeOrRecoverFuture flatMap executeOrRecoverSuccess recoverWith {
       case throwable: Throwable => Future failed {
         jobLogger.error(s"Error attempting to $mode", throwable)
         throwable
@@ -521,23 +519,23 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
     }
   }
 
-  private def executeOrRecoverSuccess(executionHandle: ExecutionHandle): ExecutionHandle = {
+  private def executeOrRecoverSuccess(executionHandle: ExecutionHandle): Future[ExecutionHandle] = {
     executionHandle match {
-      case handle: PendingExecutionHandle[
-        StandardAsyncJob@unchecked, StandardAsyncRunInfo@unchecked, StandardAsyncRunStatus@unchecked] =>
-        tellKvJobId(handle.pendingJob)
-        jobLogger.info(s"job id: ${handle.pendingJob.jobId}")
-        tellMetadata(Map(CallMetadataKeys.JobId -> handle.pendingJob.jobId))
-        /*
-        NOTE: Because of the async nature of the Scala Futures, there is a point in time where we have submitted this or
-        the prior runnable to the thread pool this actor doesn't know the job id for aborting. These runnables are
-        queued up and may still be run by the thread pool anytime in the future. Issue #1218 may address this
-        inconsistency at a later time. For now, just go back and check if we missed the abort command.
-        */
-        self ! CheckMissedAbort(handle.pendingJob)
-      case _ =>
+      case handle: PendingExecutionHandle[StandardAsyncJob@unchecked, StandardAsyncRunInfo@unchecked, StandardAsyncRunStatus@unchecked] =>
+        tellKvJobId(handle.pendingJob).map { case _ =>
+          jobLogger.info(s"job id: ${handle.pendingJob.jobId}")
+          tellMetadata(Map(CallMetadataKeys.JobId -> handle.pendingJob.jobId))
+          /*
+          NOTE: Because of the async nature of the Scala Futures, there is a point in time where we have submitted this or
+          the prior runnable to the thread pool this actor doesn't know the job id for aborting. These runnables are
+          queued up and may still be run by the thread pool anytime in the future. Issue #1218 may address this
+          inconsistency at a later time. For now, just go back and check if we missed the abort command.
+          */
+          self ! CheckMissedAbort(handle.pendingJob)
+          executionHandle
+        }
+      case _ => Future.successful(executionHandle)
     }
-    executionHandle
   }
 
   override def poll(previous: ExecutionHandle)(implicit ec: ExecutionContext): Future[ExecutionHandle] = {
@@ -546,7 +544,7 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
         StandardAsyncJob@unchecked, StandardAsyncRunInfo@unchecked, StandardAsyncRunStatus@unchecked] =>
 
         jobLogger.debug(s"$tag Polling Job ${handle.pendingJob}")
-        pollStatusAsync(handle) map {
+        pollStatusAsync(handle) flatMap {
           backendRunStatus =>
             handlePollSuccess(handle, backendRunStatus)
         } recover {
@@ -555,6 +553,7 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
         }
       case successful: SuccessfulExecutionHandle => Future.successful(successful)
       case failed: FailedNonRetryableExecutionHandle => Future.successful(failed)
+      case failedRetryable: FailedRetryableExecutionHandle => Future.successful(failedRetryable)
       case badHandle => Future.failed(new IllegalArgumentException(s"Unexpected execution handle: $badHandle"))
     }
   }
@@ -567,7 +566,7 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
     * @return The updated execution handle.
     */
   def handlePollSuccess(oldHandle: StandardAsyncPendingExecutionHandle,
-                        status: StandardAsyncRunStatus): ExecutionHandle = {
+                        status: StandardAsyncRunStatus): Future[ExecutionHandle] = {
     val previousStatus = oldHandle.previousStatus
     if (!(previousStatus contains status)) {
       /*
@@ -584,7 +583,7 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
         val metadata = getTerminalMetadata(status)
         tellMetadata(metadata)
         handleExecutionResult(status, oldHandle)
-      case s => oldHandle.copy(previousStatus = Option(s)) // Copy the current handle with updated previous status.
+      case s => Future.successful(oldHandle.copy(previousStatus = Option(s))) // Copy the current handle with updated previous status.
     }
   }
 
@@ -626,52 +625,41 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
     * @return The updated execution handle.
     */
   def handleExecutionResult(status: StandardAsyncRunStatus,
-                            oldHandle: StandardAsyncPendingExecutionHandle): ExecutionHandle = {
-    try {
-
-      lazy val returnCodeAsString: Try[String] = Try(jobPaths.returnCode.contentAsString)
-      lazy val returnCodeAsInt: Try[Int] = returnCodeAsString.map(_.trim.toInt)
+                            oldHandle: StandardAsyncPendingExecutionHandle): Future[ExecutionHandle] = {
       lazy val stderrAsOption: Option[Path] = Option(jobPaths.stderr)
-
-      if (isSuccess(status)) {
-        lazy val stderrLength: Try[Long] = Try(jobPaths.stderr.size)
-        (stderrLength, returnCodeAsString, returnCodeAsInt) match {
-            // Failed to get stderr size -> Retry
-          case (Failure(exception), _, _) =>
-            jobLogger.warn(s"could not get stderr file size, retrying", exception)
-            oldHandle
-            // Failed to get return code content -> Retry
-          case (_, Failure(exception), _) =>
-            jobLogger.warn(s"could not download return code file, retrying", exception)
-            oldHandle
-            // Failed to convert return code content to Int -> Fail
-          case (_, _, Failure(_)) =>
-            FailedNonRetryableExecutionHandle(ReturnCodeIsNotAnInt(jobDescriptor.key.tag, returnCodeAsString.get, stderrAsOption))
-            // Stderr is not empty and failOnStdErr is true -> Fail
-          case (Success(length), _, _) if failOnStdErr && length.intValue > 0 =>
-            FailedNonRetryableExecutionHandle(StderrNonEmpty(jobDescriptor.key.tag, length, stderrAsOption), returnCodeAsInt.toOption)
-            // Return code is abort code -> Abort
-          case (_, _, Success(rc)) if isAbort(rc) =>
-            AbortedExecutionHandle
-            // Return code is not valid -> Fail
-          case (_, _, Success(rc)) if !continueOnReturnCode.continueFor(rc) =>
-            FailedNonRetryableExecutionHandle(WrongReturnCode(jobDescriptor.key.tag, returnCodeAsInt.get, stderrAsOption), returnCodeAsInt.toOption)
-            // Otherwise -> Succeed
-          case (_, _, Success(rc)) => 
-            handleExecutionSuccess(status, oldHandle, rc)
-        }
-      } else {
-        handleExecutionFailure(status, oldHandle, returnCodeAsInt.toOption)
+    
+      val stderrSizeAndReturnCode = for {
+        returnCodeAsString <- contentAsStringAsync(jobPaths.returnCode)
+        // Only check stderr size if we need to, otherwise this results in a lot of unnecessary I/O that
+        // may fail due to race conditions on quickly-executing jobs.
+        stderrSize <- if (failOnStdErr) sizeAsync(jobPaths.stderr) else Future.successful(0L)
+      } yield (stderrSize, returnCodeAsString)
+
+      stderrSizeAndReturnCode flatMap {
+        case (stderrSize, returnCodeAsString) =>
+          val tryReturnCodeAsInt = Try(returnCodeAsString.trim.toInt)
+          
+          if (isSuccess(status)) {
+            tryReturnCodeAsInt match {
+              case Success(returnCodeAsInt) if failOnStdErr && stderrSize.intValue > 0 =>
+                Future.successful(FailedNonRetryableExecutionHandle(StderrNonEmpty(jobDescriptor.key.tag, stderrSize, stderrAsOption), Option(returnCodeAsInt)))
+              case Success(returnCodeAsInt) if isAbort(returnCodeAsInt) =>
+                Future.successful(AbortedExecutionHandle)
+              case Success(returnCodeAsInt) if !continueOnReturnCode.continueFor(returnCodeAsInt) =>
+                Future.successful(FailedNonRetryableExecutionHandle(WrongReturnCode(jobDescriptor.key.tag, returnCodeAsInt, stderrAsOption), Option(returnCodeAsInt)))
+              case Success(returnCodeAsInt) =>
+                Future.successful(handleExecutionSuccess(status, oldHandle, returnCodeAsInt))
+              case Failure(_) =>
+                Future.successful(FailedNonRetryableExecutionHandle(ReturnCodeIsNotAnInt(jobDescriptor.key.tag, returnCodeAsString, stderrAsOption)))
+            }
+          } else {
+            handleExecutionFailure(status, oldHandle, tryReturnCodeAsInt.toOption)
+          }
+      } recoverWith {
+        case exception => 
+          if (isSuccess(status)) Future.successful(FailedNonRetryableExecutionHandle(exception))
+          else handleExecutionFailure(status, oldHandle, None)
       }
-    } catch {
-      case exception: Exception if isFatal(exception) =>
-        jobLogger.warn("Caught fatal exception processing job result", exception)
-        FailedNonRetryableExecutionHandle(exception)
-      case exception: Exception =>
-        jobLogger.warn("Caught exception processing job result, retrying", exception)
-        // Return the original handle to try again.
-        oldHandle
-    }
   }
 
   /**
@@ -679,14 +667,14 @@ trait StandardAsyncExecutionActor extends AsyncBackendJobExecutionActor with Sta
     *
     * @param runningJob The running job.
     */
-  def tellKvJobId(runningJob: StandardAsyncJob): Unit = {
+  def tellKvJobId(runningJob: StandardAsyncJob): Future[KvResponse] = {
     val kvJobKey =
       KvJobKey(jobDescriptor.key.call.fullyQualifiedName, jobDescriptor.key.index, jobDescriptor.key.attempt)
     val scopedKey = ScopedKey(jobDescriptor.workflowDescriptor.id, kvJobKey, jobIdKey)
     val kvValue = Option(runningJob.jobId)
     val kvPair = KvPair(scopedKey, kvValue)
     val kvPut = KvPut(kvPair)
-    serviceRegistryActor ! kvPut
+    makeKvRequest(Seq(kvPut)).map(_.head)
   }
 
   /**
diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardCacheHitCopyingActor.scala b/backend/src/main/scala/cromwell/backend/standard/StandardCacheHitCopyingActor.scala
deleted file mode 100644
index 5f0ee8982..000000000
--- a/backend/src/main/scala/cromwell/backend/standard/StandardCacheHitCopyingActor.scala
+++ /dev/null
@@ -1,45 +0,0 @@
-package cromwell.backend.standard
-
-import akka.actor.ActorRef
-import cromwell.backend.callcaching.CacheHitDuplicating
-import cromwell.backend.{BackendCacheHitCopyingActor, BackendConfigurationDescriptor, BackendInitializationData, BackendJobDescriptor}
-import cromwell.core.path.{Path, PathCopier}
-
-/**
-  * Trait of parameters passed to a StandardCacheHitCopyingActor.
-  */
-trait StandardCacheHitCopyingActorParams {
-  def jobDescriptor: BackendJobDescriptor
-
-  def backendInitializationDataOption: Option[BackendInitializationData]
-
-  def serviceRegistryActor: ActorRef
-
-  def configurationDescriptor: BackendConfigurationDescriptor
-}
-
-/** A default implementation of the cache hit copying params. */
-case class DefaultStandardCacheHitCopyingActorParams
-(
-  override val jobDescriptor: BackendJobDescriptor,
-  override val backendInitializationDataOption: Option[BackendInitializationData],
-  override val serviceRegistryActor: ActorRef,
-  override val configurationDescriptor: BackendConfigurationDescriptor
-) extends StandardCacheHitCopyingActorParams
-
-/**
-  * Standard implementation of a BackendCacheHitCopyingActor.
-  */
-class StandardCacheHitCopyingActor(val standardParams: StandardCacheHitCopyingActorParams)
-  extends BackendCacheHitCopyingActor with CacheHitDuplicating with StandardCachingActorHelper {
-
-  override protected def duplicate(source: Path, destination: Path): Unit = PathCopier.copy(source, destination).get
-
-  override lazy val jobDescriptor: BackendJobDescriptor = standardParams.jobDescriptor
-  override lazy val backendInitializationDataOption: Option[BackendInitializationData] =
-    standardParams.backendInitializationDataOption
-  override lazy val serviceRegistryActor: ActorRef = standardParams.serviceRegistryActor
-  override lazy val configurationDescriptor: BackendConfigurationDescriptor = standardParams.configurationDescriptor
-  override lazy val destinationCallRootPath: Path = jobPaths.callRoot
-  override lazy val destinationJobDetritusPaths: Map[String, Path] = jobPaths.detritusPaths
-}
diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardCachingActorHelper.scala b/backend/src/main/scala/cromwell/backend/standard/StandardCachingActorHelper.scala
index 9dc801512..c6c0b5183 100644
--- a/backend/src/main/scala/cromwell/backend/standard/StandardCachingActorHelper.scala
+++ b/backend/src/main/scala/cromwell/backend/standard/StandardCachingActorHelper.scala
@@ -2,13 +2,13 @@ package cromwell.backend.standard
 
 import akka.actor.{Actor, ActorRef}
 import cromwell.backend._
-import cromwell.backend.callcaching.JobCachingActorHelper
+import cromwell.backend.standard.callcaching.JobCachingActorHelper
 import cromwell.backend.io.{JobPaths, WorkflowPaths}
 import cromwell.backend.validation.{RuntimeAttributesValidation, ValidatedRuntimeAttributes}
 import cromwell.core.logging.JobLogging
 import cromwell.core.path.Path
 import cromwell.services.metadata.CallMetadataKeys
-import wdl4s.TaskCall
+import wdl4s.wdl.WdlTaskCall
 
 import scala.util.Try
 
@@ -36,7 +36,7 @@ trait StandardCachingActorHelper extends JobCachingActorHelper {
   def serviceRegistryActor: ActorRef
 
   // So... JobPaths doesn't extend WorkflowPaths, but does contain a self-type
-  lazy val workflowPaths: WorkflowPaths = jobPaths.asInstanceOf[WorkflowPaths]
+  lazy val workflowPaths: WorkflowPaths = jobPaths.workflowPaths
 
   def getPath(str: String): Try[Path] = workflowPaths.getPath(str)
 
@@ -46,7 +46,7 @@ trait StandardCachingActorHelper extends JobCachingActorHelper {
     */
   lazy val workflowDescriptor: BackendWorkflowDescriptor = jobDescriptor.workflowDescriptor
 
-  lazy val call: TaskCall = jobDescriptor.key.call
+  lazy val call: WdlTaskCall = jobDescriptor.key.call
 
   lazy val standardInitializationData: StandardInitializationData = BackendInitializationData.
     as[StandardInitializationData](backendInitializationDataOption)
@@ -79,7 +79,7 @@ trait StandardCachingActorHelper extends JobCachingActorHelper {
   }
 
   /**
-    * Returns any custom medatata for the backend.
+    * Returns any custom metadata for the backend.
     *
     * @return any custom metadata for the backend.
     */
diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardExpressionFunctions.scala b/backend/src/main/scala/cromwell/backend/standard/StandardExpressionFunctions.scala
index 3b29b796d..010240319 100644
--- a/backend/src/main/scala/cromwell/backend/standard/StandardExpressionFunctions.scala
+++ b/backend/src/main/scala/cromwell/backend/standard/StandardExpressionFunctions.scala
@@ -4,8 +4,8 @@ import cromwell.backend.io.GlobFunctions
 import cromwell.backend.wdl.{ReadLikeFunctions, WriteFunctions}
 import cromwell.core.CallContext
 import cromwell.core.path.{Path, PathBuilder}
-import wdl4s.expression.PureStandardLibraryFunctionsLike
-import wdl4s.values.{WdlFile, WdlValue}
+import wdl4s.wdl.expression.PureStandardLibraryFunctionsLike
+import wdl4s.wdl.values.{WdlFile, WdlValue}
 
 import scala.util.{Success, Try}
 
diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardFinalizationActor.scala b/backend/src/main/scala/cromwell/backend/standard/StandardFinalizationActor.scala
index 6acbe6cb7..09d80a77e 100644
--- a/backend/src/main/scala/cromwell/backend/standard/StandardFinalizationActor.scala
+++ b/backend/src/main/scala/cromwell/backend/standard/StandardFinalizationActor.scala
@@ -6,14 +6,14 @@ import cromwell.backend.io.WorkflowPaths
 import cromwell.core.CallOutputs
 import cromwell.core.Dispatcher.IoDispatcher
 import cromwell.core.path.{Path, PathCopier}
-import wdl4s.TaskCall
+import wdl4s.wdl.WdlTaskCall
 
 import scala.concurrent.Future
 
 trait StandardFinalizationActorParams {
   def workflowDescriptor: BackendWorkflowDescriptor
 
-  def calls: Set[TaskCall]
+  def calls: Set[WdlTaskCall]
 
   def jobExecutionMap: JobExecutionMap
 
@@ -27,7 +27,7 @@ trait StandardFinalizationActorParams {
 case class DefaultStandardFinalizationActorParams
 (
   workflowDescriptor: BackendWorkflowDescriptor,
-  calls: Set[TaskCall],
+  calls: Set[WdlTaskCall],
   jobExecutionMap: JobExecutionMap,
   workflowOutputs: CallOutputs,
   initializationDataOption: Option[BackendInitializationData],
@@ -45,7 +45,7 @@ class StandardFinalizationActor(val standardParams: StandardFinalizationActorPar
   extends BackendWorkflowFinalizationActor {
 
   override lazy val workflowDescriptor: BackendWorkflowDescriptor = standardParams.workflowDescriptor
-  override lazy val calls: Set[TaskCall] = standardParams.calls
+  override lazy val calls: Set[WdlTaskCall] = standardParams.calls
   lazy val initializationDataOption: Option[BackendInitializationData] = standardParams.initializationDataOption
   lazy val jobExecutionMap: JobExecutionMap = standardParams.jobExecutionMap
   lazy val workflowOutputs: CallOutputs = standardParams.workflowOutputs
diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardInitializationActor.scala b/backend/src/main/scala/cromwell/backend/standard/StandardInitializationActor.scala
index d958cdefc..c5f2c5ed8 100644
--- a/backend/src/main/scala/cromwell/backend/standard/StandardInitializationActor.scala
+++ b/backend/src/main/scala/cromwell/backend/standard/StandardInitializationActor.scala
@@ -7,8 +7,8 @@ import cromwell.backend.wfs.WorkflowPathBuilder
 import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendWorkflowDescriptor, BackendWorkflowInitializationActor}
 import cromwell.core.WorkflowOptions
 import cromwell.core.path.{DefaultPathBuilder, PathBuilder}
-import wdl4s.TaskCall
-import wdl4s.values.WdlValue
+import wdl4s.wdl.WdlTaskCall
+import wdl4s.wdl.values.WdlValue
 
 import scala.concurrent.Future
 import scala.util.Try
@@ -16,7 +16,7 @@ import scala.util.Try
 trait StandardInitializationActorParams {
   def workflowDescriptor: BackendWorkflowDescriptor
 
-  def calls: Set[TaskCall]
+  def calls: Set[WdlTaskCall]
 
   def serviceRegistryActor: ActorRef
 
@@ -26,9 +26,11 @@ trait StandardInitializationActorParams {
 case class DefaultInitializationActorParams
 (
   workflowDescriptor: BackendWorkflowDescriptor,
-  calls: Set[TaskCall],
+  ioActor: ActorRef,
+  calls: Set[WdlTaskCall],
   serviceRegistryActor: ActorRef,
-  configurationDescriptor: BackendConfigurationDescriptor
+  configurationDescriptor: BackendConfigurationDescriptor,
+  restarting: Boolean
 ) extends StandardInitializationActorParams
 
 /**
@@ -43,21 +45,21 @@ class StandardInitializationActor(val standardParams: StandardInitializationActo
 
   override lazy val serviceRegistryActor: ActorRef = standardParams.serviceRegistryActor
 
-  override lazy val calls: Set[TaskCall] = standardParams.calls
+  override lazy val calls: Set[WdlTaskCall] = standardParams.calls
 
   override def beforeAll(): Future[Option[BackendInitializationData]] = {
-    Future.fromTry(Try(Option(initializationData)))
+    initializationData map Option.apply
   }
 
-  lazy val initializationData: StandardInitializationData =
-    new StandardInitializationData(workflowPaths, runtimeAttributesBuilder, classOf[StandardExpressionFunctions])
+  lazy val initializationData: Future[StandardInitializationData] =
+    workflowPaths map { new StandardInitializationData(_, runtimeAttributesBuilder, classOf[StandardExpressionFunctions]) }
 
   lazy val expressionFunctions: Class[_ <: StandardExpressionFunctions] = classOf[StandardExpressionFunctions]
 
-  lazy val pathBuilders: List[PathBuilder] = List(DefaultPathBuilder)
+  lazy val pathBuilders: Future[List[PathBuilder]] = Future.successful(List(DefaultPathBuilder))
 
-  lazy val workflowPaths: WorkflowPaths =
-    WorkflowPathBuilder.workflowPaths(configurationDescriptor, workflowDescriptor, pathBuilders)
+  lazy val workflowPaths: Future[WorkflowPaths] =
+    pathBuilders map { WorkflowPathBuilder.workflowPaths(configurationDescriptor, workflowDescriptor, _) }
 
   /**
     * Returns the runtime attribute builder for this backend.
@@ -67,7 +69,7 @@ class StandardInitializationActor(val standardParams: StandardInitializationActo
     * @return runtime attributes builder with possible custom validations
     */
   def runtimeAttributesBuilder: StandardValidatedRuntimeAttributesBuilder =
-    StandardValidatedRuntimeAttributesBuilder.default
+      StandardValidatedRuntimeAttributesBuilder.default(configurationDescriptor.backendRuntimeConfig)
 
   override protected lazy val runtimeAttributeValidators: Map[String, (Option[WdlValue]) => Boolean] = {
     runtimeAttributesBuilder.validatorMap
diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardInitializationData.scala b/backend/src/main/scala/cromwell/backend/standard/StandardInitializationData.scala
index e4dae0e0b..734377dd5 100644
--- a/backend/src/main/scala/cromwell/backend/standard/StandardInitializationData.scala
+++ b/backend/src/main/scala/cromwell/backend/standard/StandardInitializationData.scala
@@ -15,7 +15,7 @@ class StandardInitializationData
     standardExpressionFunctionsClass.getConstructor(classOf[StandardExpressionFunctionsParams])
 
   def expressionFunctions(jobPaths: JobPaths): StandardExpressionFunctions = {
-    val pathBuilders = jobPaths.asInstanceOf[WorkflowPaths].pathBuilders
+    val pathBuilders = jobPaths.workflowPaths.pathBuilders
     val callContext = jobPaths.callContext
     val standardParams = DefaultStandardExpressionFunctionsParams(pathBuilders, callContext)
     standardExpressionFunctionsConstructor.newInstance(standardParams)
diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardJobExecutionActorParams.scala b/backend/src/main/scala/cromwell/backend/standard/StandardJobExecutionActorParams.scala
index 83e29a14e..9a21c59c3 100644
--- a/backend/src/main/scala/cromwell/backend/standard/StandardJobExecutionActorParams.scala
+++ b/backend/src/main/scala/cromwell/backend/standard/StandardJobExecutionActorParams.scala
@@ -9,6 +9,9 @@ import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationDa
 trait StandardJobExecutionActorParams {
   /** The service registry actor for key/value and metadata. */
   def serviceRegistryActor: ActorRef
+  
+  /** Actor able to handle IO requests asynchronously */
+  def ioActor: ActorRef
 
   /** The descriptor of this job. */
   def jobDescriptor: BackendJobDescriptor
diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardLifecycleActorFactory.scala b/backend/src/main/scala/cromwell/backend/standard/StandardLifecycleActorFactory.scala
index ab11ccd82..9c692c9a2 100644
--- a/backend/src/main/scala/cromwell/backend/standard/StandardLifecycleActorFactory.scala
+++ b/backend/src/main/scala/cromwell/backend/standard/StandardLifecycleActorFactory.scala
@@ -3,11 +3,12 @@ package cromwell.backend.standard
 import akka.actor.{ActorRef, Props}
 import com.typesafe.config.Config
 import cromwell.backend._
+import cromwell.backend.standard.callcaching._
 import cromwell.core.Dispatcher.BackendDispatcher
 import cromwell.core.path.Path
 import cromwell.core.{CallOutputs, Dispatcher}
-import wdl4s.TaskCall
-import wdl4s.expression.WdlStandardLibraryFunctions
+import wdl4s.wdl.WdlTaskCall
+import wdl4s.wdl.expression.WdlStandardLibraryFunctions
 
 /**
   * May be extended for using the standard sync/async backend pattern.
@@ -56,48 +57,79 @@ trait StandardLifecycleActorFactory extends BackendLifecycleActorFactory {
     *
     * @return the cache hit copying class.
     */
-  lazy val cacheHitCopyingActorClassOption: Option[Class[_ <: StandardCacheHitCopyingActor]] =
-    Option(classOf[StandardCacheHitCopyingActor])
+  lazy val cacheHitCopyingActorClassOption: Option[Class[_ <: StandardCacheHitCopyingActor]] = Option(classOf[DefaultStandardCacheHitCopyingActor])
+
+  /**
+    * Returns the cache hit copying class.
+    *
+    * @return the cache hit copying class.
+    */
+  lazy val fileHashingActorClassOption: Option[Class[_ <: StandardFileHashingActor]] = Option(classOf[DefaultStandardFileHashingActor])
 
   /**
     * Returns the finalization class.
     *
     * @return the finalization class.
     */
-  lazy val finalizationActorClassOption: Option[Class[_ <: StandardFinalizationActor]] =
-    Option(classOf[StandardFinalizationActor])
+  lazy val finalizationActorClassOption: Option[Class[_ <: StandardFinalizationActor]] = Option(classOf[StandardFinalizationActor])
 
-  override def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[TaskCall],
-                                                serviceRegistryActor: ActorRef): Option[Props] = {
-    val params = workflowInitializationActorParams(workflowDescriptor, calls, serviceRegistryActor)
+  override def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor, ioActor: ActorRef, calls: Set[WdlTaskCall],
+                                                serviceRegistryActor: ActorRef, restart: Boolean): Option[Props] = {
+    val params = workflowInitializationActorParams(workflowDescriptor, ioActor, calls, serviceRegistryActor, restart)
     val props = Props(initializationActorClass, params).withDispatcher(Dispatcher.BackendDispatcher)
     Option(props)
   }
 
-  def workflowInitializationActorParams(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[TaskCall],
-                                        serviceRegistryActor: ActorRef): StandardInitializationActorParams = {
-    DefaultInitializationActorParams(workflowDescriptor, calls, serviceRegistryActor, configurationDescriptor)
+  def workflowInitializationActorParams(workflowDescriptor: BackendWorkflowDescriptor, ioActor: ActorRef, calls: Set[WdlTaskCall],
+                                        serviceRegistryActor: ActorRef, restarting: Boolean): StandardInitializationActorParams = {
+    DefaultInitializationActorParams(workflowDescriptor, ioActor, calls, serviceRegistryActor, configurationDescriptor, restarting)
   }
 
   override def jobExecutionActorProps(jobDescriptor: BackendJobDescriptor,
                                       initializationDataOption: Option[BackendInitializationData],
                                       serviceRegistryActor: ActorRef,
+                                      ioActor: ActorRef,
                                       backendSingletonActorOption: Option[ActorRef]): Props = {
     val params = jobExecutionActorParams(jobDescriptor, initializationDataOption, serviceRegistryActor,
-      backendSingletonActorOption)
+      ioActor, backendSingletonActorOption)
     Props(new StandardSyncExecutionActor(params)).withDispatcher(Dispatcher.BackendDispatcher)
   }
 
   def jobExecutionActorParams(jobDescriptor: BackendJobDescriptor,
                               initializationDataOption: Option[BackendInitializationData],
                               serviceRegistryActor: ActorRef,
+                              ioActor: ActorRef,
                               backendSingletonActorOption: Option[ActorRef]): StandardSyncExecutionActorParams = {
-    DefaultStandardSyncExecutionActorParams(jobIdKey, serviceRegistryActor, jobDescriptor, configurationDescriptor,
+    DefaultStandardSyncExecutionActorParams(jobIdKey, serviceRegistryActor, ioActor, jobDescriptor, configurationDescriptor,
       initializationDataOption, backendSingletonActorOption, asyncExecutionActorClass)
   }
 
+  override def fileHashingActorProps:
+  Option[(BackendJobDescriptor, Option[BackendInitializationData], ActorRef, ActorRef) => Props] = {
+    fileHashingActorClassOption map {
+      standardFileHashingActor => fileHashingActorInner(standardFileHashingActor) _
+    }
+  }
+  
+  def fileHashingActorInner(standardFileHashingActor: Class[_ <: StandardFileHashingActor])
+                               (jobDescriptor: BackendJobDescriptor,
+                                initializationDataOption: Option[BackendInitializationData],
+                                serviceRegistryActor: ActorRef,
+                                ioActor: ActorRef): Props = {
+    val params = fileHashingActorParams(jobDescriptor, initializationDataOption, serviceRegistryActor, ioActor)
+    Props(standardFileHashingActor, params).withDispatcher(BackendDispatcher)
+  }
+
+  def fileHashingActorParams(jobDescriptor: BackendJobDescriptor,
+                                 initializationDataOption: Option[BackendInitializationData],
+                                 serviceRegistryActor: ActorRef,
+                                 ioActor: ActorRef): StandardFileHashingActorParams = {
+    DefaultStandardFileHashingActorParams(
+      jobDescriptor, initializationDataOption, serviceRegistryActor, ioActor, configurationDescriptor)
+  }
+
   override def cacheHitCopyingActorProps:
-  Option[(BackendJobDescriptor, Option[BackendInitializationData], ActorRef) => Props] = {
+  Option[(BackendJobDescriptor, Option[BackendInitializationData], ActorRef, ActorRef) => Props] = {
     cacheHitCopyingActorClassOption map {
       standardCacheHitCopyingActor => cacheHitCopyingActorInner(standardCacheHitCopyingActor) _
     }
@@ -106,29 +138,31 @@ trait StandardLifecycleActorFactory extends BackendLifecycleActorFactory {
   def cacheHitCopyingActorInner(standardCacheHitCopyingActor: Class[_ <: StandardCacheHitCopyingActor])
                                (jobDescriptor: BackendJobDescriptor,
                                 initializationDataOption: Option[BackendInitializationData],
-                                serviceRegistryActor: ActorRef): Props = {
-    val params = cacheHitCopyingActorParams(jobDescriptor, initializationDataOption, serviceRegistryActor)
+                                serviceRegistryActor: ActorRef,
+                                ioActor: ActorRef): Props = {
+    val params = cacheHitCopyingActorParams(jobDescriptor, initializationDataOption, serviceRegistryActor, ioActor)
     Props(standardCacheHitCopyingActor, params).withDispatcher(BackendDispatcher)
   }
 
   def cacheHitCopyingActorParams(jobDescriptor: BackendJobDescriptor,
                                  initializationDataOption: Option[BackendInitializationData],
-                                 serviceRegistryActor: ActorRef): StandardCacheHitCopyingActorParams = {
+                                 serviceRegistryActor: ActorRef,
+                                 ioActor: ActorRef): StandardCacheHitCopyingActorParams = {
     DefaultStandardCacheHitCopyingActorParams(
-      jobDescriptor, initializationDataOption, serviceRegistryActor, configurationDescriptor)
+      jobDescriptor, initializationDataOption, serviceRegistryActor, ioActor, configurationDescriptor)
   }
 
-  override def workflowFinalizationActorProps(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[TaskCall],
+  override def workflowFinalizationActorProps(workflowDescriptor: BackendWorkflowDescriptor, ioActor: ActorRef, calls: Set[WdlTaskCall],
                                               jobExecutionMap: JobExecutionMap, workflowOutputs: CallOutputs,
                                               initializationData: Option[BackendInitializationData]): Option[Props] = {
     finalizationActorClassOption map { finalizationActorClass =>
-      val params = workflowFinalizationActorParams(workflowDescriptor, calls, jobExecutionMap, workflowOutputs,
+      val params = workflowFinalizationActorParams(workflowDescriptor, ioActor, calls, jobExecutionMap, workflowOutputs,
         initializationData)
       Props(finalizationActorClass, params).withDispatcher(BackendDispatcher)
     }
   }
 
-  def workflowFinalizationActorParams(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[TaskCall],
+  def workflowFinalizationActorParams(workflowDescriptor: BackendWorkflowDescriptor, ioActor: ActorRef, calls: Set[WdlTaskCall],
                                       jobExecutionMap: JobExecutionMap, workflowOutputs: CallOutputs,
                                       initializationDataOption: Option[BackendInitializationData]):
   StandardFinalizationActorParams = {
@@ -147,12 +181,18 @@ trait StandardLifecycleActorFactory extends BackendLifecycleActorFactory {
 
   override def getExecutionRootPath(workflowDescriptor: BackendWorkflowDescriptor, backendConfig: Config,
                                     initializationData: Option[BackendInitializationData]): Path = {
-    initializationData.get.asInstanceOf[StandardInitializationData].workflowPaths.executionRoot
+    initializationData match {
+      case Some(data) => data.asInstanceOf[StandardInitializationData].workflowPaths.executionRoot
+      case None => super.getExecutionRootPath(workflowDescriptor, backendConfig, initializationData)
+    }
   }
 
   override def getWorkflowExecutionRootPath(workflowDescriptor: BackendWorkflowDescriptor, backendConfig: Config,
                                             initializationData: Option[BackendInitializationData]): Path = {
-    initializationData.get.asInstanceOf[StandardInitializationData].workflowPaths.workflowRoot
+    initializationData match {
+      case Some(data) => data.asInstanceOf[StandardInitializationData].workflowPaths.workflowRoot
+      case None => super.getWorkflowExecutionRootPath(workflowDescriptor, backendConfig, initializationData)
+    }
   }
 
   override def runtimeAttributeDefinitions(initializationDataOption: Option[BackendInitializationData]):
diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardSyncExecutionActor.scala b/backend/src/main/scala/cromwell/backend/standard/StandardSyncExecutionActor.scala
index 11e487a0b..180f4d345 100644
--- a/backend/src/main/scala/cromwell/backend/standard/StandardSyncExecutionActor.scala
+++ b/backend/src/main/scala/cromwell/backend/standard/StandardSyncExecutionActor.scala
@@ -10,7 +10,6 @@ import cromwell.core.Dispatcher
 import cromwell.services.keyvalue.KeyValueServiceActor._
 
 import scala.concurrent.{Future, Promise}
-import scala.language.existentials
 
 trait StandardSyncExecutionActorParams extends StandardJobExecutionActorParams {
   /** The class for creating an async backend. */
@@ -21,6 +20,7 @@ case class DefaultStandardSyncExecutionActorParams
 (
   override val jobIdKey: String,
   override val serviceRegistryActor: ActorRef,
+  override val ioActor: ActorRef,
   override val jobDescriptor: BackendJobDescriptor,
   override val configurationDescriptor: BackendConfigurationDescriptor,
   override val backendInitializationDataOption: Option[BackendInitializationData],
@@ -57,7 +57,7 @@ class StandardSyncExecutionActor(val standardParams: StandardSyncExecutionActorP
   val jobIdKey: String = standardParams.jobIdKey
   val serviceRegistryActor: ActorRef = standardParams.serviceRegistryActor
 
-  context.become(startup orElse super.receive)
+  context.become(startup orElse receive)
 
   private def startup: Receive = {
     case AbortJobCommand =>
@@ -92,14 +92,14 @@ class StandardSyncExecutionActor(val standardParams: StandardSyncExecutionActorP
 
   override def execute: Future[BackendJobExecutionResponse] = {
     val executorRef = createAsyncRef()
-    context.become(running(executorRef) orElse super.receive)
+    context.become(running(executorRef) orElse receive)
     executorRef ! Execute
     completionPromise.future
   }
 
   override def recover: Future[BackendJobExecutionResponse] = {
     val executorRef = createAsyncRef()
-    context.become(running(executorRef) orElse super.receive)
+    context.become(running(executorRef) orElse receive)
     val kvJobKey =
       KvJobKey(jobDescriptor.key.call.fullyQualifiedName, jobDescriptor.key.index, jobDescriptor.key.attempt)
     val kvGet = KvGet(ScopedKey(jobDescriptor.workflowDescriptor.id, kvJobKey, jobIdKey))
@@ -111,6 +111,7 @@ class StandardSyncExecutionActor(val standardParams: StandardSyncExecutionActorP
     DefaultStandardAsyncExecutionActorParams(
       standardParams.jobIdKey,
       standardParams.serviceRegistryActor,
+      standardParams.ioActor,
       standardParams.jobDescriptor,
       standardParams.configurationDescriptor,
       standardParams.backendInitializationDataOption,
diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilder.scala b/backend/src/main/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilder.scala
index 20a649e00..37f037b57 100644
--- a/backend/src/main/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilder.scala
+++ b/backend/src/main/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilder.scala
@@ -1,5 +1,6 @@
 package cromwell.backend.standard
 
+import com.typesafe.config.Config
 import cromwell.backend.validation._
 
 /**
@@ -29,8 +30,8 @@ object StandardValidatedRuntimeAttributesBuilder {
     *
     * Additional runtime attribute validations may be added by calling `withValidation` on the default.
     */
-  lazy val default: StandardValidatedRuntimeAttributesBuilder = {
-    val required = Seq(ContinueOnReturnCodeValidation.default, FailOnStderrValidation.default)
+  def default(backendRuntimeConfig: Option[Config]): StandardValidatedRuntimeAttributesBuilder = {
+    val required = Seq(ContinueOnReturnCodeValidation.default(backendRuntimeConfig), FailOnStderrValidation.default(backendRuntimeConfig))
     val custom = Seq.empty
     StandardValidatedRuntimeAttributesBuilderImpl(custom, required)
   }
diff --git a/backend/src/main/scala/cromwell/backend/callcaching/JobCachingActorHelper.scala b/backend/src/main/scala/cromwell/backend/standard/callcaching/JobCachingActorHelper.scala
similarity index 96%
rename from backend/src/main/scala/cromwell/backend/callcaching/JobCachingActorHelper.scala
rename to backend/src/main/scala/cromwell/backend/standard/callcaching/JobCachingActorHelper.scala
index 8395f5264..89ce5739e 100644
--- a/backend/src/main/scala/cromwell/backend/callcaching/JobCachingActorHelper.scala
+++ b/backend/src/main/scala/cromwell/backend/standard/callcaching/JobCachingActorHelper.scala
@@ -1,4 +1,4 @@
-package cromwell.backend.callcaching
+package cromwell.backend.standard.callcaching
 
 import akka.actor.Actor
 import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptor}
diff --git a/backend/src/main/scala/cromwell/backend/standard/callcaching/StandardCacheHitCopyingActor.scala b/backend/src/main/scala/cromwell/backend/standard/callcaching/StandardCacheHitCopyingActor.scala
new file mode 100644
index 000000000..e79f2e863
--- /dev/null
+++ b/backend/src/main/scala/cromwell/backend/standard/callcaching/StandardCacheHitCopyingActor.scala
@@ -0,0 +1,318 @@
+package cromwell.backend.standard.callcaching
+
+import java.util.concurrent.TimeoutException
+
+import akka.actor.{ActorRef, FSM}
+import cats.instances.list._
+import cats.instances.set._
+import cats.instances.tuple._
+import cats.syntax.foldable._
+import cromwell.backend.BackendCacheHitCopyingActor.CopyOutputsCommand
+import cromwell.backend.BackendJobExecutionActor.{AbortedResponse, JobFailedNonRetryableResponse, JobSucceededResponse}
+import cromwell.backend.BackendLifecycleActor.AbortJobCommand
+import cromwell.backend.io.JobPaths
+import cromwell.backend.standard.StandardCachingActorHelper
+import cromwell.backend.standard.callcaching.StandardCacheHitCopyingActor._
+import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendJobDescriptor}
+import cromwell.core._
+import cromwell.core.io._
+import cromwell.core.logging.JobLogging
+import cromwell.core.path.{Path, PathCopier}
+import cromwell.core.simpleton.{WdlValueBuilder, WdlValueSimpleton}
+import wdl4s.wdl.values.WdlFile
+
+import scala.util.{Failure, Success, Try}
+
+/**
+  * Trait of parameters passed to a StandardCacheHitCopyingActor.
+  */
+trait StandardCacheHitCopyingActorParams {
+  def jobDescriptor: BackendJobDescriptor
+
+  def backendInitializationDataOption: Option[BackendInitializationData]
+
+  def serviceRegistryActor: ActorRef
+
+  def ioActor: ActorRef
+
+  def configurationDescriptor: BackendConfigurationDescriptor
+}
+
+/** A default implementation of the cache hit copying params. */
+case class DefaultStandardCacheHitCopyingActorParams
+(
+  override val jobDescriptor: BackendJobDescriptor,
+  override val backendInitializationDataOption: Option[BackendInitializationData],
+  override val serviceRegistryActor: ActorRef,
+  override val ioActor: ActorRef,
+  override val configurationDescriptor: BackendConfigurationDescriptor
+) extends StandardCacheHitCopyingActorParams
+
+object StandardCacheHitCopyingActor {
+  type DetritusMap = Map[String, Path]
+  type PathPair = (Path, Path)
+
+  sealed trait StandardCacheHitCopyingActorState
+  case object Idle extends StandardCacheHitCopyingActorState
+  case object WaitingForIoResponses extends StandardCacheHitCopyingActorState
+  case object FailedState extends StandardCacheHitCopyingActorState
+  case object WaitingForOnSuccessResponse extends StandardCacheHitCopyingActorState
+
+  // TODO: this mechanism here is very close to the one in CallCacheHashingJobActorData
+  // Abstracting it might be valuable
+  /**
+    * The head subset of commandsToWaitFor is sent to the IoActor as a bulk.
+    * When a response comes back, the corresponding command is removed from the head set.
+    * When the head set is empty, it is removed and the next subset is sent, until there is no subset left.
+    * If at any point a response comes back as a failure. Other responses for the current set will be awaited for 
+    * but subsequent sets will not be sent and the actor will send back a failure message.
+    */
+  case class StandardCacheHitCopyingActorData(commandsToWaitFor: List[Set[IoCommand[_]]],
+                                              newJobOutputs: CallOutputs,
+                                              newDetritus: DetritusMap,
+                                              returnCode: Option[Int]
+                                             ) {
+
+    /**
+      * Removes the command from commandsToWaitFor
+      * returns a pair of the new state data and CommandSetState giving information about what to do next
+      */
+    def commandComplete(command: IoCommand[_]): (StandardCacheHitCopyingActorData, CommandSetState) = commandsToWaitFor match {
+      // If everything was already done send back current data and AllCommandsDone
+      case Nil => (this, AllCommandsDone)
+      case lastSubset :: Nil =>
+        val updatedSubset = lastSubset - command
+        // If the last subset is now empty, we're done
+        if (updatedSubset.isEmpty) (this.copy(commandsToWaitFor = List.empty), AllCommandsDone)
+        // otherwise update commandsToWaitFor and keep waiting
+        else (this.copy(commandsToWaitFor = List(updatedSubset)), StillWaiting)
+      case currentSubset :: otherSubsets =>
+        val updatedSubset = currentSubset - command
+        // This subset is done but there are other ones, remove it from commandsToWaitFor and return the next round of commands
+        if (updatedSubset.isEmpty) (this.copy(commandsToWaitFor = otherSubsets), NextSubSet(otherSubsets.head))
+        // otherwise update the head susbset and keep waiting  
+        else (this.copy(commandsToWaitFor = List(updatedSubset) ++ otherSubsets), StillWaiting)
+    }
+  }
+
+  // Internal ADT to keep track of command set states
+  private[callcaching] sealed trait CommandSetState
+  private[callcaching] case object StillWaiting extends CommandSetState
+  private[callcaching] case object AllCommandsDone extends CommandSetState
+  private[callcaching] case class NextSubSet(commands: Set[IoCommand[_]]) extends CommandSetState
+}
+
+class DefaultStandardCacheHitCopyingActor(standardParams: StandardCacheHitCopyingActorParams) extends StandardCacheHitCopyingActor(standardParams) with DefaultIoCommandBuilder
+
+/**
+  * Standard implementation of a BackendCacheHitCopyingActor.
+  */
+abstract class StandardCacheHitCopyingActor(val standardParams: StandardCacheHitCopyingActorParams)
+  extends FSM[StandardCacheHitCopyingActorState, Option[StandardCacheHitCopyingActorData]] with JobLogging with StandardCachingActorHelper with IoClientHelper { this: IoCommandBuilder =>
+
+  override lazy val jobDescriptor: BackendJobDescriptor = standardParams.jobDescriptor
+  override lazy val backendInitializationDataOption: Option[BackendInitializationData] = standardParams.backendInitializationDataOption
+  override lazy val serviceRegistryActor: ActorRef = standardParams.serviceRegistryActor
+  override lazy val configurationDescriptor: BackendConfigurationDescriptor = standardParams.configurationDescriptor
+
+  lazy val destinationCallRootPath: Path = jobPaths.callRoot
+  lazy val destinationJobDetritusPaths: Map[String, Path] = jobPaths.detritusPaths
+  lazy val ioActor = standardParams.ioActor
+
+  startWith(Idle, None)
+
+  context.become(ioReceive orElse receive)
+
+  /** Override this method if you want to provide an alternative way to duplicate files than copying them. */
+  protected def duplicate(copyPairs: Set[PathPair]): Option[Try[Unit]] = None
+
+  when(Idle) {
+    case Event(CopyOutputsCommand(simpletons, jobDetritus, returnCode), None) =>
+
+      // Try to make a Path of the callRootPath from the detritus
+      lookupSourceCallRootPath(jobDetritus) match {
+        case Success(sourceCallRootPath) =>
+          
+          // process simpletons and detritus to get updated paths and corresponding IoCommands
+          val processed = for {
+            (destinationCallOutputs, simpletonIoCommands) <- processSimpletons(simpletons, sourceCallRootPath)
+            (destinationDetritus, detritusIoCommands) <- processDetritus(jobDetritus)
+          } yield (destinationCallOutputs, destinationDetritus, simpletonIoCommands ++ detritusIoCommands)
+
+          processed match {
+            case Success((destinationCallOutputs, destinationDetritus, detritusAndOutputsIoCommands)) =>
+              duplicate(ioCommandsToCopyPairs(detritusAndOutputsIoCommands)) match {
+                  // Use the duplicate override if exists
+                case Some(Success(_)) => succeedAndStop(returnCode, destinationCallOutputs, destinationDetritus)
+                case Some(Failure(failure)) => failAndStop(failure)
+                  // Otherwise send the first round of IoCommands (file outputs and detritus) if any
+                case None if detritusAndOutputsIoCommands.nonEmpty =>
+                    detritusAndOutputsIoCommands foreach sendIoCommand
+
+                    // Add potential additional commands to the list
+                  val additionalCommands = additionalIoCommands(sourceCallRootPath, simpletons, destinationCallOutputs, jobDetritus, destinationDetritus)
+                  val allCommands = List(detritusAndOutputsIoCommands) ++ additionalCommands
+
+                    goto(WaitingForIoResponses) using Option(StandardCacheHitCopyingActorData(allCommands, destinationCallOutputs, destinationDetritus, returnCode))
+                case _ => succeedAndStop(returnCode, destinationCallOutputs, destinationDetritus)
+              }
+
+            case Failure(failure) => failAndStop(failure)
+          }
+
+        case Failure(failure) => failAndStop(failure)
+      }
+  }
+
+  when(WaitingForIoResponses) {
+    case Event(IoSuccess(command: IoCommand[_], _), Some(data)) =>
+      val (newData, commandState) = data.commandComplete(command)
+
+      commandState match {
+        case StillWaiting => stay() using Option(newData)
+        case AllCommandsDone => succeedAndStop(newData.returnCode, newData.newJobOutputs, newData.newDetritus)
+        case NextSubSet(commands) =>
+          commands foreach sendIoCommand
+          stay() using Option(newData)
+      }
+    case Event(IoFailure(command: IoCommand[_], failure), Some(data)) =>
+      // any failure is fatal
+      context.parent ! JobFailedNonRetryableResponse(jobDescriptor.key, failure, None)
+
+      val (newData, commandState) = data.commandComplete(command)
+
+      commandState match {
+        // If we're still waiting for some responses, go to failed state
+        case StillWaiting => goto(FailedState) using Option(newData)
+        // Otherwise we're done
+        case _ =>
+          context stop self
+          stay()
+      }
+    // Should not be possible
+    case Event(IoFailure(_: IoCommand[_], failure), None) => failAndStop(failure)
+  }
+
+  when(FailedState) {
+    // At this point success or failure doesn't matter, we've already failed this hit
+    case Event(response: IoAck[_], Some(data)) =>
+      val (newData, commandState) = data.commandComplete(response.command)
+      commandState match {
+        // If we're still waiting for some responses, stay
+        case StillWaiting => stay() using Option(newData)
+        // Otherwise we're done
+        case _ =>
+          context stop self
+          stay()
+      }
+  }
+
+  whenUnhandled {
+    case Event(AbortJobCommand, _) =>
+      abort()
+    case Event(unexpected, _) =>
+      log.warning(s"Backend cache hit copying actor received an unexpected message: $unexpected in state $stateName")
+      stay()
+  }
+
+  def succeedAndStop(returnCode: Option[Int], copiedJobOutputs: CallOutputs, detritusMap: DetritusMap) = {
+    import cromwell.services.metadata.MetadataService.implicits.MetadataAutoPutter
+    serviceRegistryActor.putMetadata(jobDescriptor.workflowDescriptor.id, Option(jobDescriptor.key), startMetadataKeyValues)
+    context.parent ! JobSucceededResponse(jobDescriptor.key, returnCode, copiedJobOutputs, Option(detritusMap), Seq.empty, None)
+    context stop self
+    stay()
+  }
+
+  def failAndStop(failure: Throwable) = {
+    context.parent ! JobFailedNonRetryableResponse(jobDescriptor.key, failure, None)
+    context stop self
+    stay()
+  }
+
+  def abort() = {
+    log.warning("{}: Abort not supported during cache hit copying", jobTag)
+    context.parent ! AbortedResponse(jobDescriptor.key)
+    context stop self
+    stay()
+  }
+
+  protected def lookupSourceCallRootPath(sourceJobDetritusFiles: Map[String, String]): Try[Path] = {
+    sourceJobDetritusFiles.get(JobPaths.CallRootPathKey) match {
+      case Some(source) => getPath(source)
+      case None => Failure(new RuntimeException(s"${JobPaths.CallRootPathKey} wasn't found for call ${jobDescriptor.call.fullyQualifiedName}"))
+    }
+  }
+
+  private def ioCommandsToCopyPairs(commands: Set[IoCommand[_]]): Set[PathPair] = commands collect {
+    case copyCommand: IoCopyCommand => copyCommand.source -> copyCommand.destination
+  }
+
+  /**
+    * Returns a pair of the list of simpletons with copied paths, and copy commands necessary to perform those copies. 
+    */
+  protected def processSimpletons(wdlValueSimpletons: Seq[WdlValueSimpleton], sourceCallRootPath: Path): Try[(CallOutputs, Set[IoCommand[_]])] = Try {
+    val (destinationSimpletons, ioCommands): (List[WdlValueSimpleton], Set[IoCommand[_]]) = wdlValueSimpletons.toList.foldMap({
+      case WdlValueSimpleton(key, wdlFile: WdlFile) =>
+        val sourcePath = getPath(wdlFile.value).get
+        val destinationPath = PathCopier.getDestinationFilePath(sourceCallRootPath, sourcePath, destinationCallRootPath)
+
+        val destinationSimpleton = WdlValueSimpleton(key, WdlFile(destinationPath.pathAsString))
+
+        List(destinationSimpleton) -> Set(copyCommand(sourcePath, destinationPath, overwrite = true))
+      case nonFileSimpleton => (List(nonFileSimpleton), Set.empty[IoCommand[_]])
+    })
+
+    (WdlValueBuilder.toJobOutputs(jobDescriptor.call.task.outputs, destinationSimpletons), ioCommands)
+  }
+
+  /**
+    * Returns the file (and ONLY the file detritus) intersection between the cache hit and this call.
+    */
+  protected final def detritusFileKeys(sourceJobDetritusFiles: Map[String, String]) = {
+    val sourceKeys = sourceJobDetritusFiles.keySet
+    val destinationKeys = destinationJobDetritusPaths.keySet
+    sourceKeys.intersect(destinationKeys).filterNot(_ == JobPaths.CallRootPathKey)
+  }
+
+  /**
+    * Returns a pair of the detritus with copied paths, and copy commands necessary to perform those copies. 
+    */
+  protected def processDetritus(sourceJobDetritusFiles: Map[String, String]): Try[(Map[String, Path], Set[IoCommand[_]])] = Try {
+    val fileKeys = detritusFileKeys(sourceJobDetritusFiles)
+
+    val zero = (Map.empty[String, Path], Set.empty[IoCommand[_]])
+
+    val (destinationDetritus, ioCommands) = fileKeys.foldLeft(zero)({
+      case ((detrituses, commands), detritus) =>
+        val sourcePath = getPath(sourceJobDetritusFiles(detritus)).get
+        val destinationPath = destinationJobDetritusPaths(detritus)
+
+        val newDetrituses = detrituses + (detritus -> destinationPath)
+
+        (newDetrituses, commands + copyCommand(sourcePath, destinationPath, overwrite = true))
+    })
+
+    (destinationDetritus + (JobPaths.CallRootPathKey -> destinationCallRootPath), ioCommands)
+  }
+
+  /**
+    * Additional IoCommands that will be sent after (and only after) output and detritus commands complete successfully.
+    * See StandardCacheHitCopyingActorData
+    */
+  protected def additionalIoCommands(sourceCallRootPath: Path,
+                                     originalSimpletons: Seq[WdlValueSimpleton],
+                                     newOutputs: CallOutputs,
+                                     originalDetritus:  Map[String, String],
+                                     newDetritus: Map[String, Path]): List[Set[IoCommand[_]]] = List.empty
+
+  override protected def onTimeout(message: Any, to: ActorRef): Unit = {
+    val exceptionMessage = message match {
+      case copyCommand: IoCopyCommand => s"The Cache hit copying actor timed out waiting for a response to copy ${copyCommand.source.pathAsString} to ${copyCommand.destination.pathAsString}"
+      case touchCommand: IoTouchCommand => s"The Cache hit copying actor timed out waiting for a response to touch ${touchCommand.file.pathAsString}"
+      case other => s"The Cache hit copying actor timed out waiting for an unknown I/O operation: $other"
+    }
+
+    failAndStop(new TimeoutException(exceptionMessage))
+    ()
+  }
+}
diff --git a/backend/src/main/scala/cromwell/backend/standard/callcaching/StandardFileHashingActor.scala b/backend/src/main/scala/cromwell/backend/standard/callcaching/StandardFileHashingActor.scala
new file mode 100644
index 000000000..bcb08d333
--- /dev/null
+++ b/backend/src/main/scala/cromwell/backend/standard/callcaching/StandardFileHashingActor.scala
@@ -0,0 +1,96 @@
+package cromwell.backend.standard.callcaching
+
+import akka.actor.{Actor, ActorLogging, ActorRef}
+import akka.event.LoggingAdapter
+import cromwell.backend.standard.StandardCachingActorHelper
+import cromwell.backend.standard.callcaching.StandardFileHashingActor.{FileHashResponse, SingleFileHashRequest}
+import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendJobDescriptor}
+import cromwell.core.JobKey
+import cromwell.core.callcaching._
+import cromwell.core.io._
+import cromwell.core.logging.JobLogging
+import wdl4s.wdl.values.WdlFile
+
+import scala.util.{Failure, Success, Try}
+
+/**
+  * Trait of parameters passed to a StandardCacheHitCopyingActor.
+  */
+trait StandardFileHashingActorParams {
+  def jobDescriptor: BackendJobDescriptor
+
+  def backendInitializationDataOption: Option[BackendInitializationData]
+
+  def serviceRegistryActor: ActorRef
+
+  def ioActor: ActorRef
+
+  def configurationDescriptor: BackendConfigurationDescriptor
+}
+
+/** A default implementation of the cache hit copying params. */
+case class DefaultStandardFileHashingActorParams
+(
+  override val jobDescriptor: BackendJobDescriptor,
+  override val backendInitializationDataOption: Option[BackendInitializationData],
+  override val serviceRegistryActor: ActorRef,
+  override val ioActor: ActorRef,
+  override val configurationDescriptor: BackendConfigurationDescriptor
+) extends StandardFileHashingActorParams
+
+class DefaultStandardFileHashingActor(standardParams: StandardFileHashingActorParams) extends StandardFileHashingActor(standardParams) with DefaultIoCommandBuilder
+
+object StandardFileHashingActor {
+  case class FileHashingFunction(work: (SingleFileHashRequest, LoggingAdapter) => Try[String])
+
+  sealed trait BackendSpecificHasherCommand { def jobKey: JobKey }
+  final case class SingleFileHashRequest(jobKey: JobKey, hashKey: HashKey, file: WdlFile, initializationData: Option[BackendInitializationData]) extends BackendSpecificHasherCommand
+
+  sealed trait BackendSpecificHasherResponse extends SuccessfulHashResultMessage
+  case class FileHashResponse(hashResult: HashResult) extends BackendSpecificHasherResponse { override def hashes = Set(hashResult) }
+}
+
+abstract class StandardFileHashingActor(standardParams: StandardFileHashingActorParams) extends Actor with ActorLogging with JobLogging with IoClientHelper with StandardCachingActorHelper {
+  this: IoCommandBuilder =>
+  override val ioActor = standardParams.ioActor
+  override lazy val jobDescriptor: BackendJobDescriptor = standardParams.jobDescriptor
+  override lazy val backendInitializationDataOption: Option[BackendInitializationData] = standardParams.backendInitializationDataOption
+  override lazy val serviceRegistryActor: ActorRef = standardParams.serviceRegistryActor
+  override lazy val configurationDescriptor: BackendConfigurationDescriptor = standardParams.configurationDescriptor
+
+  def customHashStrategy(fileRequest: SingleFileHashRequest): Option[Try[String]] = None
+
+  def fileHashingReceive: Receive = {
+    // Hash Request
+    case fileRequest: SingleFileHashRequest =>
+      val replyTo = sender()
+
+      customHashStrategy(fileRequest) match {
+        case Some(Success(result)) => context.parent ! FileHashResponse(HashResult(fileRequest.hashKey, HashValue(result)))
+        case Some(Failure(failure)) => context.parent ! HashingFailedMessage(fileRequest.file.value, failure)
+        case None => asyncHashing(fileRequest, replyTo)
+      }
+
+    // Hash Success
+    case (fileHashRequest: SingleFileHashRequest, IoSuccess(_, result: String)) =>
+      context.parent ! FileHashResponse(HashResult(fileHashRequest.hashKey, HashValue(result)))
+
+    // Hash Failure
+    case (fileHashRequest: SingleFileHashRequest, IoFailure(_, failure: Throwable)) =>
+      context.parent ! HashingFailedMessage(fileHashRequest.file.value, failure)
+
+    case other =>
+      log.warning(s"Async File hashing actor received unexpected message: $other")
+  }
+
+  def asyncHashing(fileRequest: SingleFileHashRequest, replyTo: ActorRef) = getPath(fileRequest.file.value) match {
+    case Success(gcsPath) => sendIoCommandWithContext(hashCommand(gcsPath), fileRequest)
+    case Failure(failure) => replyTo ! HashingFailedMessage(fileRequest.file.value, failure)
+  }
+
+  override def receive: Receive = ioReceive orElse fileHashingReceive
+
+  protected def onTimeout(message: Any, to: ActorRef): Unit = {
+    context.parent ! HashingServiceUnvailable
+  }
+}
diff --git a/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCode.scala b/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCode.scala
index 0639b4f42..ed2bcfad0 100644
--- a/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCode.scala
+++ b/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCode.scala
@@ -1,6 +1,6 @@
 package cromwell.backend.validation
 
-import wdl4s.types.{WdlArrayType, WdlBooleanType, WdlIntegerType, WdlType}
+import wdl4s.wdl.types._
 
 object ContinueOnReturnCode {
   val validWdlTypes = Set[WdlType](WdlArrayType(WdlIntegerType), WdlBooleanType, WdlIntegerType)
diff --git a/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala b/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala
index d0ca49339..27ae8170e 100644
--- a/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala
+++ b/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala
@@ -4,10 +4,11 @@ import cats.data.Validated.{Invalid, Valid}
 import cats.instances.list._
 import cats.syntax.traverse._
 import cats.syntax.validated._
+import com.typesafe.config.Config
 import cromwell.backend.validation.RuntimeAttributesValidation._
 import lenthall.validation.ErrorOr._
-import wdl4s.types.{WdlArrayType, WdlIntegerType, WdlStringType, WdlType}
-import wdl4s.values.{WdlArray, WdlBoolean, WdlInteger, WdlString, WdlValue}
+import wdl4s.wdl.types._
+import wdl4s.wdl.values._
 
 import scala.util.Try
 
@@ -17,15 +18,16 @@ import scala.util.Try
   *
   * `instance` returns an validation that errors when no attribute is specified.
   *
-  * The default returns a `ContinueOnReturnCodeSet(0)` when no attribute is specified.
+  * `configDefaultWdlValue` returns the value of the attribute as specified by the
+  * reference.conf file, coerced into a WdlValue.
   *
-  * `optional` can be used return the validated value as an `Option`, wrapped in a `Some`, if present, or `None` if not
-  * found.
+  * `default` a validation with the default value specified by the reference.conf file.
   */
 object ContinueOnReturnCodeValidation {
   lazy val instance: RuntimeAttributesValidation[ContinueOnReturnCode] = new ContinueOnReturnCodeValidation
-  lazy val default: RuntimeAttributesValidation[ContinueOnReturnCode] = instance.withDefault(WdlInteger(0))
-  lazy val optional: OptionalRuntimeAttributesValidation[ContinueOnReturnCode] = default.optional
+  def default(runtimeConfig: Option[Config]): RuntimeAttributesValidation[ContinueOnReturnCode] = instance.withDefault(
+    configDefaultWdlValue(runtimeConfig) getOrElse WdlInteger(0))
+  def configDefaultWdlValue(runtimeConfig: Option[Config]): Option[WdlValue] = instance.configDefaultWdlValue(runtimeConfig)
 }
 
 class ContinueOnReturnCodeValidation extends RuntimeAttributesValidation[ContinueOnReturnCode] {
diff --git a/backend/src/main/scala/cromwell/backend/validation/CpuValidation.scala b/backend/src/main/scala/cromwell/backend/validation/CpuValidation.scala
index 5a5d09730..60950ed00 100644
--- a/backend/src/main/scala/cromwell/backend/validation/CpuValidation.scala
+++ b/backend/src/main/scala/cromwell/backend/validation/CpuValidation.scala
@@ -1,24 +1,26 @@
 package cromwell.backend.validation
 
 import cats.syntax.validated._
+import com.typesafe.config.Config
 import lenthall.validation.ErrorOr.ErrorOr
-import wdl4s.types.WdlIntegerType
-import wdl4s.values.{WdlInteger, WdlValue}
+import wdl4s.wdl.types.WdlIntegerType
+import wdl4s.wdl.values.{WdlInteger, WdlValue}
 
 /**
   * Validates the "cpu" runtime attribute an Integer greater than 0, returning the value as an `Int`.
   *
   * `instance` returns an validation that errors when no attribute is specified.
   *
-  * The default returns `1` when no attribute is specified.
+  * `default` a hardcoded default WdlValue for Cpu.
   *
-  * `optional` can be used return the validated value as an `Option`, wrapped in a `Some`, if present, or `None` if not
-  * found.
+  * `configDefaultWdlValue` returns the value of the attribute as specified by the
+  * reference.conf file, coerced into a WdlValue.
   */
-object CpuValidation extends {
+object CpuValidation {
   lazy val instance: RuntimeAttributesValidation[Int] = new CpuValidation
-  lazy val default: RuntimeAttributesValidation[Int] = instance.withDefault(WdlInteger(1))
-  lazy val optional: OptionalRuntimeAttributesValidation[Int] = default.optional
+  lazy val optional: OptionalRuntimeAttributesValidation[Int] = instance.optional
+  lazy val default: WdlValue = WdlInteger(1)
+  def configDefaultWdlValue(config: Option[Config]): Option[WdlValue] = instance.configDefaultWdlValue(config)
 }
 
 class CpuValidation extends IntRuntimeAttributesValidation(RuntimeAttributesKeys.CpuKey) {
diff --git a/backend/src/main/scala/cromwell/backend/validation/DockerValidation.scala b/backend/src/main/scala/cromwell/backend/validation/DockerValidation.scala
index f793d1397..788f7a7e6 100644
--- a/backend/src/main/scala/cromwell/backend/validation/DockerValidation.scala
+++ b/backend/src/main/scala/cromwell/backend/validation/DockerValidation.scala
@@ -2,7 +2,7 @@ package cromwell.backend.validation
 
 import cats.syntax.validated._
 import lenthall.validation.ErrorOr.ErrorOr
-import wdl4s.values.{WdlString, WdlValue}
+import wdl4s.wdl.values.{WdlString, WdlValue}
 
 /**
   * Validates the "docker" runtime attribute as a String, returning it as `String`.
diff --git a/backend/src/main/scala/cromwell/backend/validation/FailOnStderrValidation.scala b/backend/src/main/scala/cromwell/backend/validation/FailOnStderrValidation.scala
index 6a7ea6ee2..bf4f024c6 100644
--- a/backend/src/main/scala/cromwell/backend/validation/FailOnStderrValidation.scala
+++ b/backend/src/main/scala/cromwell/backend/validation/FailOnStderrValidation.scala
@@ -1,6 +1,7 @@
 package cromwell.backend.validation
 
-import wdl4s.values.WdlBoolean
+import com.typesafe.config.Config
+import wdl4s.wdl.values.{WdlBoolean, WdlValue}
 
 /**
   * Validates the "failOnStderr" runtime attribute as a Boolean or a String 'true' or 'false', returning the value as a
@@ -8,12 +9,17 @@ import wdl4s.values.WdlBoolean
   *
   * `instance` returns an validation that errors when no attribute is specified.
   *
-  * The default returns `false` when no attribute is specified.
+  * `configDefaultWdlValue` returns the value of the attribute as specified by the
+  * reference.conf file, coerced into a WdlValue.
+  *
+  * `default` a validation with the default value specified by the reference.conf file.
   */
+
 object FailOnStderrValidation {
   lazy val instance: RuntimeAttributesValidation[Boolean] = new FailOnStderrValidation
-  lazy val default: RuntimeAttributesValidation[Boolean] = instance.withDefault(WdlBoolean(false))
-  lazy val optional: OptionalRuntimeAttributesValidation[Boolean] = default.optional
+  def default(runtimeConfig: Option[Config]): RuntimeAttributesValidation[Boolean] = instance.withDefault(
+    configDefaultWdlValue(runtimeConfig) getOrElse WdlBoolean(false))
+  def configDefaultWdlValue(runtimeConfig: Option[Config]): Option[WdlValue] = instance.configDefaultWdlValue(runtimeConfig)
 }
 
 class FailOnStderrValidation extends BooleanRuntimeAttributesValidation(RuntimeAttributesKeys.FailOnStderrKey) {
diff --git a/backend/src/main/scala/cromwell/backend/validation/MemoryValidation.scala b/backend/src/main/scala/cromwell/backend/validation/MemoryValidation.scala
index d493e357e..c2c97343a 100644
--- a/backend/src/main/scala/cromwell/backend/validation/MemoryValidation.scala
+++ b/backend/src/main/scala/cromwell/backend/validation/MemoryValidation.scala
@@ -1,11 +1,14 @@
 package cromwell.backend.validation
 
 import cats.syntax.validated._
+import com.typesafe.config.Config
 import cromwell.backend.MemorySize
 import lenthall.validation.ErrorOr._
 import wdl4s.parser.MemoryUnit
-import wdl4s.types.{WdlIntegerType, WdlStringType}
-import wdl4s.values.{WdlInteger, WdlString, WdlValue}
+import wdl4s.wdl.types.{WdlIntegerType, WdlStringType}
+import wdl4s.wdl.values.{WdlInteger, WdlString, WdlValue}
+
+import scala.util.{Failure, Success}
 
 /**
   * Validates the "memory" runtime attribute as an Integer or String with format '8 GB', returning the value as a
@@ -13,61 +16,71 @@ import wdl4s.values.{WdlInteger, WdlString, WdlValue}
   *
   * `instance` returns an validation that errors when no attribute is specified.
   *
-  * There is no default, however `optional` can be used to validate the attribute and return the validated value as an
-  * `Option`, wrapped in an `Some`, if present, or `None` if not found.
+  * `configDefaultWdlValue` returns the value of the attribute as specified by the
+  * reference.conf file, coerced into a WdlValue.
+  *
+  * `optional` can be used to return the validated value as an `Option`,
+  * wrapped in a `Some`, if present, or `None` if not found.
   *
   * `withDefaultMemory` can be used to create a memory validation that defaults to a particular memory size.
   */
 object MemoryValidation {
-  lazy val instance: RuntimeAttributesValidation[MemorySize] = new MemoryValidation
-  lazy val optional: OptionalRuntimeAttributesValidation[MemorySize] = instance.optional
-
-  def withDefaultMemory(memorySize: MemorySize): RuntimeAttributesValidation[MemorySize] =
-    instance.withDefault(WdlInteger(memorySize.bytes.toInt))
+  def instance(attributeName: String = RuntimeAttributesKeys.MemoryKey): RuntimeAttributesValidation[MemorySize] =
+    new MemoryValidation(attributeName)
+  def optional(attributeName: String = RuntimeAttributesKeys.MemoryKey): OptionalRuntimeAttributesValidation[MemorySize] =
+    instance(attributeName).optional
+  def configDefaultString(attributeName: String = RuntimeAttributesKeys.MemoryKey, config: Option[Config]): Option[String] =
+    instance(attributeName).configDefaultValue(config)
+  def withDefaultMemory(attributeName: String = RuntimeAttributesKeys.MemoryKey, memorySize: String): RuntimeAttributesValidation[MemorySize] = {
+    MemorySize.parse(memorySize) match {
+      case Success(memory) => instance(attributeName).withDefault(WdlInteger(memory.bytes.toInt))
+      case Failure(_) => instance(attributeName).withDefault(BadDefaultAttribute(WdlString(memorySize.toString)))
+    }
+  }
 
   private[validation] val wrongAmountFormat =
-    s"Expecting ${RuntimeAttributesKeys.MemoryKey} runtime attribute value greater than 0 but got %s"
+    "Expecting %s runtime attribute value greater than 0 but got %s"
   private[validation] val wrongTypeFormat =
-    s"Expecting ${RuntimeAttributesKeys.MemoryKey} runtime attribute to be an Integer or String with format '8 GB'." +
-      s" Exception: %s"
+    "Expecting %s runtime attribute to be an Integer or String with format '8 GB'." +
+      " Exception: %s"
 
-  private[validation] def validateMemoryString(wdlString: WdlString): ErrorOr[MemorySize] =
-    validateMemoryString(wdlString.value)
+  private[validation] def validateMemoryString(attributeName: String, wdlString: WdlString): ErrorOr[MemorySize] =
+    validateMemoryString(attributeName, wdlString.value)
 
-  private[validation] def validateMemoryString(value: String): ErrorOr[MemorySize] = {
+  private[validation] def validateMemoryString(attributeName: String, value: String): ErrorOr[MemorySize] = {
     MemorySize.parse(value) match {
       case scala.util.Success(memorySize: MemorySize) if memorySize.amount > 0 =>
         memorySize.to(MemoryUnit.GB).validNel
       case scala.util.Success(memorySize: MemorySize) =>
-        wrongAmountFormat.format(memorySize.amount).invalidNel
+        wrongAmountFormat.format(attributeName, memorySize.amount).invalidNel
       case scala.util.Failure(throwable) =>
-        wrongTypeFormat.format(throwable.getMessage).invalidNel
+        wrongTypeFormat.format(attributeName, throwable.getMessage).invalidNel
     }
   }
 
-  private[validation] def validateMemoryInteger(wdlInteger: WdlInteger): ErrorOr[MemorySize] =
-    validateMemoryInteger(wdlInteger.value)
+  private[validation] def validateMemoryInteger(attributeName: String, wdlInteger: WdlInteger): ErrorOr[MemorySize] =
+    validateMemoryInteger(attributeName, wdlInteger.value)
 
-  private[validation] def validateMemoryInteger(value: Int): ErrorOr[MemorySize] = {
+  private[validation] def validateMemoryInteger(attributeName: String, value: Int): ErrorOr[MemorySize] = {
     if (value <= 0)
-      wrongAmountFormat.format(value).invalidNel
+      wrongAmountFormat.format(attributeName, value).invalidNel
     else
       MemorySize(value.toDouble, MemoryUnit.Bytes).to(MemoryUnit.GB).validNel
   }
 }
 
-class MemoryValidation extends RuntimeAttributesValidation[MemorySize] {
+class MemoryValidation(attributeName: String = RuntimeAttributesKeys.MemoryKey) extends RuntimeAttributesValidation[MemorySize] {
 
   import MemoryValidation._
 
-  override def key = RuntimeAttributesKeys.MemoryKey
+  override def key = attributeName
 
   override def coercion = Seq(WdlIntegerType, WdlStringType)
 
   override protected def validateValue: PartialFunction[WdlValue, ErrorOr[MemorySize]] = {
-    case WdlInteger(value) => MemoryValidation.validateMemoryInteger(value)
-    case WdlString(value) => MemoryValidation.validateMemoryString(value)
+    case WdlInteger(value) => MemoryValidation.validateMemoryInteger(key, value)
+    case WdlString(value) => MemoryValidation.validateMemoryString(key, value)
   }
 
-  override def missingValueMessage: String = wrongTypeFormat.format("Not supported WDL type value")
+  override def missingValueMessage: String = wrongTypeFormat.format(key, "Not supported WDL type value")
 }
diff --git a/backend/src/main/scala/cromwell/backend/validation/PrimitiveRuntimeAttributesValidation.scala b/backend/src/main/scala/cromwell/backend/validation/PrimitiveRuntimeAttributesValidation.scala
index aa07afcdc..477f9f9c7 100644
--- a/backend/src/main/scala/cromwell/backend/validation/PrimitiveRuntimeAttributesValidation.scala
+++ b/backend/src/main/scala/cromwell/backend/validation/PrimitiveRuntimeAttributesValidation.scala
@@ -2,8 +2,8 @@ package cromwell.backend.validation
 
 import cats.syntax.validated._
 import lenthall.validation.ErrorOr.ErrorOr
-import wdl4s.types._
-import wdl4s.values.{WdlBoolean, WdlFloat, WdlInteger, WdlPrimitive, WdlString, WdlValue}
+import wdl4s.wdl.types._
+import wdl4s.wdl.values._
 
 /**
   * Validates one of the wdl primitive types: Boolean, Float, Integer, or String. WdlFile is not supported.
diff --git a/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesDefault.scala b/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesDefault.scala
index 30ffe1043..a111fe554 100644
--- a/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesDefault.scala
+++ b/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesDefault.scala
@@ -4,8 +4,8 @@ import cats.data.ValidatedNel
 import cats.syntax.validated._
 import cromwell.core.{EvaluatedRuntimeAttributes, OptionNotFoundException, WorkflowOptions}
 import lenthall.util.TryUtil
-import wdl4s.types.WdlType
-import wdl4s.values.WdlValue
+import wdl4s.wdl.types.WdlType
+import wdl4s.wdl.values.WdlValue
 
 import scala.util.{Failure, Try}
 
diff --git a/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala b/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala
index f8ff1441a..ffba5bbd8 100644
--- a/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala
+++ b/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala
@@ -2,13 +2,14 @@ package cromwell.backend.validation
 
 import cats.data.{NonEmptyList, Validated}
 import cats.syntax.validated._
+import com.typesafe.config.Config
 import cromwell.backend.{MemorySize, RuntimeAttributeDefinition}
 import lenthall.validation.ErrorOr._
 import org.slf4j.Logger
-import wdl4s.expression.PureStandardLibraryFunctions
-import wdl4s.types.{WdlBooleanType, WdlIntegerType, WdlType}
-import wdl4s.values._
-import wdl4s.{NoLookup, WdlExpression}
+import wdl4s.wdl.expression.PureStandardLibraryFunctions
+import wdl4s.wdl.types.{WdlBooleanType, WdlIntegerType, WdlStringType, WdlType}
+import wdl4s.wdl.values._
+import wdl4s.wdl.{NoLookup, WdlExpression}
 
 import scala.util.{Failure, Success}
 
@@ -20,24 +21,24 @@ object RuntimeAttributesValidation {
   }
 
   def validateDocker(docker: Option[WdlValue], onMissingKey: => ErrorOr[Option[String]]): ErrorOr[Option[String]] = {
-    validateWithValidation(docker, DockerValidation.optional, onMissingKey)
+    validateWithValidation(docker, DockerValidation.instance.optional, onMissingKey)
   }
 
   def validateFailOnStderr(value: Option[WdlValue], onMissingKey: => ErrorOr[Boolean]): ErrorOr[Boolean] = {
-    validateWithValidation(value, FailOnStderrValidation.default, onMissingKey)
+    validateWithValidation(value, FailOnStderrValidation.instance, onMissingKey)
   }
 
   def validateContinueOnReturnCode(value: Option[WdlValue],
                                    onMissingKey: => ErrorOr[ContinueOnReturnCode]): ErrorOr[ContinueOnReturnCode] = {
-    validateWithValidation(value, ContinueOnReturnCodeValidation.default, onMissingKey)
+    validateWithValidation(value, ContinueOnReturnCodeValidation.instance, onMissingKey)
   }
 
   def validateMemory(value: Option[WdlValue], onMissingKey: => ErrorOr[MemorySize]): ErrorOr[MemorySize] = {
-    validateWithValidation(value, MemoryValidation.instance, onMissingKey)
+    validateWithValidation(value, MemoryValidation.instance(), onMissingKey)
   }
 
   def validateCpu(cpu: Option[WdlValue], onMissingKey: => ErrorOr[Int]): ErrorOr[Int] = {
-    validateWithValidation(cpu, CpuValidation.default, onMissingKey)
+    validateWithValidation(cpu, CpuValidation.instance, onMissingKey)
   }
 
   private def validateWithValidation[T](valueOption: Option[WdlValue],
@@ -64,12 +65,12 @@ object RuntimeAttributesValidation {
     }
   }
 
-  def parseMemoryString(s: WdlString): ErrorOr[MemorySize] = {
-    MemoryValidation.validateMemoryString(s)
+  def parseMemoryString(k: String, s: WdlString): ErrorOr[MemorySize] = {
+    MemoryValidation.validateMemoryString(k, s)
   }
 
-  def parseMemoryInteger(i: WdlInteger): ErrorOr[MemorySize] = {
-    MemoryValidation.validateMemoryInteger(i)
+  def parseMemoryInteger(k: String, i: WdlInteger): ErrorOr[MemorySize] = {
+    MemoryValidation.validateMemoryInteger(k, i)
   }
 
   def withDefault[ValidatedType](validation: RuntimeAttributesValidation[ValidatedType],
@@ -213,6 +214,15 @@ object RuntimeAttributesValidation {
 }
 
 /**
+  * A wrapper class to classify config-based default runtime attributes
+  * that cannot be coerced into an acceptable WdlType.
+  */
+case class BadDefaultAttribute(badDefaultValue: WdlValue) extends WdlValue {
+  val wdlType = WdlStringType
+}
+
+
+/**
   * Performs a validation on a runtime attribute and returns some value.
   *
   * @tparam ValidatedType The type of the validated value.
@@ -371,6 +381,35 @@ trait RuntimeAttributesValidation[ValidatedType] {
   final def withDefault(wdlValue: WdlValue): RuntimeAttributesValidation[ValidatedType] =
     RuntimeAttributesValidation.withDefault(this, wdlValue)
 
+  /**
+    * Returns the value of the default runtime attribute of a
+    * validation key as specified in the reference.conf. Given
+    * a value, this method coerces it into an optional
+    * WdlValue. In case the value cannot be succesfully coerced
+    * the value is wrapped as a "BadDefaultAttributeValue" type that
+    * is failed downstream by the ValidatedRuntimeAttributesBuilder.
+    *
+    * @param optionalRuntimeConfig Optional default runtime attributes config of a particular backend.
+    * @return The new version of this validation.
+    */
+  final def configDefaultWdlValue(optionalRuntimeConfig: Option[Config]): Option[WdlValue] = {
+    optionalRuntimeConfig flatMap { config =>
+      val value = config.getValue(key).unwrapped()
+      coercion.collectFirst({
+        case wdlType if wdlType.coerceRawValue(value).isSuccess => {
+          wdlType.coerceRawValue(value).get
+        }
+      }) orElse Option(BadDefaultAttribute(WdlString(value.toString)))
+    }
+  }
+
+  final def configDefaultValue(optionalRuntimeConfig: Option[Config]): Option[String] = {
+    optionalRuntimeConfig match {
+      case Some(config) if config.hasPath(key) => Option(config.getValue(key).unwrapped().toString)
+      case _ => None
+    }
+  }
+
   /*
   Methods below provide aliases to expose protected methods to the package.
   Allows wrappers to wire their overrides to invoke the corresponding method on the inner object.
diff --git a/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala b/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala
index ed3981817..c125e24b8 100644
--- a/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala
+++ b/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala
@@ -6,8 +6,8 @@ import cromwell.backend.RuntimeAttributeDefinition
 import lenthall.exception.MessageAggregation
 import lenthall.validation.ErrorOr._
 import org.slf4j.Logger
-import wdl4s.types.WdlType
-import wdl4s.values.WdlValue
+import wdl4s.wdl.types.WdlType
+import wdl4s.wdl.values.WdlValue
 
 final case class ValidatedRuntimeAttributes(attributes: Map[String, Any])
 
diff --git a/backend/src/main/scala/cromwell/backend/wdl/Command.scala b/backend/src/main/scala/cromwell/backend/wdl/Command.scala
index b0e3c9294..5e0d1d424 100644
--- a/backend/src/main/scala/cromwell/backend/wdl/Command.scala
+++ b/backend/src/main/scala/cromwell/backend/wdl/Command.scala
@@ -1,9 +1,9 @@
 package cromwell.backend.wdl
 
 import cromwell.backend.BackendJobDescriptor
-import wdl4s.EvaluatedTaskInputs
-import wdl4s.expression.WdlFunctions
-import wdl4s.values.WdlValue
+import wdl4s.wdl.EvaluatedTaskInputs
+import wdl4s.wdl.expression.WdlFunctions
+import wdl4s.wdl.values.WdlValue
 
 import scala.util.{Success, Try}
 
diff --git a/backend/src/main/scala/cromwell/backend/wdl/FileSizeLimitationConfig.scala b/backend/src/main/scala/cromwell/backend/wdl/FileSizeLimitationConfig.scala
new file mode 100644
index 000000000..44f2835a3
--- /dev/null
+++ b/backend/src/main/scala/cromwell/backend/wdl/FileSizeLimitationConfig.scala
@@ -0,0 +1,48 @@
+package cromwell.backend.wdl
+
+import com.typesafe.config.ConfigFactory
+import net.ceedubs.ficus.readers.ValueReader
+import net.ceedubs.ficus.Ficus._
+
+trait FileSizeLimitationConfig {
+
+  def readLinesLimit: Int
+
+  def readBoolLimit: Int
+
+  def readIntLimit: Int
+
+  def readFloatLimit: Int
+
+  def readStringLimit: Int
+
+  def readJsonLimit: Int
+
+  def readTsvLimit: Int
+
+  def readMapLimit: Int
+
+  def readObjectLimit: Int
+}
+
+object FileSizeLimitationConfig {
+  private val config = ConfigFactory.load.getConfig("system")
+
+  def fileSizeLimitationConfig: FileSizeLimitationConfig = config.as[FileSizeLimitationConfig]("input-read-limits")
+
+  implicit val configReader : ValueReader[FileSizeLimitationConfig] = ValueReader.relative{c =>
+    def f(s: String) = c.as[Int](s)
+    new FileSizeLimitationConfig {
+      val readLinesLimit =  f("lines")
+      val readBoolLimit =   f("bool")
+      val readIntLimit =    f("int")
+      val readFloatLimit =  f("float")
+      val readStringLimit = f("string")
+      val readJsonLimit =   f("json")
+      val readTsvLimit =    f("tsv")
+      val readMapLimit =    f("map")
+      val readObjectLimit = f("object")
+    }
+  }
+}
+
diff --git a/backend/src/main/scala/cromwell/backend/wdl/FileSizeTooBig.scala b/backend/src/main/scala/cromwell/backend/wdl/FileSizeTooBig.scala
new file mode 100644
index 000000000..a0cf7477a
--- /dev/null
+++ b/backend/src/main/scala/cromwell/backend/wdl/FileSizeTooBig.scala
@@ -0,0 +1,4 @@
+package cromwell.backend.wdl
+
+case class FileSizeTooBig(override val getMessage: String) extends Exception
+
diff --git a/backend/src/main/scala/cromwell/backend/wdl/OutputEvaluator.scala b/backend/src/main/scala/cromwell/backend/wdl/OutputEvaluator.scala
index d41d25b19..93d3563b6 100644
--- a/backend/src/main/scala/cromwell/backend/wdl/OutputEvaluator.scala
+++ b/backend/src/main/scala/cromwell/backend/wdl/OutputEvaluator.scala
@@ -2,9 +2,9 @@ package cromwell.backend.wdl
 
 import cromwell.backend.BackendJobDescriptor
 import cromwell.core.JobOutput
-import wdl4s.LocallyQualifiedName
-import wdl4s.expression.WdlStandardLibraryFunctions
-import wdl4s.values.WdlValue
+import wdl4s.wdl.LocallyQualifiedName
+import wdl4s.wdl.expression.WdlStandardLibraryFunctions
+import wdl4s.wdl.values.WdlValue
 
 import scala.util.{Success, Try}
 
diff --git a/backend/src/main/scala/cromwell/backend/wdl/ReadLikeFunctions.scala b/backend/src/main/scala/cromwell/backend/wdl/ReadLikeFunctions.scala
index 5a45cef96..349cefbe6 100644
--- a/backend/src/main/scala/cromwell/backend/wdl/ReadLikeFunctions.scala
+++ b/backend/src/main/scala/cromwell/backend/wdl/ReadLikeFunctions.scala
@@ -2,15 +2,21 @@ package cromwell.backend.wdl
 
 import cromwell.backend.MemorySize
 import cromwell.core.path.PathFactory
-import wdl4s.expression.WdlStandardLibraryFunctions
+import wdl4s.wdl.expression.WdlStandardLibraryFunctions
 import wdl4s.parser.MemoryUnit
-import wdl4s.types.{WdlArrayType, WdlFileType, WdlObjectType, WdlStringType}
-import wdl4s.values._
+import wdl4s.wdl.types._
+import wdl4s.wdl.values._
 
 import scala.util.{Failure, Success, Try}
 
 trait ReadLikeFunctions extends PathFactory { this: WdlStandardLibraryFunctions =>
 
+  val fileSizeLimitationConfig =  FileSizeLimitationConfig.fileSizeLimitationConfig
+  import fileSizeLimitationConfig._
+
+  def fileSize: WdlValue=> Try[Long] = 
+    w => Try(buildPath(w.valueString).size)
+
   /**
     * Asserts that the parameter list contains a single parameter which will be interpreted
     * as a File and attempts to read the contents of that file
@@ -22,10 +28,12 @@ trait ReadLikeFunctions extends PathFactory { this: WdlStandardLibraryFunctions
     } yield string
   }
 
-  private def extractObjects(functionName: String, params: Seq[Try[WdlValue]]): Try[Array[WdlObject]] = for {
-    contents <- readContentsFromSingleFileParameter(functionName, params)
-    wdlObjects <- WdlObject.fromTsv(contents)
-  } yield wdlObjects
+  private def extractObjects(functionName: String, params: Seq[Try[WdlValue]]): Try[Array[WdlObject]] =
+    for {
+      _ <- validateFileSizeIsWithinLimits("read_object", params, readObjectLimit)
+      contents <- readContentsFromSingleFileParameter(functionName, params)
+      wdlObjects <- WdlObject.fromTsv(contents)
+    } yield wdlObjects
 
   override def readFile(path: String): String = buildPath(path).contentAsString
 
@@ -34,13 +42,25 @@ trait ReadLikeFunctions extends PathFactory { this: WdlStandardLibraryFunctions
     */
   override def read_lines(params: Seq[Try[WdlValue]]): Try[WdlArray] = {
     for {
+      _ <- validateFileSizeIsWithinLimits("read_lines", params, readLinesLimit)
       contents <- readContentsFromSingleFileParameter("read_lines", params)
       lines = contents.split("\n")
     } yield WdlArray(WdlArrayType(WdlStringType), lines map WdlString)
   }
 
+  def validateFileSizeIsWithinLimits(functionName: String, params: Seq[Try[WdlValue]], limit: Int): Try[Unit] = 
+    for {
+      fileName <- extractSingleArgument(functionName, params)
+      fileSize <- fileSize(fileName)
+      _ = if (fileSize > limit) {
+        val errorMsg = s"Use of $fileName failed because the file was too big ($fileSize bytes when only files of up to $limit bytes are permissible"
+        throw FileSizeTooBig(errorMsg)
+      }
+    } yield ()
+
   override def read_map(params: Seq[Try[WdlValue]]): Try[WdlMap] = {
     for {
+      _ <- validateFileSizeIsWithinLimits("read_map", params, readMapLimit)
       contents <- readContentsFromSingleFileParameter("read_map", params)
       wdlMap <- WdlMap.fromTsv(contents)
     } yield wdlMap
@@ -58,13 +78,18 @@ trait ReadLikeFunctions extends PathFactory { this: WdlStandardLibraryFunctions
   /**
     * Try to read a string from the file referenced by the specified `WdlValue`.
     */
-  override def read_string(params: Seq[Try[WdlValue]]): Try[WdlString] = readContentsFromSingleFileParameter("read_string", params).map(s => WdlString(s.trim))
+  override def read_string(params: Seq[Try[WdlValue]]): Try[WdlString] =
+    for {
+      _ <- validateFileSizeIsWithinLimits("read_string", params, readStringLimit)
+      string <- readContentsFromSingleFileParameter("read_string", params)
+    } yield WdlString(string.trim)
 
   /**
     * Read a file in TSV format into an Array[Array[String]]
     */
   override def read_tsv(params: Seq[Try[WdlValue]]): Try[WdlArray] = {
     for {
+      _ <- validateFileSizeIsWithinLimits("read_tsv", params, readTsvLimit)
       contents <- readContentsFromSingleFileParameter("read_tsv", params)
       wdlArray = WdlArray.fromTsv(contents)
     } yield wdlArray
@@ -73,12 +98,20 @@ trait ReadLikeFunctions extends PathFactory { this: WdlStandardLibraryFunctions
   /**
     * Try to read an integer from the file referenced by the specified `WdlValue`.
     */
-  override def read_int(params: Seq[Try[WdlValue]]): Try[WdlInteger] = read_string(params) map { s => WdlInteger(s.value.trim.toInt) }
+  override def read_int(params: Seq[Try[WdlValue]]): Try[WdlInteger] = 
+    for {
+      _ <- validateFileSizeIsWithinLimits("read_int", params, readIntLimit)
+      r <- read_string(params) map { s => WdlInteger(s.value.trim.toInt) }
+    } yield r
 
   /**
     * Try to read a float from the file referenced by the specified `WdlValue`.
     */
-  override def read_float(params: Seq[Try[WdlValue]]): Try[WdlFloat] = read_string(params) map { s => WdlFloat(s.value.trim.toDouble) }
+  override def read_float(params: Seq[Try[WdlValue]]): Try[WdlFloat] =
+    for {
+      _ <- validateFileSizeIsWithinLimits("read_float", params, readFloatLimit)
+      s <- read_string(params)
+    } yield WdlFloat(s.value.trim.toDouble)
 
   /**
     * Try to read a boolean from the file referenced by the specified `WdlValue`.
@@ -86,14 +119,39 @@ trait ReadLikeFunctions extends PathFactory { this: WdlStandardLibraryFunctions
   override def read_boolean(params: Seq[Try[WdlValue]]): Try[WdlBoolean] =
     read_string(params) map { s => WdlBoolean(java.lang.Boolean.parseBoolean(s.value.trim.toLowerCase)) }
 
+  protected def size(file: WdlValue): Try[Double] = Try(buildPath(file.valueString).size.toDouble)
+
+  /**
+    * Gets the size of a file.
+    *
+    * @param params First parameter must be a File or File? or coerceable to one. The second is an optional string containing the size unit (eg "MB", "GiB")
+    */
   override def size(params: Seq[Try[WdlValue]]): Try[WdlFloat] = {
+    // Inner function: get the memory unit from the second (optional) parameter
     def toUnit(wdlValue: Try[WdlValue]) = wdlValue flatMap { unit => Try(MemoryUnit.fromSuffix(unit.valueString)) }
 
+    // Inner function: is this a file type, or an optional containing a file type?
+    def isOptionalOfFileType(wdlType: WdlType): Boolean = wdlType match {
+      case f if WdlFileType.isCoerceableFrom(f) => true
+      case WdlOptionalType(inner) => isOptionalOfFileType(inner)
+      case _ => false
+    }
+
+    // Inner function: Get the file size, allowing for unpacking of optionals
+    def optionalSafeFileSize(value: WdlValue): Try[Double] = value match {
+      case f if f.isInstanceOf[WdlFile] || WdlFileType.isCoerceableFrom(f.wdlType) => size(f)
+      case WdlOptionalValue(_, Some(o)) => optionalSafeFileSize(o)
+      case WdlOptionalValue(f, None) if isOptionalOfFileType(f) => Success(0d)
+      case _ => Failure(new Exception(s"The 'size' method expects a 'File' or 'File?' argument but instead got ${value.wdlType.toWdlString}."))
+    }
+
+    // Inner function: get the file size and convert into the requested memory unit
     def fileSize(wdlValue: Try[WdlValue], convertTo: Try[MemoryUnit] = Success(MemoryUnit.Bytes)) = {
       for {
         value <- wdlValue
         unit <- convertTo
-      } yield MemorySize(buildPath(value.valueString).size.toDouble, MemoryUnit.Bytes).to(unit).amount
+        fileSize <- optionalSafeFileSize(value)
+      } yield MemorySize(fileSize, MemoryUnit.Bytes).to(unit).amount
     }
 
     params match {
diff --git a/backend/src/main/scala/cromwell/backend/wdl/WdlFileMapper.scala b/backend/src/main/scala/cromwell/backend/wdl/WdlFileMapper.scala
index 1c39c4d89..746b71ecd 100644
--- a/backend/src/main/scala/cromwell/backend/wdl/WdlFileMapper.scala
+++ b/backend/src/main/scala/cromwell/backend/wdl/WdlFileMapper.scala
@@ -1,7 +1,7 @@
 package cromwell.backend.wdl
 
 import lenthall.util.TryUtil
-import wdl4s.values.{WdlArray, WdlFile, WdlMap, WdlOptionalValue, WdlPair, WdlValue}
+import wdl4s.wdl.values._
 
 import scala.util.{Success, Try}
 
diff --git a/backend/src/main/scala/cromwell/backend/wdl/WriteFunctions.scala b/backend/src/main/scala/cromwell/backend/wdl/WriteFunctions.scala
index b8568f840..28ff91145 100644
--- a/backend/src/main/scala/cromwell/backend/wdl/WriteFunctions.scala
+++ b/backend/src/main/scala/cromwell/backend/wdl/WriteFunctions.scala
@@ -1,10 +1,10 @@
 package cromwell.backend.wdl
 
 import cromwell.core.path.Path
-import wdl4s.TsvSerializable
-import wdl4s.expression.WdlStandardLibraryFunctions
-import wdl4s.types._
-import wdl4s.values._
+import wdl4s.wdl.TsvSerializable
+import wdl4s.wdl.expression.WdlStandardLibraryFunctions
+import wdl4s.wdl.types._
+import wdl4s.wdl.values._
 
 import scala.util.{Failure, Try}
 
diff --git a/backend/src/test/scala/cromwell/backend/BackendSpec.scala b/backend/src/test/scala/cromwell/backend/BackendSpec.scala
index 549f79f86..ec7d92949 100644
--- a/backend/src/test/scala/cromwell/backend/BackendSpec.scala
+++ b/backend/src/test/scala/cromwell/backend/BackendSpec.scala
@@ -1,9 +1,8 @@
 package cromwell.backend
 
-import com.typesafe.config.ConfigFactory
 import cromwell.backend.BackendJobExecutionActor.{BackendJobExecutionResponse, JobFailedNonRetryableResponse, JobFailedRetryableResponse, JobSucceededResponse}
 import cromwell.backend.io.TestWorkflows._
-import cromwell.core.callcaching.CallCachingEligible
+import cromwell.core.callcaching.NoDocker
 import cromwell.core.labels.Labels
 import cromwell.core.{WorkflowId, WorkflowOptions}
 import lenthall.exception.AggregatedException
@@ -12,25 +11,25 @@ import org.scalatest.concurrent.ScalaFutures
 import org.scalatest.time.{Millis, Seconds, Span}
 import org.specs2.mock.Mockito
 import spray.json.{JsObject, JsValue}
-import wdl4s._
-import wdl4s.expression.NoFunctions
-import wdl4s.values.WdlValue
+import wdl4s.wdl._
+import wdl4s.wdl.expression.NoFunctions
+import wdl4s.wdl.values.WdlValue
 
 trait BackendSpec extends ScalaFutures with Matchers with Mockito {
 
-  implicit val defaultPatience = PatienceConfig(timeout = Span(5, Seconds), interval = Span(500, Millis))
+  implicit val defaultPatience = PatienceConfig(timeout = Span(10, Seconds), interval = Span(500, Millis))
 
   def testWorkflow(workflow: TestWorkflow, backend: BackendJobExecutionActor, inputs: Map[String, WdlValue] = Map.empty) = {
     executeJobAndAssertOutputs(backend, workflow.expectedResponse)
   }
 
-  def buildWorkflowDescriptor(wdl: WdlSource,
+  def buildWorkflowDescriptor(workflowSource: WorkflowSource,
                               inputs: Map[String, WdlValue] = Map.empty,
                               options: WorkflowOptions = WorkflowOptions(JsObject(Map.empty[String, JsValue])),
                               runtime: String = "") = {
     BackendWorkflowDescriptor(
       WorkflowId.randomId(),
-      WdlNamespaceWithWorkflow.load(wdl.replaceAll("RUNTIME", runtime), Seq.empty[ImportResolver]).get.workflow, // Get ok, this is a test!
+      WdlNamespaceWithWorkflow.load(workflowSource.replaceAll("RUNTIME", runtime), Seq.empty[ImportResolver]).get.workflow, // Get ok, this is a test!
       inputs,
       options,
       Labels.empty
@@ -56,10 +55,10 @@ trait BackendSpec extends ScalaFutures with Matchers with Mockito {
     val inputDeclarations = call.evaluateTaskInputs(inputs, NoFunctions).get // .get is ok because this is a test
     val evaluatedAttributes = RuntimeAttributeDefinition.evaluateRuntimeAttributes(call.task.runtimeAttributes, NoFunctions, inputDeclarations).get // .get is OK here because this is a test
     val runtimeAttributes = RuntimeAttributeDefinition.addDefaultsToAttributes(runtimeAttributeDefinitions, options)(evaluatedAttributes)
-    BackendJobDescriptor(workflowDescriptor, jobKey, runtimeAttributes, inputDeclarations, CallCachingEligible)
+    BackendJobDescriptor(workflowDescriptor, jobKey, runtimeAttributes, inputDeclarations, NoDocker, Map.empty)
   }
 
-  def jobDescriptorFromSingleCallWorkflow(wdl: WdlSource,
+  def jobDescriptorFromSingleCallWorkflow(wdl: WorkflowSource,
                                           options: WorkflowOptions,
                                           runtimeAttributeDefinitions: Set[RuntimeAttributeDefinition]): BackendJobDescriptor = {
     val workflowDescriptor = buildWorkflowDescriptor(wdl)
@@ -68,10 +67,10 @@ trait BackendSpec extends ScalaFutures with Matchers with Mockito {
     val inputDeclarations = fqnMapToDeclarationMap(workflowDescriptor.knownValues)
     val evaluatedAttributes = RuntimeAttributeDefinition.evaluateRuntimeAttributes(call.task.runtimeAttributes, NoFunctions, inputDeclarations).get // .get is OK here because this is a test
     val runtimeAttributes = RuntimeAttributeDefinition.addDefaultsToAttributes(runtimeAttributeDefinitions, options)(evaluatedAttributes)
-    BackendJobDescriptor(workflowDescriptor, jobKey, runtimeAttributes, inputDeclarations, CallCachingEligible)
+    BackendJobDescriptor(workflowDescriptor, jobKey, runtimeAttributes, inputDeclarations, NoDocker, Map.empty)
   }
 
-  def jobDescriptorFromSingleCallWorkflow(wdl: WdlSource,
+  def jobDescriptorFromSingleCallWorkflow(wdl: WorkflowSource,
                                           runtime: String,
                                           attempt: Int,
                                           options: WorkflowOptions,
@@ -82,12 +81,12 @@ trait BackendSpec extends ScalaFutures with Matchers with Mockito {
     val inputDeclarations = fqnMapToDeclarationMap(workflowDescriptor.knownValues)
     val evaluatedAttributes = RuntimeAttributeDefinition.evaluateRuntimeAttributes(call.task.runtimeAttributes, NoFunctions, inputDeclarations).get // .get is OK here because this is a test
     val runtimeAttributes = RuntimeAttributeDefinition.addDefaultsToAttributes(runtimeAttributeDefinitions, options)(evaluatedAttributes)
-    BackendJobDescriptor(workflowDescriptor, jobKey, runtimeAttributes, inputDeclarations, CallCachingEligible)
+    BackendJobDescriptor(workflowDescriptor, jobKey, runtimeAttributes, inputDeclarations, NoDocker, Map.empty)
   }
 
   def assertResponse(executionResponse: BackendJobExecutionResponse, expectedResponse: BackendJobExecutionResponse) = {
     (executionResponse, expectedResponse) match {
-      case (JobSucceededResponse(_, _, responseOutputs, _, _), JobSucceededResponse(_, _, expectedOutputs, _, _)) =>
+      case (JobSucceededResponse(_, _, responseOutputs, _, _, _), JobSucceededResponse(_, _, expectedOutputs, _, _, _)) =>
         responseOutputs.size shouldBe expectedOutputs.size
         responseOutputs foreach {
           case (fqn, out) =>
@@ -111,16 +110,12 @@ trait BackendSpec extends ScalaFutures with Matchers with Mockito {
     case other: Throwable => other.getMessage + concatenateCauseMessages(t.getCause)
   }
 
-
   def executeJobAndAssertOutputs(backend: BackendJobExecutionActor, expectedResponse: BackendJobExecutionResponse) = {
     whenReady(backend.execute) { executionResponse =>
       assertResponse(executionResponse, expectedResponse)
     }
   }
 
-  lazy val emptyBackendConfig = BackendConfigurationDescriptor(
-    ConfigFactory.parseString("{}"), ConfigFactory.load())
-
   def firstJobDescriptorKey(workflowDescriptor: BackendWorkflowDescriptor): BackendJobDescriptorKey = {
     val call = workflowDescriptor.workflow.taskCalls.head
     BackendJobDescriptorKey(call, None, 1)
@@ -128,7 +123,7 @@ trait BackendSpec extends ScalaFutures with Matchers with Mockito {
 
   def firstJobDescriptor(workflowDescriptor: BackendWorkflowDescriptor,
                          inputs: Map[String, WdlValue] = Map.empty) = {
-    BackendJobDescriptor(workflowDescriptor, firstJobDescriptorKey(workflowDescriptor), Map.empty, fqnMapToDeclarationMap(inputs), CallCachingEligible)
+    BackendJobDescriptor(workflowDescriptor, firstJobDescriptorKey(workflowDescriptor), Map.empty, fqnMapToDeclarationMap(inputs), NoDocker, Map.empty)
   }
 }
 
diff --git a/backend/src/test/scala/cromwell/backend/BackendWorkflowInitializationActorSpec.scala b/backend/src/test/scala/cromwell/backend/BackendWorkflowInitializationActorSpec.scala
index 82e62d641..7438282ba 100644
--- a/backend/src/test/scala/cromwell/backend/BackendWorkflowInitializationActorSpec.scala
+++ b/backend/src/test/scala/cromwell/backend/BackendWorkflowInitializationActorSpec.scala
@@ -2,13 +2,14 @@ package cromwell.backend
 
 import akka.actor.ActorRef
 import akka.testkit.TestActorRef
+import com.typesafe.config.ConfigFactory
 import cromwell.backend.validation.{ContinueOnReturnCodeFlag, ContinueOnReturnCodeSet, ContinueOnReturnCodeValidation, RuntimeAttributesKeys}
 import cromwell.core.{TestKitSuite, WorkflowOptions}
 import org.scalatest.prop.TableDrivenPropertyChecks
 import org.scalatest.{FlatSpecLike, Matchers}
-import wdl4s.types._
-import wdl4s.values.{WdlArray, WdlBoolean, WdlFloat, WdlInteger, WdlString, WdlValue}
-import wdl4s.{TaskCall, WdlExpression}
+import wdl4s.wdl.types._
+import wdl4s.wdl.values.{WdlArray, WdlBoolean, WdlFloat, WdlInteger, WdlString, WdlValue}
+import wdl4s.wdl.{WdlTaskCall, WdlExpression}
 
 import scala.concurrent.Future
 import scala.util.Try
@@ -30,9 +31,11 @@ class BackendWorkflowInitializationActorSpec extends TestKitSuite("BackendWorkfl
     testPredicateBackendWorkflowInitializationActor.continueOnReturnCodePredicate(valueRequired = false)
   }
 
+  val optionalConfig = Option(TestConfig.optionalRuntimeConfig)
+
   it should "continueOnReturnCodePredicate" in {
     testContinueOnReturnCode(None) should be(true)
-    ContinueOnReturnCodeValidation.default.validateOptionalExpression(None) should be(true)
+    ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalExpression(None) should be(true)
 
     val booleanRows = Table(
       "value",
@@ -66,9 +69,9 @@ class BackendWorkflowInitializationActorSpec extends TestKitSuite("BackendWorkfl
       val wdlValue = WdlBoolean(value)
       val result = true
       testContinueOnReturnCode(Option(wdlValue)) should be(result)
-      ContinueOnReturnCodeValidation.default.validateOptionalExpression(Option(wdlValue)) should be(result)
+      ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalExpression(Option(wdlValue)) should be(result)
       val valid =
-        ContinueOnReturnCodeValidation.default.validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> wdlValue))
+        ContinueOnReturnCodeValidation.default(optionalConfig).validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> wdlValue))
       valid.isValid should be(result)
       valid.toEither.right.get should be(ContinueOnReturnCodeFlag(value))
     }
@@ -77,9 +80,9 @@ class BackendWorkflowInitializationActorSpec extends TestKitSuite("BackendWorkfl
       val wdlValue = WdlString(value.toString)
       val result = true
       testContinueOnReturnCode(Option(wdlValue)) should be(result)
-      ContinueOnReturnCodeValidation.default.validateOptionalExpression(Option(wdlValue)) should be(result)
+      ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalExpression(Option(wdlValue)) should be(result)
       val valid =
-        ContinueOnReturnCodeValidation.default.validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> wdlValue))
+        ContinueOnReturnCodeValidation.default(optionalConfig).validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> wdlValue))
       valid.isValid should be(result)
       valid.toEither.right.get should be(ContinueOnReturnCodeFlag(value))
     }
@@ -88,7 +91,7 @@ class BackendWorkflowInitializationActorSpec extends TestKitSuite("BackendWorkfl
       val wdlValue = WdlExpression.fromString(value.toString)
       val result = true
       testContinueOnReturnCode(Option(wdlValue)) should be(result)
-      ContinueOnReturnCodeValidation.default.validateOptionalExpression(Option(wdlValue)) should be(result)
+      ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalExpression(Option(wdlValue)) should be(result)
       // NOTE: expressions are never valid to validate
     }
 
@@ -96,9 +99,9 @@ class BackendWorkflowInitializationActorSpec extends TestKitSuite("BackendWorkfl
       val wdlValue = WdlInteger(value)
       val result = true
       testContinueOnReturnCode(Option(wdlValue)) should be(result)
-      ContinueOnReturnCodeValidation.default.validateOptionalExpression(Option(wdlValue)) should be(result)
+      ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalExpression(Option(wdlValue)) should be(result)
       val valid =
-        ContinueOnReturnCodeValidation.default.validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> wdlValue))
+        ContinueOnReturnCodeValidation.default(optionalConfig).validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> wdlValue))
       valid.isValid should be(result)
       valid.toEither.right.get should be(ContinueOnReturnCodeSet(Set(value)))
     }
@@ -107,9 +110,9 @@ class BackendWorkflowInitializationActorSpec extends TestKitSuite("BackendWorkfl
       val wdlValue = WdlString(value.toString)
       val result = true
       testContinueOnReturnCode(Option(wdlValue)) should be(result)
-      ContinueOnReturnCodeValidation.default.validateOptionalExpression(Option(wdlValue)) should be(result)
+      ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalExpression(Option(wdlValue)) should be(result)
       val valid =
-        ContinueOnReturnCodeValidation.default.validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> wdlValue))
+        ContinueOnReturnCodeValidation.default(optionalConfig).validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> wdlValue))
       valid.isValid should be(result)
       valid.toEither.right.get should be(ContinueOnReturnCodeSet(Set(value)))
     }
@@ -118,7 +121,7 @@ class BackendWorkflowInitializationActorSpec extends TestKitSuite("BackendWorkfl
       val wdlValue = WdlExpression.fromString(value.toString)
       val result = true
       testContinueOnReturnCode(Option(wdlValue)) should be(result)
-      ContinueOnReturnCodeValidation.default.validateOptionalExpression(Option(wdlValue)) should be(result)
+      ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalExpression(Option(wdlValue)) should be(result)
       // NOTE: expressions are never valid to validate
     }
 
@@ -126,9 +129,9 @@ class BackendWorkflowInitializationActorSpec extends TestKitSuite("BackendWorkfl
       val wdlValue = WdlArray(WdlArrayType(WdlIntegerType), Seq(WdlInteger(value)))
       val result = true
       testContinueOnReturnCode(Option(wdlValue)) should be(result)
-      ContinueOnReturnCodeValidation.default.validateOptionalExpression(Option(wdlValue)) should be(result)
+      ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalExpression(Option(wdlValue)) should be(result)
       val valid =
-        ContinueOnReturnCodeValidation.default.validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> wdlValue))
+        ContinueOnReturnCodeValidation.default(optionalConfig).validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> wdlValue))
       valid.isValid should be(result)
       valid.toEither.right.get should be(ContinueOnReturnCodeSet(Set(value)))
     }
@@ -137,9 +140,9 @@ class BackendWorkflowInitializationActorSpec extends TestKitSuite("BackendWorkfl
       val wdlValue = WdlArray(WdlArrayType(WdlStringType), Seq(WdlString(value.toString)))
       val result = true
       testContinueOnReturnCode(Option(wdlValue)) should be(result)
-      ContinueOnReturnCodeValidation.default.validateOptionalExpression(Option(wdlValue)) should be(result)
+      ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalExpression(Option(wdlValue)) should be(result)
       val valid =
-        ContinueOnReturnCodeValidation.default.validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> wdlValue))
+        ContinueOnReturnCodeValidation.default(optionalConfig).validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> wdlValue))
       valid.isValid should be(result)
       valid.toEither.right.get should be(ContinueOnReturnCodeSet(Set(value)))
     }
@@ -148,7 +151,7 @@ class BackendWorkflowInitializationActorSpec extends TestKitSuite("BackendWorkfl
       val wdlValue = WdlArray(WdlArrayType(WdlExpressionType), Seq(WdlExpression.fromString(value.toString)))
       val result = false
       testContinueOnReturnCode(Option(wdlValue)) should be(result)
-      ContinueOnReturnCodeValidation.default.validateOptionalExpression(Option(wdlValue)) should be(result)
+      ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalExpression(Option(wdlValue)) should be(result)
       // NOTE: expressions are never valid to validate
     }
 
@@ -156,16 +159,16 @@ class BackendWorkflowInitializationActorSpec extends TestKitSuite("BackendWorkfl
       val wdlValue = WdlExpression.fromString(expression)
       val result = true
       testContinueOnReturnCode(Option(wdlValue)) should be(result)
-      ContinueOnReturnCodeValidation.default.validateOptionalExpression(Option(wdlValue)) should be(result)
+      ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalExpression(Option(wdlValue)) should be(result)
       // NOTE: expressions are never valid to validate
     }
 
     forAll(invalidWdlValueRows) { wdlValue =>
       val result = false
       testContinueOnReturnCode(Option(wdlValue)) should be(result)
-      ContinueOnReturnCodeValidation.default.validateOptionalExpression(Option(wdlValue)) should be(result)
+      ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalExpression(Option(wdlValue)) should be(result)
       val valid =
-        ContinueOnReturnCodeValidation.default.validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> wdlValue))
+        ContinueOnReturnCodeValidation.default(optionalConfig).validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> wdlValue))
       valid.isValid should be(result)
       valid.toEither.left.get.toList should contain theSameElementsAs List(
         "Expecting continueOnReturnCode runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]"
@@ -177,9 +180,10 @@ class BackendWorkflowInitializationActorSpec extends TestKitSuite("BackendWorkfl
 }
 
 class TestPredicateBackendWorkflowInitializationActor extends BackendWorkflowInitializationActor {
+
   override val serviceRegistryActor: ActorRef = context.system.deadLetters
 
-  override def calls: Set[TaskCall] = throw new NotImplementedError("calls")
+  override def calls: Set[WdlTaskCall] = throw new NotImplementedError("calls")
 
   override protected def runtimeAttributeValidators: Map[String, (Option[WdlValue]) => Boolean] =
     throw new NotImplementedError("runtimeAttributeValidators")
@@ -194,8 +198,7 @@ class TestPredicateBackendWorkflowInitializationActor extends BackendWorkflowIni
   override protected def workflowDescriptor: BackendWorkflowDescriptor =
     throw new NotImplementedError("workflowDescriptor")
 
-  override protected def configurationDescriptor: BackendConfigurationDescriptor =
-    throw new NotImplementedError("configurationDescriptor")
+  override protected def configurationDescriptor: BackendConfigurationDescriptor = BackendConfigurationDescriptor(TestConfig.sampleBackendRuntimeConfig, ConfigFactory.empty())
 
   override def continueOnReturnCodePredicate(valueRequired: Boolean)
                                             (wdlExpressionMaybe: Option[WdlValue]): Boolean = {
diff --git a/backend/src/test/scala/cromwell/backend/TestConfig.scala b/backend/src/test/scala/cromwell/backend/TestConfig.scala
new file mode 100644
index 000000000..42050dbe3
--- /dev/null
+++ b/backend/src/test/scala/cromwell/backend/TestConfig.scala
@@ -0,0 +1,44 @@
+package cromwell.backend
+
+import com.typesafe.config.ConfigFactory
+
+object TestConfig {
+
+  lazy val sampleBackendRuntimeConfigString =
+    s"""
+       |default-runtime-attributes {
+       |    failOnStderr: false
+       |    continueOnReturnCode: 0
+       |    memory: "2 GB"
+       |}
+       |""".stripMargin
+
+  lazy val allBackendRuntimeAttrsString =
+    s"""
+       |default-runtime-attributes {
+       |  cpu: 1
+       |  failOnStderr: false
+       |  continueOnReturnCode: 0
+       |  memory: "1 GB"
+       |  bootDiskSizeGb: 10
+       |  disks: "local-disk 10 SSD"
+       |  noAddress: false
+       |  preemptible: 0
+       |  zones: ["us-central1-a", "us-central1-b"]
+       |}
+     """.stripMargin
+
+  lazy val sampleBackendRuntimeConfig = ConfigFactory.parseString(sampleBackendRuntimeConfigString)
+
+  lazy val allRuntimeAttrsConfig = ConfigFactory.parseString(allBackendRuntimeAttrsString).getConfig("default-runtime-attributes")
+
+  lazy val optionalRuntimeConfig = sampleBackendRuntimeConfig.getConfig("default-runtime-attributes")
+
+  lazy val globalConfig = ConfigFactory.load()
+
+  lazy val emptyConfig = ConfigFactory.empty()
+
+  lazy val emptyBackendConfigDescriptor = BackendConfigurationDescriptor(emptyConfig, globalConfig)
+
+  lazy val backendRuntimeConfigDescriptor = BackendConfigurationDescriptor(sampleBackendRuntimeConfig, emptyConfig)
+}
diff --git a/backend/src/test/scala/cromwell/backend/io/JobPathsSpec.scala b/backend/src/test/scala/cromwell/backend/io/JobPathsSpec.scala
index 0a8cb5936..eeccc39e1 100644
--- a/backend/src/test/scala/cromwell/backend/io/JobPathsSpec.scala
+++ b/backend/src/test/scala/cromwell/backend/io/JobPathsSpec.scala
@@ -1,10 +1,10 @@
 package cromwell.backend.io
 
 import com.typesafe.config.ConfigFactory
-import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptorKey, BackendSpec}
+import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptorKey, BackendSpec, TestConfig}
 import cromwell.core.path.DefaultPathBuilder
 import org.scalatest.{FlatSpec, Matchers}
-import wdl4s.TaskCall
+import wdl4s.wdl.WdlTaskCall
 
 class JobPathsSpec extends FlatSpec with Matchers with BackendSpec {
 
@@ -23,16 +23,16 @@ class JobPathsSpec extends FlatSpec with Matchers with BackendSpec {
       |        }
     """.stripMargin
 
-  val globalConfig = ConfigFactory.load()
   val backendConfig =  ConfigFactory.parseString(configString)
-  val defaultBackendConfigDescriptor = BackendConfigurationDescriptor(backendConfig, globalConfig)
+  val defaultBackendConfigDescriptor = BackendConfigurationDescriptor(backendConfig, TestConfig.globalConfig)
 
   "JobPaths" should "provide correct paths for a job" in {
 
     val wd = buildWorkflowDescriptor(TestWorkflows.HelloWorld)
-    val call: TaskCall = wd.workflow.taskCalls.head
+    val call: WdlTaskCall = wd.workflow.taskCalls.head
     val jobKey = BackendJobDescriptorKey(call, None, 1)
-    val jobPaths = new JobPathsWithDocker(jobKey, wd, backendConfig)
+    val workflowPaths = new WorkflowPathsWithDocker(wd, backendConfig)
+    val jobPaths = new JobPathsWithDocker(workflowPaths, jobKey)
     val id = wd.id
     jobPaths.callRoot.pathAsString shouldBe
       fullPath(s"local-cromwell-executions/wf_hello/$id/call-hello")
@@ -59,17 +59,17 @@ class JobPathsSpec extends FlatSpec with Matchers with BackendSpec {
       fullPath("/cromwell-executions/dock/path")
 
     val jobKeySharded = BackendJobDescriptorKey(call, Option(0), 1)
-    val jobPathsSharded = new JobPathsWithDocker(jobKeySharded, wd, backendConfig)
+    val jobPathsSharded = new JobPathsWithDocker(workflowPaths, jobKeySharded)
     jobPathsSharded.callExecutionRoot.pathAsString shouldBe
       fullPath(s"local-cromwell-executions/wf_hello/$id/call-hello/shard-0/execution")
 
     val jobKeyAttempt = BackendJobDescriptorKey(call, None, 2)
-    val jobPathsAttempt = new JobPathsWithDocker(jobKeyAttempt, wd, backendConfig)
+    val jobPathsAttempt = new JobPathsWithDocker(workflowPaths, jobKeyAttempt)
     jobPathsAttempt.callExecutionRoot.pathAsString shouldBe
       fullPath(s"local-cromwell-executions/wf_hello/$id/call-hello/attempt-2/execution")
 
     val jobKeyShardedAttempt = BackendJobDescriptorKey(call, Option(0), 2)
-    val jobPathsShardedAttempt = new JobPathsWithDocker(jobKeyShardedAttempt, wd, backendConfig)
+    val jobPathsShardedAttempt = new JobPathsWithDocker(workflowPaths, jobKeyShardedAttempt)
     jobPathsShardedAttempt.callExecutionRoot.pathAsString shouldBe
       fullPath(s"local-cromwell-executions/wf_hello/$id/call-hello/shard-0/attempt-2/execution")
   }
diff --git a/backend/src/test/scala/cromwell/backend/io/WorkflowPathsSpec.scala b/backend/src/test/scala/cromwell/backend/io/WorkflowPathsSpec.scala
index c16348da9..66b48467b 100644
--- a/backend/src/test/scala/cromwell/backend/io/WorkflowPathsSpec.scala
+++ b/backend/src/test/scala/cromwell/backend/io/WorkflowPathsSpec.scala
@@ -6,7 +6,7 @@ import cromwell.core.path.DefaultPathBuilder
 import cromwell.core.{JobKey, WorkflowId}
 import org.mockito.Mockito._
 import org.scalatest.{FlatSpec, Matchers}
-import wdl4s.{Call, Workflow}
+import wdl4s.wdl.{WdlCall, WdlWorkflow}
 
 class WorkflowPathsSpec extends FlatSpec with Matchers with BackendSpec {
 
@@ -29,22 +29,22 @@ class WorkflowPathsSpec extends FlatSpec with Matchers with BackendSpec {
     when(backendConfig.getString(any[String])).thenReturn("local-cromwell-executions") // This is the folder defined in the config as the execution root dir
     
     val rootWd = mock[BackendWorkflowDescriptor]
-    val rootWorkflow = mock[Workflow]
+    val rootWorkflow = mock[WdlWorkflow]
     val rootWorkflowId = WorkflowId.randomId()
     rootWorkflow.unqualifiedName returns "rootWorkflow"
     rootWd.workflow returns rootWorkflow
     rootWd.id returns rootWorkflowId
 
     val subWd = mock[BackendWorkflowDescriptor]
-    val subWorkflow = mock[Workflow]
+    val subWorkflow = mock[WdlWorkflow]
     val subWorkflowId = WorkflowId.randomId()
     subWorkflow.unqualifiedName returns "subWorkflow"
     subWd.workflow returns subWorkflow
     subWd.id returns subWorkflowId
     
-    val call1 = mock[Call]
+    val call1 = mock[WdlCall]
     call1.unqualifiedName returns "call1"
-    val call2 = mock[Call]
+    val call2 = mock[WdlCall]
     call2.unqualifiedName returns "call2"
     
     val jobKey = new JobKey {
diff --git a/backend/src/test/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala b/backend/src/test/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala
index b116aae72..7c78177b3 100644
--- a/backend/src/test/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala
+++ b/backend/src/test/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala
@@ -1,6 +1,6 @@
 package cromwell.backend.standard
 
-import cromwell.backend.RuntimeAttributeDefinition
+import cromwell.backend.{RuntimeAttributeDefinition, TestConfig}
 import cromwell.backend.validation.RuntimeAttributesKeys._
 import cromwell.backend.validation._
 import cromwell.core.WorkflowOptions
@@ -8,7 +8,7 @@ import org.scalatest.{Matchers, WordSpecLike}
 import org.slf4j.{Logger, LoggerFactory}
 import org.specs2.mock.Mockito
 import spray.json.{JsArray, JsBoolean, JsNumber, JsObject, JsValue}
-import wdl4s.values.{WdlBoolean, WdlInteger, WdlString, WdlValue}
+import wdl4s.wdl.values.{WdlBoolean, WdlInteger, WdlString, WdlValue}
 
 class StandardValidatedRuntimeAttributesBuilderSpec extends WordSpecLike with Matchers with Mockito {
 
@@ -153,6 +153,8 @@ class StandardValidatedRuntimeAttributesBuilderSpec extends WordSpecLike with Ma
   val defaultLogger: Logger = LoggerFactory.getLogger(classOf[StandardValidatedRuntimeAttributesBuilderSpec])
   val emptyWorkflowOptions: WorkflowOptions = WorkflowOptions.fromMap(Map.empty).get
 
+  val mockBackendRuntimeConfig = Option(TestConfig.optionalRuntimeConfig)
+
   private def assertRuntimeAttributesSuccessfulCreation(runtimeAttributes: Map[String, WdlValue],
                                                         expectedRuntimeAttributes: Map[String, Any],
                                                         includeDockerSupport: Boolean = true,
@@ -160,9 +162,9 @@ class StandardValidatedRuntimeAttributesBuilderSpec extends WordSpecLike with Ma
                                                         logger: Logger = defaultLogger): Unit = {
 
     val builder = if (includeDockerSupport) {
-      StandardValidatedRuntimeAttributesBuilder.default.withValidation(DockerValidation.optional)
+      StandardValidatedRuntimeAttributesBuilder.default(mockBackendRuntimeConfig).withValidation(DockerValidation.optional)
     } else {
-      StandardValidatedRuntimeAttributesBuilder.default
+      StandardValidatedRuntimeAttributesBuilder.default(mockBackendRuntimeConfig)
     }
     val runtimeAttributeDefinitions = builder.definitions.toSet
     val addDefaultsToAttributes = RuntimeAttributeDefinition.addDefaultsToAttributes(runtimeAttributeDefinitions, workflowOptions) _
@@ -189,9 +191,9 @@ class StandardValidatedRuntimeAttributesBuilderSpec extends WordSpecLike with Ma
                                                     logger: Logger = defaultLogger): Unit = {
     val thrown = the[RuntimeException] thrownBy {
       val builder = if (supportsDocker) {
-        StandardValidatedRuntimeAttributesBuilder.default.withValidation(DockerValidation.optional)
+        StandardValidatedRuntimeAttributesBuilder.default(mockBackendRuntimeConfig).withValidation(DockerValidation.optional)
       } else {
-        StandardValidatedRuntimeAttributesBuilder.default
+        StandardValidatedRuntimeAttributesBuilder.default(mockBackendRuntimeConfig)
       }
       val runtimeAttributeDefinitions = builder.definitions.toSet
       val addDefaultsToAttributes = RuntimeAttributeDefinition.addDefaultsToAttributes(runtimeAttributeDefinitions, workflowOptions) _
diff --git a/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesDefaultSpec.scala b/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesDefaultSpec.scala
index 7420c2d89..05edd089e 100644
--- a/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesDefaultSpec.scala
+++ b/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesDefaultSpec.scala
@@ -4,8 +4,8 @@ import cromwell.backend.validation.RuntimeAttributesDefault._
 import cromwell.core.WorkflowOptions
 import org.scalatest.{FlatSpec, Matchers}
 import spray.json._
-import wdl4s.types._
-import wdl4s.values.{WdlArray, WdlBoolean, WdlInteger, WdlString}
+import wdl4s.wdl.types._
+import wdl4s.wdl.values.{WdlArray, WdlBoolean, WdlInteger, WdlString}
 
 class RuntimeAttributesDefaultSpec extends FlatSpec with Matchers {
 
diff --git a/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala b/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala
index 22180bb77..76e47ce2a 100644
--- a/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala
+++ b/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala
@@ -2,12 +2,16 @@ package cromwell.backend.validation
 
 import cats.data.Validated.{Invalid, Valid}
 import cats.syntax.validated._
+import com.typesafe.config.{Config, ConfigFactory}
+import cromwell.backend.TestConfig
 import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpecLike}
-import wdl4s.types.{WdlArrayType, WdlIntegerType, WdlStringType}
-import wdl4s.values.{WdlArray, WdlBoolean, WdlInteger, WdlString}
+import wdl4s.wdl.types.{WdlArrayType, WdlIntegerType, WdlStringType}
+import wdl4s.wdl.values.{WdlArray, WdlBoolean, WdlInteger, WdlString}
 
 class RuntimeAttributesValidationSpec extends WordSpecLike with Matchers with BeforeAndAfterAll {
 
+  val mockBackendRuntimeConfig = TestConfig.allRuntimeAttrsConfig
+
   "RuntimeAttributesValidation" should {
     "return success when tries to validate a valid Docker entry" in {
       val dockerValue = Some(WdlString("someImage"))
@@ -33,7 +37,7 @@ class RuntimeAttributesValidationSpec extends WordSpecLike with Matchers with Be
       val result = RuntimeAttributesValidation.validateDocker(dockerValue,
         "Failed to get Docker mandatory key from runtime attributes".invalidNel)
       result match {
-        case Valid(x) => fail("A failure was expected.")
+        case Valid(_) => fail("A failure was expected.")
         case Invalid(e) => assert(e.head == "Failed to get Docker mandatory key from runtime attributes")
       }
     }
@@ -43,7 +47,7 @@ class RuntimeAttributesValidationSpec extends WordSpecLike with Matchers with Be
       val result = RuntimeAttributesValidation.validateDocker(dockerValue,
         "Failed to get Docker mandatory key from runtime attributes".invalidNel)
       result match {
-        case Valid(x) => fail("A failure was expected.")
+        case Valid(_) => fail("A failure was expected.")
         case Invalid(e) => assert(e.head == "Expecting docker runtime attribute to be a String")
       }
     }
@@ -83,7 +87,7 @@ class RuntimeAttributesValidationSpec extends WordSpecLike with Matchers with Be
       val result = RuntimeAttributesValidation.validateFailOnStderr(failOnStderrValue,
         "Failed to get failOnStderr mandatory key from runtime attributes".invalidNel)
       result match {
-        case Valid(x) => fail("A failure was expected.")
+        case Valid(_) => fail("A failure was expected.")
         case Invalid(e) => assert(e.head == "Expecting failOnStderr runtime attribute to be a Boolean or a String with values of 'true' or 'false'")
       }
     }
@@ -142,7 +146,7 @@ class RuntimeAttributesValidationSpec extends WordSpecLike with Matchers with Be
       val result = RuntimeAttributesValidation.validateContinueOnReturnCode(continueOnReturnCodeValue,
         "Failed to get continueOnReturnCode mandatory key from runtime attributes".invalidNel)
       result match {
-        case Valid(x) => fail("A failure was expected.")
+        case Valid(_) => fail("A failure was expected.")
         case Invalid(e) =>
           assert(e.head == "Expecting continueOnReturnCode runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]")
       }
@@ -163,7 +167,7 @@ class RuntimeAttributesValidationSpec extends WordSpecLike with Matchers with Be
       val result = RuntimeAttributesValidation.validateContinueOnReturnCode(continueOnReturnCodeValue,
         "Failed to get continueOnReturnCode mandatory key from runtime attributes".invalidNel)
       result match {
-        case Valid(x) => fail("A failure was expected.")
+        case Valid(_) => fail("A failure was expected.")
         case Invalid(e) => assert(e.head == "Expecting continueOnReturnCode runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]")
       }
     }
@@ -193,7 +197,7 @@ class RuntimeAttributesValidationSpec extends WordSpecLike with Matchers with Be
       val result = RuntimeAttributesValidation.validateMemory(memoryValue,
         "Failed to get memory mandatory key from runtime attributes".invalidNel)
       result match {
-        case Valid(x) => fail("A failure was expected.")
+        case Valid(_) => fail("A failure was expected.")
         case Invalid(e) => assert(e.head == "Expecting memory runtime attribute value greater than 0 but got -1")
       }
     }
@@ -214,7 +218,7 @@ class RuntimeAttributesValidationSpec extends WordSpecLike with Matchers with Be
       val result = RuntimeAttributesValidation.validateMemory(memoryValue,
         "Failed to get memory mandatory key from runtime attributes".invalidNel)
       result match {
-        case Valid(x) => fail("A failure was expected.")
+        case Valid(_) => fail("A failure was expected.")
         case Invalid(e) => assert(e.head == "Expecting memory runtime attribute value greater than 0 but got 0.0")
       }
     }
@@ -224,7 +228,7 @@ class RuntimeAttributesValidationSpec extends WordSpecLike with Matchers with Be
       val result = RuntimeAttributesValidation.validateMemory(memoryValue,
         "Failed to get memory mandatory key from runtime attributes".invalidNel)
       result match {
-        case Valid(x) => fail("A failure was expected.")
+        case Valid(_) => fail("A failure was expected.")
         case Invalid(e) => assert(e.head == "Expecting memory runtime attribute to be an Integer or String with format '8 GB'. Exception: value should be of the form 'X Unit' where X is a number, e.g. 8 GB")
       }
     }
@@ -234,7 +238,7 @@ class RuntimeAttributesValidationSpec extends WordSpecLike with Matchers with Be
       val result = RuntimeAttributesValidation.validateMemory(memoryValue,
         "Failed to get memory mandatory key from runtime attributes".invalidNel)
       result match {
-        case Valid(x) => fail("A failure was expected.")
+        case Valid(_) => fail("A failure was expected.")
         case Invalid(e) => assert(e.head == "Expecting memory runtime attribute to be an Integer or String with format '8 GB'. Exception: Not supported WDL type value")
       }
     }
@@ -244,7 +248,7 @@ class RuntimeAttributesValidationSpec extends WordSpecLike with Matchers with Be
       val result = RuntimeAttributesValidation.validateMemory(memoryValue,
         "Failed to get memory mandatory key from runtime attributes".invalidNel)
       result match {
-        case Valid(x) => fail("A failure was expected.")
+        case Valid(_) => fail("A failure was expected.")
         case Invalid(e) => assert(e.head == "Failed to get memory mandatory key from runtime attributes")
       }
     }
@@ -264,7 +268,7 @@ class RuntimeAttributesValidationSpec extends WordSpecLike with Matchers with Be
       val result = RuntimeAttributesValidation.validateCpu(cpuValue,
         "Failed to get cpu mandatory key from runtime attributes".invalidNel)
       result match {
-        case Valid(x) => fail("A failure was expected.")
+        case Valid(_) => fail("A failure was expected.")
         case Invalid(e) => assert(e.head == "Expecting cpu runtime attribute value greater than 0")
       }
     }
@@ -274,9 +278,87 @@ class RuntimeAttributesValidationSpec extends WordSpecLike with Matchers with Be
       val result = RuntimeAttributesValidation.validateMemory(cpuValue,
         "Failed to get cpu mandatory key from runtime attributes".invalidNel)
       result match {
-        case Valid(x) => fail("A failure was expected.")
+        case Valid(_) => fail("A failure was expected.")
         case Invalid(e) => assert(e.head == "Failed to get cpu mandatory key from runtime attributes")
       }
     }
+    
+    "return default values as WdlValues when they can be coerced into expected WdlTypes" in {
+      val optionalConfig = Option(TestConfig.allRuntimeAttrsConfig)
+
+      val defaultVals = Map(
+        "cpu" -> CpuValidation.configDefaultWdlValue(optionalConfig).get,
+        "failOnStderr" -> FailOnStderrValidation.configDefaultWdlValue(optionalConfig).get,
+        "continueOnReturnCode" -> ContinueOnReturnCodeValidation.configDefaultWdlValue(optionalConfig).get
+      )
+
+      val expectedDefaultVals = Map(
+        "cpu" -> WdlInteger(1),
+        "failOnStderr" -> WdlBoolean(false),
+        "continueOnReturnCode" -> WdlInteger(0)
+      )
+
+      defaultVals shouldBe expectedDefaultVals
+    }
+
+    "return default values as BadDefaultAttribute when they can't be coerced to expected WdlTypes" in {
+     val optionalInvalidAttrsConfig = Option(ConfigFactory.parseString(
+       """
+         |cpu = 1.4
+         |failOnStderr = "notReal"
+         |continueOnReturnCode = 0
+       """.stripMargin))
+
+     val defaultVals = Map(
+       "cpu" -> CpuValidation.configDefaultWdlValue(optionalInvalidAttrsConfig).get,
+       "failOnStderr" -> FailOnStderrValidation.configDefaultWdlValue(optionalInvalidAttrsConfig).get,
+       "continueOnReturnCode" -> ContinueOnReturnCodeValidation.configDefaultWdlValue(optionalInvalidAttrsConfig).get
+     )
+
+     val expectedDefaultVals = Map(
+       "cpu" -> BadDefaultAttribute(WdlString("1.4")),
+       "failOnStderr" -> BadDefaultAttribute(WdlString("notReal")),
+       "continueOnReturnCode" -> WdlInteger(0)
+     )
+
+     defaultVals shouldBe expectedDefaultVals
+    }
+
+    "should parse memory successfully" in {
+      val backendConfigTemplate: String =
+        s"""
+           |  default-runtime-attributes {
+           |     memory: "2 GB"
+           |  }
+           |""".stripMargin
+
+      val backendConfig: Config = ConfigFactory.parseString(backendConfigTemplate).getConfig("default-runtime-attributes")
+
+      val memoryVal = MemoryValidation.configDefaultString(RuntimeAttributesKeys.MemoryKey, Some(backendConfig))
+      MemoryValidation.withDefaultMemory(RuntimeAttributesKeys.MemoryKey, memoryVal.get).runtimeAttributeDefinition.factoryDefault shouldBe Some((WdlInteger(2000000000)))
+    }
+
+    "shouldn't throw up if the value for a default-runtime-attribute key cannot be coerced into an expected WdlType" in {
+      val backendConfigTemplate: String =
+        s"""
+           |  default-runtime-attributes {
+           |     memory: "blahblah"
+           |  }
+           |""".stripMargin
+
+      val backendConfig: Config = ConfigFactory.parseString(backendConfigTemplate).getConfig("default-runtime-attributes")
+
+      val memoryVal = MemoryValidation.configDefaultString(RuntimeAttributesKeys.MemoryKey, Some(backendConfig))
+      MemoryValidation.withDefaultMemory(RuntimeAttributesKeys.MemoryKey, memoryVal.get).runtimeAttributeDefinition.factoryDefault shouldBe Some(BadDefaultAttribute(WdlString("blahblah")))
+    }
+
+    "should be able to coerce a list of return codes into an WdlArray" in {
+      val optinalBackendConfig = Option(ConfigFactory.parseString(
+        s"""
+           |continueOnReturnCode = [0,1,2]
+           |""".stripMargin))
+
+      ContinueOnReturnCodeValidation.configDefaultWdlValue(optinalBackendConfig).get shouldBe WdlArray(WdlArrayType(WdlIntegerType), Array(WdlInteger(0), WdlInteger(1), WdlInteger(2)))
+    }
   }
 }
diff --git a/backend/src/test/scala/cromwell/backend/wdl/FileSizeSpec.scala b/backend/src/test/scala/cromwell/backend/wdl/FileSizeSpec.scala
new file mode 100644
index 000000000..bc265b9aa
--- /dev/null
+++ b/backend/src/test/scala/cromwell/backend/wdl/FileSizeSpec.scala
@@ -0,0 +1,109 @@
+package cromwell.backend.wdl
+
+import java.nio.file.{Paths, Path}
+import java.nio.file.StandardOpenOption._
+import scala.util.{Failure, Success, Try}
+
+import cromwell.backend.standard.{DefaultStandardExpressionFunctionsParams, StandardExpressionFunctions}
+import cromwell.core.CallContext
+import cromwell.core.path.DefaultPathBuilder
+import org.scalatest.{FlatSpec, Matchers}
+import wdl4s.wdl.values._
+import com.google.common.io.Files
+import fs2.{Task, Stream}
+
+class FileSizeSpec extends FlatSpec with Matchers {
+  val _readLinesLimit = 4
+  val _readBoolLimit = 5
+  val _readIntLimit = 6
+  val _readFloatLimit = 7
+  val _readStringLimit = 8
+  val _readJsonLimit = 9
+  val _readTsvLimit = 10
+  val _readMapLimit = 11
+  val _readObjectLimit = 12
+
+  val rlf = {
+    val path = DefaultPathBuilder.build("/tmp").get
+
+    val dp = DefaultStandardExpressionFunctionsParams(List(cromwell.core.path.DefaultPathBuilder), CallContext(path, "stdout", "stderr"))
+
+    new StandardExpressionFunctions(dp) {
+      override val fileSizeLimitationConfig =
+        new FileSizeLimitationConfig {
+          val readLinesLimit = _readLinesLimit
+          val readIntLimit = _readIntLimit
+          val readFloatLimit = _readFloatLimit
+          val readStringLimit = _readStringLimit
+          val readJsonLimit = _readJsonLimit
+          val readTsvLimit = _readTsvLimit
+          val readMapLimit = _readMapLimit
+          val readObjectLimit = _readObjectLimit
+          val readBoolLimit = _readBoolLimit
+        }
+    }
+  }
+
+  val tempDir = Files.createTempDir
+  tempDir.deleteOnExit
+
+  def testOverUnder(command: String, n: Int, f: ReadLikeFunctions => (Seq[Try[WdlValue]] => Try[WdlValue])) = {
+
+    def testInner(n: Int, test: Try[WdlValue] => Unit) = {
+
+      def createTempFileOfSize(size: Int): Path = {
+
+        val fn = tempDir.toString + "/" + scala.util.Random.alphanumeric.take(5).mkString
+        val jPath = Paths.get(fn)
+        jPath.toFile.deleteOnExit
+        val start = Stream[Task, Byte](1).repeat.take(size.toLong)
+        val end = fs2.io.file.writeAll[Task](jPath, Seq(CREATE_NEW, WRITE))
+        (start to end).run.unsafeRunSync
+        //jPath is now a file of n bytes, we can return it
+        jPath
+      }
+
+      val file = createTempFileOfSize(n)
+      val params = Seq(Try(WdlString(file.toString)))
+
+      f(rlf)(params) match {
+        case t => test(t)
+      }
+    }
+
+    def testOver() = {
+      testInner(n + 1, {
+        case Failure(_: FileSizeTooBig) => //success
+        case t => throw new RuntimeException(s"should not have eaten this file that is too big! msg: $t")
+      })
+    }
+
+    def testUnder() = {
+      testInner(n - 1, {
+        case Success(_) =>
+        case Failure(_: NumberFormatException) => //we're not testing parsing
+        case Failure(_: UnsupportedOperationException) => //we're not testing tsv compatibility
+        case Failure(t) => throw t
+      })
+    }
+
+    //construct a test for both over and under
+    List(
+      s"read $command" should "limit according to a setting" in testOver,
+      it should "allow when under the  limit" in testUnder
+    )
+  }
+
+  //test all the functions
+  List[(String, Int, ReadLikeFunctions => (Seq[Try[WdlValue]] => Try[WdlValue]))](
+    ("lines", _readLinesLimit, _.read_lines),
+    ("int", _readIntLimit, _.read_int),
+    ("map", _readMapLimit, _.read_map),
+    ("float", _readFloatLimit, _.read_float),
+    ("String", _readStringLimit, _.read_string),
+    ("tsv", _readTsvLimit, _.read_tsv),
+    ("object", _readObjectLimit, _.read_object)
+  ).flatMap {
+    (testOverUnder _).tupled
+  }
+}
diff --git a/backend/src/test/scala/cromwell/backend/wdl/ReadLikeFunctionsSpec.scala b/backend/src/test/scala/cromwell/backend/wdl/ReadLikeFunctionsSpec.scala
new file mode 100644
index 000000000..ad4716b95
--- /dev/null
+++ b/backend/src/test/scala/cromwell/backend/wdl/ReadLikeFunctionsSpec.scala
@@ -0,0 +1,99 @@
+package cromwell.backend.wdl
+
+import cromwell.core.path.PathBuilder
+import org.apache.commons.lang3.NotImplementedException
+import org.scalatest.{FlatSpec, Matchers}
+import wdl4s.wdl.expression.PureStandardLibraryFunctionsLike
+import wdl4s.wdl.types.{WdlFileType, WdlIntegerType, WdlOptionalType}
+import wdl4s.wdl.values.{WdlFloat, WdlInteger, WdlOptionalValue, WdlSingleFile, WdlString, WdlValue}
+
+import scala.util.{Failure, Success, Try}
+
+class ReadLikeFunctionsSpec extends FlatSpec with Matchers {
+
+  behavior of "ReadLikeFunctions.size"
+
+  it should "correctly report a 2048 byte file, in bytes by default" in {
+    val readLike = new TestReadLikeFunctions(Success(2048d))
+    readLike.size(Seq(Success(WdlSingleFile("blah")))) should be(Success(WdlFloat(2048d)))
+  }
+
+  it should "correctly report a 2048 byte file, in bytes" in {
+    val readLike = new TestReadLikeFunctions(Success(2048d))
+    readLike.size(Seq(Success(WdlSingleFile("blah")), Success(WdlString("B")))) should be(Success(WdlFloat(2048d)))
+  }
+
+  it should "correctly report a 2048 byte file, in KB" in {
+    val readLike = new TestReadLikeFunctions(Success(2048d))
+    readLike.size(Seq(Success(WdlSingleFile("blah")), Success(WdlString("KB")))) should be(Success(WdlFloat(2.048d)))
+  }
+
+  it should "correctly report a 2048 byte file, in KiB" in {
+    val readLike = new TestReadLikeFunctions(Success(2048d))
+    readLike.size(Seq(Success(WdlSingleFile("blah")), Success(WdlString("Ki")))) should be(Success(WdlFloat(2d)))
+  }
+
+  it should "correctly report the size of a supplied, optional, 2048 byte file" in {
+    val readLike = new TestReadLikeFunctions(Success(2048d))
+    readLike.size(Seq(Success(WdlOptionalValue(WdlFileType, Some(WdlSingleFile("blah")))))) should be(Success(WdlFloat(2048d)))
+  }
+
+  it should "correctly report the size of a supplied, optional optional, 2048 byte file" in {
+    val readLike = new TestReadLikeFunctions(Success(2048d))
+    readLike.size(Seq(Success(WdlOptionalValue(WdlOptionalType(WdlFileType), Some(WdlOptionalValue(WdlFileType, Some(WdlSingleFile("blah")))))))) should be(Success(WdlFloat(2048d)))
+  }
+
+  it should "correctly report the size of a supplied, optional, 2048 byte file, in MB" in {
+    val readLike = new TestReadLikeFunctions(Success(2048d))
+    readLike.size(Seq(Success(WdlOptionalValue(WdlFileType, Some(WdlSingleFile("blah")))), Success(WdlString("MB")))) should be(Success(WdlFloat(0.002048d)))
+  }
+
+  it should "correctly report that an unsupplied optional file is empty" in {
+    val readLike = new TestReadLikeFunctions(Success(2048d))
+    readLike.size(Seq(Success(WdlOptionalValue(WdlFileType, None)))) should be(Success(WdlFloat(0d)))
+  }
+
+  it should "correctly report that an unsupplied File?? is empty" in {
+    val readLike = new TestReadLikeFunctions(Success(2048d))
+    readLike.size(Seq(Success(WdlOptionalValue(WdlOptionalType(WdlFileType), None)))) should be(Success(WdlFloat(0d)))
+  }
+
+  it should "correctly report that an unsupplied optional file is empty, even in MB" in {
+    val readLike = new TestReadLikeFunctions(Success(2048d))
+    readLike.size(Seq(Success(WdlOptionalValue(WdlFileType, None)), Success(WdlString("MB")))) should be(Success(WdlFloat(0d)))
+  }
+
+  it should "refuse to report file sizes for Ints" in {
+    val readLike = new TestReadLikeFunctions(Failure(new Exception("Bad result: WdlIntegers shouldn't even be tried for getting file size")))
+    val oops = readLike.size(Seq(Success(WdlInteger(7))))
+    oops match {
+      case Success(x) => fail(s"Expected a string to not have a file length but instead got $x")
+      case Failure(e) => e.getMessage should be("The 'size' method expects a 'File' or 'File?' argument but instead got Int.")
+    }
+  }
+
+  it should "refuse to report file sizes for Int?s" in {
+    val readLike = new TestReadLikeFunctions(Failure(new Exception("Bad result: WdlIntegers shouldn't even be tried for getting file size")))
+    val oops = readLike.size(Seq(Success(WdlOptionalValue(WdlIntegerType, None))))
+    oops match {
+      case Success(x) => fail(s"Expected a string to not have a file length but instead got $x")
+      case Failure(e) => e.getMessage should be("The 'size' method expects a 'File' or 'File?' argument but instead got Int?.")
+    }
+  }
+
+  it should "pass on underlying size reading errors" in {
+    val readLike = new TestReadLikeFunctions(Failure(new Exception("'size' inner exception, expect me to be passed on")))
+    val oops = readLike.size(Seq(Success(WdlSingleFile("blah"))))
+    oops match {
+      case Success(_) => fail(s"The 'size' engine function didn't return the error generated in the inner 'size' method")
+      case Failure(e) => e.getMessage should be("'size' inner exception, expect me to be passed on")
+    }
+  }
+}
+
+
+class TestReadLikeFunctions(sizeResult: Try[Double]) extends PureStandardLibraryFunctionsLike with ReadLikeFunctions {
+  override protected def size(file: WdlValue): Try[Double] = sizeResult
+  override def pathBuilders: List[PathBuilder] = throw new NotImplementedException("Didn't expect ReadLikefunctionsSpec to need pathBuilders")
+}
+
diff --git a/build.sbt b/build.sbt
index a8f7e37f6..98789be3a 100644
--- a/build.sbt
+++ b/build.sbt
@@ -20,6 +20,16 @@ lazy val databaseMigration = (project in file("database/migration"))
   .dependsOn(core)
   .withTestSettings
 
+lazy val dockerHashing = (project in file("dockerHashing"))
+  .settings(dockerHashingSettings: _*)
+  .dependsOn(core)
+  .dependsOn(core % "test->test")
+  .withTestSettings
+
+lazy val cromwellApiClient = (project in file("cromwellApiClient"))
+  .settings(cromwellApiClientSettings: _*)
+  .withTestSettings
+
 lazy val services = (project in file("services"))
   .settings(servicesSettings:_*)
   .withTestSettings
@@ -43,6 +53,7 @@ lazy val sfsBackend = (project in backendRoot / "sfs")
   .dependsOn(backend)
   .dependsOn(gcsFileSystem)
   .dependsOn(backend % "test->test")
+  .dependsOn(services % "test->test")
 
 lazy val tesBackend = (project in backendRoot / "tes")
   .settings(tesBackendSettings:_*)
@@ -50,12 +61,6 @@ lazy val tesBackend = (project in backendRoot / "tes")
   .dependsOn(sfsBackend)
   .dependsOn(backend % "test->test")
 
-lazy val htCondorBackend = (project in backendRoot / "htcondor")
-  .settings(htCondorBackendSettings:_*)
-  .withTestSettings
-  .dependsOn(sfsBackend)
-  .dependsOn(backend % "test->test")
-
 lazy val sparkBackend = (project in backendRoot / "spark")
   .settings(sparkBackendSettings:_*)
   .withTestSettings
@@ -69,11 +74,13 @@ lazy val jesBackend = (project in backendRoot / "jes")
   .dependsOn(gcsFileSystem)
   .dependsOn(backend % "test->test")
   .dependsOn(gcsFileSystem % "test->test")
+  .dependsOn(services % "test->test")
 
 lazy val engine = (project in file("engine"))
   .settings(engineSettings: _*)
   .withTestSettings
   .dependsOn(core)
+  .dependsOn(dockerHashing)
   .dependsOn(services)
   .dependsOn(backend)
   .dependsOn(gcsFileSystem)
@@ -90,22 +97,22 @@ lazy val root = (project in file("."))
   .withTestSettings
   // Full list of all sub-projects to build with the root (ex: include in `sbt test`)
   .aggregate(core)
+  .aggregate(dockerHashing)
   .aggregate(gcsFileSystem)
   .aggregate(databaseSql)
   .aggregate(databaseMigration)
   .aggregate(services)
   .aggregate(backend)
   .aggregate(sfsBackend)
-  .aggregate(htCondorBackend)
   .aggregate(sparkBackend)
   .aggregate(jesBackend)
   .aggregate(tesBackend)
   .aggregate(engine)
+  .aggregate(cromwellApiClient)
   // Next level of projects to include in the fat jar (their dependsOn will be transitively included)
   .dependsOn(engine)
   .dependsOn(jesBackend)
   .dependsOn(tesBackend)
-  .dependsOn(htCondorBackend)
   .dependsOn(sparkBackend)
   // Dependencies for tests
   .dependsOn(engine % "test->test")
diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf
index 6405857af..865559f87 100644
--- a/core/src/main/resources/reference.conf
+++ b/core/src/main/resources/reference.conf
@@ -38,7 +38,7 @@ akka {
     }
 
     # A dispatcher for engine actors
-    # Because backends behaviour is unpredictable (potentially blocking, slow) the engine runs
+    # Because backends behavior is unpredictable (potentially blocking, slow) the engine runs
     # on its own dispatcher to prevent backends from affecting its performance.
     engine-dispatcher {
       type = Dispatcher
@@ -59,14 +59,33 @@ akka {
 
     # Note that without further configuration, all other actors run on the default dispatcher
   }
+
+  coordinated-shutdown.phases {
+    abort-all-workflows {
+      # This phase is used to give time to Cromwell to abort all workflows upon shutdown.
+      # It's only used if system.abort-jobs-on-terminate = true
+      # This timeout can be adusted to give more or less time to Cromwell to abort workflows
+      timeout = 1 hour
+      depends-on = [service-unbind]
+    }
+
+    stop-io-activity{
+      # Adjust this timeout according to the maximum amount of time Cromwell
+      # should be allowed to spend flushing its database queues
+      timeout = 30 minutes
+      depends-on = [service-stop]
+    }
+  }
 }
 
 system {
   # If 'true', a SIGINT will trigger Cromwell to attempt to abort all currently running jobs before exiting
-  #abort-jobs-on-terminate = false
+  abort-jobs-on-terminate = false
 
-  # Max number of retries per job that the engine will attempt in case of a retryable failure received from the backend
-  max-retries = 10
+  # If 'true', a SIGTERM or SIGINT will trigger Cromwell to attempt to gracefully shutdown in server mode,
+  # in particular clearing up all queued database writes before letting the JVM shut down.
+  # The shutdown is a multi-phase process, each phase having its own configurable timeout. See the Dev Wiki for more details.
+  graceful-server-shutdown = true
 
   # If 'true' then when Cromwell starts up, it tries to restart incomplete workflows
   workflow-restart = true
@@ -85,6 +104,39 @@ system {
 
   # Default number of cache read workers
   number-of-cache-read-workers = 25
+
+  io {
+    # Global Throttling - This is mostly useful for GCS and can be adjusted to match
+    # the quota availble on the GCS API
+    number-of-requests = 100000
+    per = 100 seconds
+
+    # Number of times an I/O operation should be attempted before giving up and failing it.
+    number-of-attempts = 5
+  }
+
+  # Maximum number of input file bytes allowed in order to read each type. 
+  # If exceeded a FileSizeTooBig exception will be thrown.
+  input-read-limits {
+
+    lines = 128000
+
+    bool = 7
+
+    int = 19
+
+    float = 50
+
+    string = 128000
+
+    json = 128000
+
+    tsv = 128000
+
+    map = 128000
+
+    object = 128000
+  }
 }
 
 workflow-options {
@@ -135,19 +187,29 @@ google {
     #{
     #  name = "service-account"
     #  scheme = "service_account"
+    #  Choose between PEM file and JSON file as a credential format. They're mutually exclusive.
+    #  PEM format:
     #  service-account-id = "my-service-account"
     #  pem-file = "/path/to/file.pem"
+    #  JSON format:
+    #  json-file = "/path/to/file.json"
     #}
   ]
 }
 
 docker {
-  // Set this to match your available quota against the Google Container Engine API
-  gcr-api-queries-per-100-seconds = 1000
-  // Time in minutes before an entry expires from the docker hashes cache and needs to be fetched again
-  cache-entry-ttl = "20 minutes"
-  // Maximum number of elements to be kept in the cache. If the limit is reached, old elements will be removed from the cache
-  cache-size = 200
+  hash-lookup {
+    // Set this to match your available quota against the Google Container Engine API
+    gcr-api-queries-per-100-seconds = 1000
+    // Time in minutes before an entry expires from the docker hashes cache and needs to be fetched again
+    cache-entry-ttl = "20 minutes"
+    // Maximum number of elements to be kept in the cache. If the limit is reached, old elements will be removed from the cache
+    cache-size = 200
+    // How should docker hashes be looked up. Possible values are "local" and "remote"
+    // "local": Lookup hashes on the local docker daemon using the cli
+    // "remote": Lookup hashes on docker hub and gcr
+    method = "remote"
+  }
 }
 
 engine {
@@ -159,11 +221,14 @@ engine {
   # }
   # You will need to provide the engine with a gcs filesystem
   # Note that the default filesystem (local) is always available.
-  #filesystems {
-  #  gcs {
-  #    auth = "application-default"
-  #  }
-  #}
+  filesystems {
+    #  gcs {
+    #    auth = "application-default"
+    #  }
+    local {
+      enabled: true
+    }
+  }
 }
 
 backend {
@@ -177,6 +242,14 @@ backend {
         #concurrent-job-limit = 5
 
         run-in-background = true
+        # `script-epilogue` configures a shell command to run after the execution of every command block.
+        #
+        # If this value is not set explicitly, the default value is `sync`, equivalent to:
+        # script-epilogue = "sync"
+        #
+        # To turn off the default `sync` behavior set this value to an empty string:
+        # script-epilogue = ""
+
         runtime-attributes = """
         String? docker
         String? docker_user
@@ -186,11 +259,13 @@ backend {
         docker run \
           --rm -i \
           ${"--user " + docker_user} \
+          --entrypoint /bin/bash \
           -v ${cwd}:${docker_cwd} \
-          ${docker} \
-          /bin/bash ${script}
+          ${docker} ${script}
         """
 
+
+
         # Root directory where Cromwell writes job results.  This directory must be
         # visible and writeable by the Cromwell process as well as the jobs that Cromwell
         # launches.
@@ -220,6 +295,11 @@ backend {
             }
           }
         }
+
+        default-runtime-attributes {
+          failOnStderr: false
+          continueOnReturnCode: 0
+        }
       }
     }
 
@@ -229,6 +309,13 @@ backend {
     #    root = "cromwell-executions"
     #    dockerRoot = "/cromwell-executions"
     #    endpoint = "http://127.0.0.1:9000/v1/jobs"
+    #    default-runtime-attributes {
+    #      cpu: 1
+    #      failOnStderr: false
+    #      continueOnReturnCode: 0
+    #      memory: "2 GB"
+    #      disk: "2 GB"
+    #    }
     #  }
     #}
 
@@ -278,6 +365,28 @@ backend {
     #  }
     #}
 
+    #SLURM {
+    #  actor-factory = "cromwell.backend.impl.sfs.config.ConfigBackendLifecycleActorFactory"
+    #  config {
+    #    runtime-attributes = """
+    #    Int runtime_minutes = 600
+    #    Int cpus = 2
+    #    Int requested_memory_mb_per_core = 8000
+    #    String queue = "short"
+    #    """
+    #  
+    #    submit = """
+    #        sbatch -J ${job_name} -D ${cwd} -o ${out} -e ${err} -t ${runtime_minutes} -p ${queue} \
+    #        ${"-n " + cpus} \
+    #        --mem-per-cpu=${requested_memory_mb_per_core} \
+    #        --wrap "/bin/bash ${script}"
+    #    """
+    #    kill = "scancel ${job_id}"
+    #    check-alive = "squeue -j ${job_id}"
+    #    job-id-regex = "Submitted batch job (\\d+).*"
+    #  }
+    #}
+
     # Example backend that _only_ runs workflows that specify docker for every command.
     #Docker {
     #  actor-factory = "cromwell.backend.impl.sfs.config.ConfigBackendLifecycleActorFactory"
@@ -289,48 +398,62 @@ backend {
     #}
 
     #HtCondor {
-    #  actor-factory = "cromwell.backend.impl.htcondor.HtCondorBackendFactory"
+    #  actor-factory = "cromwell.backend.impl.sfs.config.ConfigBackendLifecycleActorFactory"
     #  config {
-    #    # Root directory where Cromwell writes job results.  This directory must be
-    #    # visible and writeable by the Cromwell process as well as the jobs that Cromwell
-    #    # launches.
-    #    root: "cromwell-executions"
+    #    runtime-attributes = """
+    #      Int cpu = 1
+    #      Float memory_mb = 512.0
+    #      Float disk_kb = 256000.0
+    #      String? nativeSpecs
+    #      String? docker
+    #    """
     #
-    #    #Placeholders:
-    #    #1. Working directory.
-    #    #2. Working directory volume.
-    #    #3. Inputs volumes.
-    #    #4. Output volume.
-    #    #5. Docker image.
-    #    #6. Job command.
-    #    docker {
-    #      #Allow soft links in dockerized jobs
-    #      cmd = "docker run -w %s %s %s %s --rm %s /bin/bash -c \"%s\""
-    #      defaultWorkingDir = "/workingDir/"
-    #      defaultOutputDir = "/output/"
-    #    }
+    #    submit = """
+    #      chmod 755 ${script}
+    #      cat > ${cwd}/execution/submitFile <<EOF
+    #      Iwd=${cwd}/execution
+    #      requirements=${nativeSpecs}
+    #      leave_in_queue=true
+    #      request_memory=${memory_mb}
+    #      request_disk=${disk_kb}
+    #      error=${err}
+    #      output=${out}
+    #      log_xml=true
+    #      request_cpus=${cpu}
+    #      executable=${script}
+    #      log=${cwd}/execution/execution.log
+    #      queue
+    #      EOF
+    #      condor_submit ${cwd}/execution/submitFile
+    #    """
     #
-    #    cache {
-    #      provider = "cromwell.backend.impl.htcondor.caching.provider.mongodb.MongoCacheActorFactory"
-    #      enabled = false
-    #      forceRewrite = false
-    #      db {
-    #        host = "127.0.0.1"
-    #        port = 27017
-    #        name = "htcondor"
-    #        collection = "cache"
-    #      }
-    #    }
+    #    submit-docker = """
+    #      chmod 755 ${script}
+    #      cat > ${cwd}/execution/dockerScript <<EOF
+    #      #!/bin/bash
+    #      docker run --rm -i -v ${cwd}:${docker_cwd} ${docker} /bin/bash ${script}
+    #      EOF
+    #      chmod 755 ${cwd}/execution/dockerScript
+    #      cat > ${cwd}/execution/submitFile <<EOF
+    #      Iwd=${cwd}/execution
+    #      requirements=${nativeSpecs}
+    #      leave_in_queue=true
+    #      request_memory=${memory_mb}
+    #      request_disk=${disk_kb}
+    #      error=${cwd}/execution/stderr
+    #      output=${cwd}/execution/stdout
+    #      log_xml=true
+    #      request_cpus=${cpu}
+    #      executable=${cwd}/execution/dockerScript
+    #      log=${cwd}/execution/execution.log
+    #      queue
+    #      EOF
+    #      condor_submit ${cwd}/execution/submitFile
+    #    """    
     #
-    #    filesystems {
-    #      local {
-    #        localization: [
-    #          "hard-link", "soft-link", "copy"
-    #        ]
-    #      }
-    #    }
-    #    # Time (in seconds) to wait before re-checking the status of the job again
-    #    poll-interval = 3
+    #    kill = "condor_rm ${job_id}"
+    #    check-alive = "condor_q ${job_id}"
+    #    job-id-regex = "(?sm).*cluster (\\d+)..*"
     #  }
     #}
 
@@ -389,8 +512,6 @@ backend {
     #      # Pipelines and manipulate auth JSONs.
     #      auth = "application-default"
     #
-    #      # Specifies the zone(s) to use for JES jobs unless overridden by a task's runtime attributes
-    #      default-zones = ["us-central1-b"]
     #
     #      // alternative service account to use on the launched compute instance
     #      // NOTE: If combined with service account authorization, both that serivce account and this service account
@@ -399,14 +520,41 @@ backend {
     #
     #      # Endpoint for APIs, no reason to change this unless directed by Google.
     #      endpoint-url = "https://genomics.googleapis.com/"
+    #
+    #      # Restrict access to VM metadata. Useful in cases when untrusted containers are running under a service
+    #      # account not owned by the submitting user
+    #      restrict-metadata-access = false
     #    }
     #
     #    filesystems {
     #      gcs {
     #        # A reference to a potentially different auth for manipulating files via engine functions.
     #        auth = "application-default"
+    #
+    #        caching {
+    #          # When a cache hit is found, the following duplication strategy will be followed to use the cached outputs
+    #          # Possible values: "copy", "reference". Defaults to "copy"
+    #          # "copy": Copy the output files
+    #          # "reference": DO NOT copy the output files but point to the original output files instead.
+    #          #              Will still make sure than all the original output files exist and are accessible before
+    #          #              going forward with the cache hit.
+    #          duplication-strategy = "copy"
+    #        }
     #      }
     #    }
+    #
+    #    default-runtime-attributes {
+    #      cpu: 1
+    #      failOnStderr: false
+    #      continueOnReturnCode: 0
+    #      memory: "2 GB"
+    #      bootDiskSizeGb: 10
+    #      # Allowed to be a String, or a list of Strings
+    #      disks: "local-disk 10 SSD"
+    #      noAddress: false
+    #      preemptible: 0
+    #      zones: ["us-central1-a", "us-central1-b"]
+    #    }
     #  }
     #}
 
@@ -436,10 +584,10 @@ services {
       # performance but will both lead to a higher memory usage as well as increase the risk that metadata events
       # might not have been persisted in the event of a Cromwell crash.
       #
-      # For normal usage the default value of 1 (effectively no batching) should be fine but for larger/production
-      # environments we recommend a value of at least 500. There'll be no one size fits all number here so we recommend
-      # benchmarking performance and tuning the value to match your environment
-      # db-batch-size = 1
+      # For normal usage the default value of 200 should be fine but for larger/production environments we recommend a
+      # value of at least 500. There'll be no one size fits all number here so we recommend benchmarking performance and
+      # tuning the value to match your environment.
+      # db-batch-size = 200
       #
       # Periodically the stored metadata events will be forcibly written to the DB regardless of if the batch size
       # has been reached. This is to prevent situations where events wind up never being written to an incomplete batch
@@ -451,7 +599,7 @@ services {
 
 database {
   # hsql default
-  driver = "slick.driver.HsqldbDriver$"
+  profile = "slick.jdbc.HsqldbProfile$"
   db {
     driver = "org.hsqldb.jdbcDriver"
     url = "jdbc:hsqldb:mem:${uniqueSchema};shutdown=false;hsqldb.tx=mvcc"
diff --git a/core/src/main/scala/cromwell/core/BackendDockerConfiguration.scala b/core/src/main/scala/cromwell/core/BackendDockerConfiguration.scala
new file mode 100644
index 000000000..5957bd50c
--- /dev/null
+++ b/core/src/main/scala/cromwell/core/BackendDockerConfiguration.scala
@@ -0,0 +1,31 @@
+package cromwell.core
+
+import com.typesafe.config.Config
+import cromwell.core.ConfigUtil._
+
+/**
+  * Encapsulate docker credential information.
+  */
+case class DockerCredentials(account: String, token: String)
+
+case class BackendDockerConfiguration(dockerCredentials: Option[DockerCredentials])
+
+/**
+  * Singleton encapsulating a DockerConf instance.
+  */
+object BackendDockerConfiguration {
+
+  private val dockerKeys = Set("account", "token")
+
+  def build(config: Config) = {
+    import net.ceedubs.ficus.Ficus._
+    val dockerConf: Option[DockerCredentials] = for {
+      dockerConf <- config.as[Option[Config]]("dockerhub")
+      _ = dockerConf.warnNotRecognized(dockerKeys, "dockerhub")
+      account <- dockerConf.validateString("account").toOption
+      token <- dockerConf.validateString("token").toOption
+    } yield DockerCredentials(account, token)
+
+    new BackendDockerConfiguration(dockerConf)
+  }
+}
diff --git a/core/src/main/scala/cromwell/core/CallKey.scala b/core/src/main/scala/cromwell/core/CallKey.scala
index 547eadabd..02a987431 100644
--- a/core/src/main/scala/cromwell/core/CallKey.scala
+++ b/core/src/main/scala/cromwell/core/CallKey.scala
@@ -1,7 +1,7 @@
 package cromwell.core
 
-import wdl4s.Call
+import wdl4s.wdl.WdlCall
 
 trait CallKey extends JobKey {
-  def scope: Call
+  def scope: WdlCall
 }
diff --git a/core/src/main/scala/cromwell/core/ConfigUtil.scala b/core/src/main/scala/cromwell/core/ConfigUtil.scala
index 881fec686..40dae4784 100644
--- a/core/src/main/scala/cromwell/core/ConfigUtil.scala
+++ b/core/src/main/scala/cromwell/core/ConfigUtil.scala
@@ -7,7 +7,7 @@ import cats.syntax.validated._
 import com.typesafe.config.{Config, ConfigException, ConfigValue}
 import org.slf4j.LoggerFactory
 
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
 import scala.reflect.{ClassTag, classTag}
 
 object ConfigUtil {
@@ -15,7 +15,7 @@ object ConfigUtil {
   val validationLogger = LoggerFactory.getLogger("ConfigurationValidation")
 
   implicit class EnhancedConfig(val config: Config) extends AnyVal {
-    def keys = config.entrySet().toSet map { v: java.util.Map.Entry[String, ConfigValue] => v.getKey }
+    def keys = config.entrySet().asScala.toSet map { v: java.util.Map.Entry[String, ConfigValue] => v.getKey }
 
     /**
      * For keys that are in the configuration but not in the reference keySet, log a warning.
@@ -37,14 +37,14 @@ object ConfigUtil {
     def validateString(key: String): ValidatedNel[String, String] = try {
       config.getString(key).validNel
     } catch {
-      case e: ConfigException.Missing => s"Could not find key: $key".invalidNel
+      case _: ConfigException.Missing => s"Could not find key: $key".invalidNel
     }
 
     def validateConfig(key: String): ValidatedNel[String, Config] = try {
       config.getConfig(key).validNel
     } catch {
-      case e: ConfigException.Missing => s"Could not find key: $key".invalidNel
-      case e: ConfigException.WrongType => s"key $key cannot be parsed to a Config".invalidNel
+      case _: ConfigException.Missing => s"Could not find key: $key".invalidNel
+      case _: ConfigException.WrongType => s"key $key cannot be parsed to a Config".invalidNel
     }
 
   }
diff --git a/core/src/main/scala/cromwell/core/CromwellUserGuardianStrategy.scala b/core/src/main/scala/cromwell/core/CromwellUserGuardianStrategy.scala
index 52af92e5c..79e1621ca 100644
--- a/core/src/main/scala/cromwell/core/CromwellUserGuardianStrategy.scala
+++ b/core/src/main/scala/cromwell/core/CromwellUserGuardianStrategy.scala
@@ -5,7 +5,7 @@ import akka.actor.{ActorInitializationException, OneForOneStrategy, SupervisorSt
 
 class CromwellUserGuardianStrategy extends SupervisorStrategyConfigurator {
   override def create(): SupervisorStrategy = OneForOneStrategy() {
-    case aie: ActorInitializationException => Escalate
+    case _: ActorInitializationException => Escalate
     case t => akka.actor.SupervisorStrategy.defaultDecider.applyOrElse(t, (_: Any) => Escalate)
   }
 }
diff --git a/core/src/main/scala/cromwell/core/DockerConfiguration.scala b/core/src/main/scala/cromwell/core/DockerConfiguration.scala
new file mode 100644
index 000000000..ca0f872fc
--- /dev/null
+++ b/core/src/main/scala/cromwell/core/DockerConfiguration.scala
@@ -0,0 +1,45 @@
+package cromwell.core
+
+import com.typesafe.config.ConfigFactory
+
+import scala.concurrent.duration.FiniteDuration
+import cats.data.Validated._
+import cats.syntax.cartesian._
+import lenthall.exception.AggregatedMessageException
+import net.ceedubs.ficus.Ficus._
+import lenthall.validation.Validation._
+
+object DockerConfiguration {
+  private lazy val dockerConfig = ConfigFactory.load().getConfig("docker")
+  private lazy val dockerHashLookupConfig = dockerConfig.getConfig("hash-lookup")
+  
+  lazy val instance: DockerConfiguration = {
+    val gcrApiQueriesPer100Seconds = validate { dockerHashLookupConfig.as[Int]("gcr-api-queries-per-100-seconds") }
+    val cacheEntryTtl = validate { dockerHashLookupConfig.as[FiniteDuration]("cache-entry-ttl") }
+    val cacheSize = validate { dockerHashLookupConfig.as[Long]("cache-size") }
+    val method = validate { dockerHashLookupConfig.as[String]("method") } map {
+      case "local" => DockerLocalLookup
+      case "remote" => DockerRemoteLookup
+      case other => throw new IllegalArgumentException(s"Unrecognized docker hash lookup method: $other")
+    }
+
+    val dockerConfiguration = (gcrApiQueriesPer100Seconds |@| cacheEntryTtl |@| cacheSize |@| method) map  DockerConfiguration.apply
+    
+    dockerConfiguration match {
+      case Valid(conf) => conf
+      case Invalid(errors) => throw AggregatedMessageException("Invalid docker configuration", errors.toList)
+    }
+  }
+}
+
+case class DockerConfiguration(
+                                gcrApiQueriesPer100Seconds: Int,
+                                cacheEntryTtl: FiniteDuration,
+                                cacheSize: Long,
+                                method: DockerHashLookupMethod
+                              )
+
+sealed trait DockerHashLookupMethod
+
+case object DockerLocalLookup extends DockerHashLookupMethod
+case object DockerRemoteLookup extends DockerHashLookupMethod
diff --git a/core/src/main/scala/cromwell/core/DockerCredentials.scala b/core/src/main/scala/cromwell/core/DockerCredentials.scala
deleted file mode 100644
index cfe7be01f..000000000
--- a/core/src/main/scala/cromwell/core/DockerCredentials.scala
+++ /dev/null
@@ -1,40 +0,0 @@
-package cromwell.core
-
-import com.typesafe.config.Config
-import cromwell.core.ConfigUtil._
-
-/**
- * Encapsulate docker credential information.
- */
-case class DockerCredentials(account: String, token: String)
-
-case class DockerHubConfiguration(namespace: String, v1Registry: String, v2Registry: String)
-
-case class DockerConfiguration(dockerCredentials: Option[DockerCredentials], dockerHubConf: DockerHubConfiguration)
-
-/**
- * Singleton encapsulating a DockerConf instance.
- */
-object DockerConfiguration {
-
-  private val dockerKeys = Set("account", "token")
-
-  def build(config: Config) = {
-    import net.ceedubs.ficus.Ficus._
-    val dockerConf: Option[DockerCredentials] = for {
-      dockerConf <- config.as[Option[Config]]("dockerhub")
-      _ = dockerConf.warnNotRecognized(dockerKeys, "dockerhub")
-      account <- dockerConf.validateString("account").toOption
-      token <- dockerConf.validateString("token").toOption
-    } yield DockerCredentials(account, token)
-
-    val dockerHubConf = {
-      DockerHubConfiguration(
-        namespace = config.as[Option[String]]("docker.hub.namespace").getOrElse("docker.io"),
-        v1Registry = config.as[Option[String]]("docker.hub.v1Registry").getOrElse("index.docker.io"),
-        v2Registry = config.as[Option[String]]("docker.hub.v2Registry").getOrElse("registry-1.docker.io")
-      )
-    }
-    new DockerConfiguration(dockerConf, dockerHubConf)
-  }
-}
diff --git a/core/src/main/scala/cromwell/core/ExecutionStatus.scala b/core/src/main/scala/cromwell/core/ExecutionStatus.scala
index 2ef66629c..b4ee84f9e 100644
--- a/core/src/main/scala/cromwell/core/ExecutionStatus.scala
+++ b/core/src/main/scala/cromwell/core/ExecutionStatus.scala
@@ -2,13 +2,28 @@ package cromwell.core
 
 object ExecutionStatus extends Enumeration {
   type ExecutionStatus = Value
-  val NotStarted, QueuedInCromwell, Starting, Running, Failed, Preempted, Done, Bypassed, Aborted = Value
-  val TerminalStatuses = Set(Failed, Done, Aborted, Preempted, Bypassed)
+  val NotStarted, QueuedInCromwell, Starting, Running, Failed, RetryableFailure, Done, Bypassed, Aborted = Value
+  val TerminalStatuses = Set(Failed, Done, Aborted, Bypassed)
+  val TerminalOrRetryableStatuses = TerminalStatuses + RetryableFailure
 
-  implicit class EnhancedExecutionStatus(val status: ExecutionStatus) extends AnyVal {
-    def isTerminal: Boolean = {
-      TerminalStatuses contains status
+  implicit val ExecutionStatusOrdering = Ordering.by { status: ExecutionStatus =>
+    status match {
+      case NotStarted => 0
+      case QueuedInCromwell => 1
+      case Starting => 2
+      case Running => 3
+      case Aborted => 4
+      case Bypassed => 5
+      case RetryableFailure => 6
+      case Failed => 7
+      case Done => 8
     }
+  }
+  
+  implicit class EnhancedExecutionStatus(val status: ExecutionStatus) extends AnyVal {
+    def isTerminal: Boolean = TerminalStatuses contains status
+
+    def isTerminalOrRetryable: Boolean = TerminalOrRetryableStatuses contains status
 
     def isDoneOrBypassed: Boolean = status == Done || status == Bypassed
   }
diff --git a/core/src/main/scala/cromwell/core/JobKey.scala b/core/src/main/scala/cromwell/core/JobKey.scala
index 9fd22b31e..f230134ce 100644
--- a/core/src/main/scala/cromwell/core/JobKey.scala
+++ b/core/src/main/scala/cromwell/core/JobKey.scala
@@ -1,9 +1,9 @@
 package cromwell.core
 
-import wdl4s.{GraphNode, Scope}
+import wdl4s.wdl.{Scope, WdlGraphNode}
 
 trait JobKey {
-  def scope: Scope with GraphNode
+  def scope: Scope with WdlGraphNode
   def index: Option[Int]
   def attempt: Int
   def tag: String
diff --git a/core/src/main/scala/cromwell/core/MonitoringCompanionActor.scala b/core/src/main/scala/cromwell/core/MonitoringCompanionActor.scala
new file mode 100644
index 000000000..53f0e4de6
--- /dev/null
+++ b/core/src/main/scala/cromwell/core/MonitoringCompanionActor.scala
@@ -0,0 +1,46 @@
+package cromwell.core
+
+import akka.actor.{Actor, ActorLogging, ActorRef, Props}
+import cromwell.core.MonitoringCompanionActor._
+import cromwell.util.GracefulShutdownHelper.ShutdownCommand
+
+import scala.concurrent.duration._
+import scala.language.postfixOps
+
+object MonitoringCompanionActor {
+  sealed trait MonitoringCompanionCommand
+  private [core] case object AddWork extends MonitoringCompanionCommand
+  private [core] case object RemoveWork extends MonitoringCompanionCommand
+  private [core] def props(actorToMonitor: ActorRef) = Props(new MonitoringCompanionActor(actorToMonitor))
+}
+
+private [core] class MonitoringCompanionActor(actorToMonitor: ActorRef) extends Actor with ActorLogging {
+  private var workCount: Int = 0
+  
+  override def receive = {
+    case AddWork => workCount += 1
+    case RemoveWork => workCount -= 1
+    case ShutdownCommand if workCount <= 0 =>
+      context stop actorToMonitor
+      context stop self
+    case ShutdownCommand => 
+      log.info(s"{} is still processing {} messages", actorToMonitor.path.name, workCount)
+      context.system.scheduler.scheduleOnce(1 second, self, ShutdownCommand)(context.dispatcher)
+      ()
+  }
+}
+
+trait MonitoringCompanionHelper { this: Actor =>
+  private val monitoringActor = context.actorOf(MonitoringCompanionActor.props(self))
+  private var shuttingDown: Boolean = false
+  
+  def addWork() = monitoringActor ! AddWork
+  def removeWork() = monitoringActor ! RemoveWork
+
+  val monitoringReceive: Receive = {
+    case ShutdownCommand if !shuttingDown => 
+      shuttingDown = true
+      monitoringActor ! ShutdownCommand
+    case ShutdownCommand => // Ignore if we're already shutting down
+  }
+}
diff --git a/core/src/main/scala/cromwell/core/WorkflowMetadataKeys.scala b/core/src/main/scala/cromwell/core/WorkflowMetadataKeys.scala
index 922be7ffb..00afb2b61 100644
--- a/core/src/main/scala/cromwell/core/WorkflowMetadataKeys.scala
+++ b/core/src/main/scala/cromwell/core/WorkflowMetadataKeys.scala
@@ -20,4 +20,9 @@ object WorkflowMetadataKeys {
   val SubmissionSection_Inputs = "inputs"
   val SubmissionSection_Options = "options"
   val SubmissionSection_Imports = "imports"
+  val SubmissionSection_WorkflowType = "workflowType"
+  val SubmissionSection_Labels = "labels"
+  val SubmissionSection_WorkflowTypeVersion = "workflowTypeVersion"
+
+  val Labels = "labels"
 }
diff --git a/core/src/main/scala/cromwell/core/WorkflowOptions.scala b/core/src/main/scala/cromwell/core/WorkflowOptions.scala
index 709e667e1..701ea5428 100644
--- a/core/src/main/scala/cromwell/core/WorkflowOptions.scala
+++ b/core/src/main/scala/cromwell/core/WorkflowOptions.scala
@@ -152,7 +152,7 @@ case class WorkflowOptions(jsObject: JsObject) {
   }
 
   lazy val defaultRuntimeOptions = jsObject.fields.get(defaultRuntimeOptionKey) match {
-    case Some(jsObj: JsObject) => TryUtil.sequenceMap(jsObj.fields map { case (k, v) => k -> WorkflowOptions.getAsJson(k, jsObj) })
+    case Some(jsObj: JsObject) => TryUtil.sequenceMap(jsObj.fields map { case (k, _) => k -> WorkflowOptions.getAsJson(k, jsObj) })
     case Some(jsVal) => Failure(new IllegalArgumentException(s"Unsupported JsValue for $defaultRuntimeOptionKey: $jsVal. Expected a JSON object."))
     case None => Failure(OptionNotFoundException(s"Cannot find definition for default runtime attributes"))
   }
diff --git a/core/src/main/scala/cromwell/core/WorkflowSourceFilesCollection.scala b/core/src/main/scala/cromwell/core/WorkflowSourceFilesCollection.scala
index 55fa68b47..281cdf653 100644
--- a/core/src/main/scala/cromwell/core/WorkflowSourceFilesCollection.scala
+++ b/core/src/main/scala/cromwell/core/WorkflowSourceFilesCollection.scala
@@ -1,59 +1,58 @@
 package cromwell.core
 
-import wdl4s.{WdlJson, WdlSource}
+import wdl4s.wdl.{WorkflowJson, WorkflowSource}
 
 /**
   * Represents the collection of source files that a user submits to run a workflow
   */
 
 sealed trait WorkflowSourceFilesCollection {
-  def wdlSource: WdlSource
-  def inputsJson: WdlJson
+  def workflowSource: WorkflowSource
+  def inputsJson: WorkflowJson
   def workflowOptionsJson: WorkflowOptionsJson
-  def labelsJson: WdlJson
-
+  def labelsJson: WorkflowJson
+  def workflowType: Option[WorkflowType]
+  def workflowTypeVersion: Option[WorkflowTypeVersion]
 
   def importsZipFileOption: Option[Array[Byte]] = this match {
     case _: WorkflowSourceFilesWithoutImports => None
-    case WorkflowSourceFilesWithDependenciesZip(_, _, _, _, importsZip) => Option(importsZip) // i.e. Some(importsZip) if our wiring is correct
+    case w: WorkflowSourceFilesWithDependenciesZip => Option(w.importsZip) // i.e. Some(importsZip) if our wiring is correct
   }
 
   def copyOptions(workflowOptions: WorkflowOptionsJson) = this match {
-    case w: WorkflowSourceFilesWithoutImports => WorkflowSourceFilesWithoutImports(
-      wdlSource = w.wdlSource,
-      inputsJson = w.inputsJson,
-      workflowOptionsJson = workflowOptions,
-      labelsJson = w.labelsJson)
-
-    case w: WorkflowSourceFilesWithDependenciesZip => WorkflowSourceFilesWithDependenciesZip(
-      wdlSource = w.wdlSource,
-      inputsJson = w.inputsJson,
-      workflowOptionsJson = workflowOptions,
-      labelsJson = w.labelsJson,
-      importsZip = w.importsZip)
+    case w: WorkflowSourceFilesWithoutImports => w.copy(workflowOptionsJson = workflowOptions)
+    case w: WorkflowSourceFilesWithDependenciesZip => w.copy(workflowOptionsJson = workflowOptions)
   }
 }
 
 object WorkflowSourceFilesCollection {
-  def apply(wdlSource: WdlSource,
-            inputsJson: WdlJson,
+  def apply(workflowSource: WorkflowSource,
+            workflowType: Option[WorkflowType],
+            workflowTypeVersion: Option[WorkflowTypeVersion],
+            inputsJson: WorkflowJson,
             workflowOptionsJson: WorkflowOptionsJson,
-            labelsJson: WdlJson,
+            labelsJson: WorkflowJson,
             importsFile: Option[Array[Byte]]): WorkflowSourceFilesCollection = importsFile match {
-    case Some(imports) => WorkflowSourceFilesWithDependenciesZip(wdlSource, inputsJson, workflowOptionsJson, labelsJson, imports)
-    case None => WorkflowSourceFilesWithoutImports(wdlSource, inputsJson, workflowOptionsJson, labelsJson)
+    case Some(imports) =>
+      WorkflowSourceFilesWithDependenciesZip(workflowSource, workflowType, workflowTypeVersion, inputsJson, workflowOptionsJson, labelsJson, imports)
+    case None =>
+      WorkflowSourceFilesWithoutImports(workflowSource, workflowType, workflowTypeVersion, inputsJson, workflowOptionsJson, labelsJson)
   }
 }
 
-final case class WorkflowSourceFilesWithoutImports(wdlSource: WdlSource,
-                                                   inputsJson: WdlJson,
+final case class WorkflowSourceFilesWithoutImports(workflowSource: WorkflowSource,
+                                                   workflowType: Option[WorkflowType],
+                                                   workflowTypeVersion: Option[WorkflowTypeVersion],
+                                                   inputsJson: WorkflowJson,
                                                    workflowOptionsJson: WorkflowOptionsJson,
-                                                   labelsJson: WdlJson) extends WorkflowSourceFilesCollection
+                                                   labelsJson: WorkflowJson) extends WorkflowSourceFilesCollection
 
-final case class WorkflowSourceFilesWithDependenciesZip(wdlSource: WdlSource,
-                                                        inputsJson: WdlJson,
+final case class WorkflowSourceFilesWithDependenciesZip(workflowSource: WorkflowSource,
+                                                        workflowType: Option[WorkflowType],
+                                                        workflowTypeVersion: Option[WorkflowTypeVersion],
+                                                        inputsJson: WorkflowJson,
                                                         workflowOptionsJson: WorkflowOptionsJson,
-                                                        labelsJson: WdlJson,
+                                                        labelsJson: WorkflowJson,
                                                         importsZip: Array[Byte]) extends WorkflowSourceFilesCollection {
-  override def toString = s"WorkflowSourceFilesWithDependenciesZip($wdlSource, $inputsJson, $workflowOptionsJson, $labelsJson, <<ZIP BINARY CONTENT>>)"
+  override def toString = s"WorkflowSourceFilesWithDependenciesZip($workflowSource, $inputsJson, $workflowOptionsJson, $labelsJson, <<ZIP BINARY CONTENT>>)"
 }
diff --git a/core/src/main/scala/cromwell/core/WorkflowState.scala b/core/src/main/scala/cromwell/core/WorkflowState.scala
index 41ac2bb97..98cef7896 100644
--- a/core/src/main/scala/cromwell/core/WorkflowState.scala
+++ b/core/src/main/scala/cromwell/core/WorkflowState.scala
@@ -10,9 +10,9 @@ sealed trait WorkflowState {
 }
 
 object WorkflowState {
-  private lazy val WorkflowState = Seq(WorkflowSubmitted, WorkflowRunning, WorkflowFailed, WorkflowSucceeded, WorkflowAborting, WorkflowAborted)
+  private lazy val WorkflowStateValues = Seq(WorkflowSubmitted, WorkflowRunning, WorkflowFailed, WorkflowSucceeded, WorkflowAborting, WorkflowAborted)
 
-  def fromString(str: String): WorkflowState = WorkflowState.find(_.toString == str).getOrElse(
+  def withName(str: String): WorkflowState = WorkflowStateValues.find(_.toString == str).getOrElse(
     throw new NoSuchElementException(s"No such WorkflowState: $str"))
 
   implicit val WorkflowStateSemigroup = new Semigroup[WorkflowState] {
diff --git a/core/src/main/scala/cromwell/core/actor/BatchingDbWriter.scala b/core/src/main/scala/cromwell/core/actor/BatchingDbWriter.scala
new file mode 100644
index 000000000..db5be437b
--- /dev/null
+++ b/core/src/main/scala/cromwell/core/actor/BatchingDbWriter.scala
@@ -0,0 +1,106 @@
+package cromwell.core.actor
+
+import akka.actor.{ActorRef, Cancellable, FSM}
+import cats.data.NonEmptyVector
+import cromwell.core.actor.BatchingDbWriter._
+import cromwell.util.GracefulShutdownHelper.ShutdownCommand
+import org.slf4j.LoggerFactory
+
+import scala.util.{Failure, Success, Try}
+
+
+/** A collection of state, data, and message types to support batched database writes. */
+object BatchingDbWriter {
+  val logger = LoggerFactory.getLogger("BatchingDbWriteActor")
+
+  /** Data for batched database writes. */
+  sealed trait BatchingDbWriterData {
+    def addData[D](datum: D): BatchingDbWriterData = addData(Vector(datum))
+    def addData[D](data: Iterable[D]): BatchingDbWriterData = {
+      Try(NonEmptyVector.fromVector(data.toVector)) match {
+        case Success(Some(v)) =>
+          val newEvents = this match {
+            case NoData => v
+            case HasData(e) => e.concatNev(v)
+          }
+          HasData(newEvents)
+        case Success(None) => this
+        case Failure(f) =>
+          val dataSample = data.take(3).mkString(", ") + (if (data.size > 3) ", ..." else "")
+          logger.error(s"Failed processing batched data: $dataSample. Data will be dropped and not be sent to the database.", f)
+          this
+      }
+    }
+
+    def length: Int = this match {
+      case NoData => 0
+      case HasData(e) => e.length
+    }
+  }
+
+  case object NoData extends BatchingDbWriterData
+  case class HasData[E](events: NonEmptyVector[E]) extends BatchingDbWriterData
+
+  /** The states for batched database writes. */
+  sealed trait BatchingDbWriterState
+  case object WaitingToWrite extends BatchingDbWriterState
+  case object WritingToDb extends BatchingDbWriterState
+
+  /** The message types for batched database writes. */
+  sealed trait BatchingDbWriterMessage
+  case object DbWriteComplete extends BatchingDbWriterMessage
+  case object FlushBatchToDb extends BatchingDbWriterMessage
+  case object ScheduledFlushToDb extends BatchingDbWriterMessage
+
+  case class CommandAndReplyTo[C](command: C, replyTo: ActorRef)
+}
+
+/**
+  * Trait that contains some common batch-related and graceful shutdown logic.
+  * Be careful NOT to add a custom whenUnhandled state function when mixing in this trait as it will override the 
+  * graceful shutdown handling logic.
+  * 
+  * Note that there is more common logic that could be abstracted here.
+  */
+trait BatchingDbWriterActor { this: FSM[BatchingDbWriterState, BatchingDbWriterData] =>
+  import scala.concurrent.duration._
+  
+  private var shuttingDown: Boolean = false
+  
+  def isShuttingDown: Boolean = shuttingDown
+  def dbFlushRate: FiniteDuration
+  var periodicFlush: Option[Cancellable] = None
+
+  override def preStart(): Unit = {
+    periodicFlush = Option(context.system.scheduler.schedule(0.seconds, dbFlushRate, self, ScheduledFlushToDb)(context.dispatcher))
+  }
+
+  /**
+    * WhenUnhandled state function that handles reception of ShutdownCommand and acts appropriately
+    */
+  private val whenUnhandledFunction: StateFunction = {
+    case Event(ShutdownCommand, NoData) if stateName == WaitingToWrite =>
+      periodicFlush foreach { _.cancel() }
+      context stop self
+      stay()
+    case Event(ShutdownCommand, _) if stateName == WaitingToWrite =>
+      logger.info("{} flushing database writes...", self.path.name)
+      shuttingDown = true
+      // transitioning to WritingToDb triggers a FlushBatchToDb to be sent to self
+      goto(WritingToDb)
+    case Event(ShutdownCommand, _) if stateName == WritingToDb =>
+      logger.info("{} waiting for database writes to be flushed...", self.path.name)
+      shuttingDown = true
+      stay()
+  }
+
+  whenUnhandled(whenUnhandledFunction)
+
+  onTransition {
+    case WaitingToWrite -> WritingToDb => self ! FlushBatchToDb
+    // When transitioning back to WaitingToWrite, if there's no data left to process, and we're trying to shutdown, then stop
+    case _ -> WaitingToWrite if shuttingDown && nextStateData == NoData =>
+      periodicFlush foreach { _.cancel() }
+      context stop self
+  }
+}
diff --git a/core/src/main/scala/cromwell/core/actor/RobustClientHelper.scala b/core/src/main/scala/cromwell/core/actor/RobustClientHelper.scala
new file mode 100644
index 000000000..ff413ee78
--- /dev/null
+++ b/core/src/main/scala/cromwell/core/actor/RobustClientHelper.scala
@@ -0,0 +1,74 @@
+package cromwell.core.actor
+
+import akka.actor.{Actor, ActorLogging, ActorRef, Cancellable}
+import cromwell.core.actor.RobustClientHelper._
+import cromwell.core.actor.StreamIntegration._
+
+import scala.concurrent.duration.{FiniteDuration, _}
+import scala.language.postfixOps
+import scala.util.Random
+
+object RobustClientHelper {
+  case class RequestTimeout(msg: Any, to: ActorRef)
+  val DefaultRequestLostTimeout = 5 minutes
+}
+
+trait RobustClientHelper { this: Actor with ActorLogging =>
+  private [actor] implicit val robustActorHelperEc = context.dispatcher
+
+  private final val random = new Random()
+  
+  // package private for testing
+  private [core] var timeouts = Map.empty[Any, (Cancellable, FiniteDuration)]
+  
+  protected def backpressureTimeout: FiniteDuration = 10 seconds
+  protected def backpressureRandomizerFactor: Double = 0.5D
+  
+ def robustReceive: Receive = {
+    case BackPressure(request) => 
+      val snd = sender()
+      newTimer(request, snd, generateBackpressureTime)
+      resetTimeout(request, snd)
+      ()
+    case RequestTimeout(request, to) => onTimeout(request, to)
+  }
+
+  private final def newTimer(msg: Any, to: ActorRef, in: FiniteDuration) = {
+    context.system.scheduler.scheduleOnce(in, to, msg)(robustActorHelperEc, self)
+  }
+  
+  def robustSend(msg: Any, to: ActorRef, timeout: FiniteDuration = DefaultRequestLostTimeout): Unit = {
+    to ! msg
+    addTimeout(msg, to, timeout)
+  }
+  
+  private final def addTimeout(command: Any, to: ActorRef, timeout: FiniteDuration) = {
+    val cancellable = newTimer(RequestTimeout(command, to), self, timeout)
+    timeouts = timeouts + (command -> (cancellable -> timeout))
+  }
+
+  protected final def hasTimeout(command: Any) = timeouts.get(command).isDefined
+
+  protected final def cancelTimeout(command: Any) = {
+    timeouts.get(command) foreach { case (cancellable, _) => cancellable.cancel() }
+    timeouts = timeouts - command
+  }
+
+  private final def resetTimeout(command: Any, to: ActorRef) = {
+    val timeout = timeouts.get(command) map { _._2 }
+    cancelTimeout(command)
+    timeout foreach { addTimeout(command, to, _) }
+  }
+  
+  private [actor] final def generateBackpressureTime = {
+    val backpressureTimeoutInMillis = backpressureTimeout.toMillis
+    
+    val delta = backpressureRandomizerFactor * backpressureTimeoutInMillis
+    val minInterval = backpressureTimeoutInMillis - delta
+    val maxInterval = backpressureTimeoutInMillis + delta
+    val randomValue = (minInterval + (random.nextDouble() * (maxInterval - minInterval + 1))).toInt
+    randomValue.milliseconds
+  }
+  
+  protected def onTimeout(message: Any, to: ActorRef): Unit
+}
diff --git a/core/src/main/scala/cromwell/core/actor/StreamActorHelper.scala b/core/src/main/scala/cromwell/core/actor/StreamActorHelper.scala
new file mode 100644
index 000000000..cb4dd6e67
--- /dev/null
+++ b/core/src/main/scala/cromwell/core/actor/StreamActorHelper.scala
@@ -0,0 +1,93 @@
+package cromwell.core.actor
+
+import akka.actor.{Actor, ActorLogging}
+import akka.pattern.pipe
+import akka.stream.{ActorAttributes, ActorMaterializer, QueueOfferResult, Supervision}
+import akka.stream.QueueOfferResult.{Dropped, Enqueued, QueueClosed}
+import akka.stream.scaladsl.{Sink, Source, SourceQueueWithComplete}
+import cromwell.core.actor.StreamActorHelper.{ActorRestartException, StreamCompleted, StreamFailed}
+import cromwell.core.actor.StreamIntegration._
+import cromwell.util.GracefulShutdownHelper.ShutdownCommand
+
+import scala.concurrent.{ExecutionContext, Future}
+import scala.util.{Failure, Success}
+
+object StreamActorHelper {
+  private [actor] case class StreamFailed(failure: Throwable)
+  private [actor] case object StreamCompleted
+  class ActorRestartException(throwable: Throwable) extends RuntimeException(throwable)
+}
+
+trait StreamActorHelper[T <: StreamContext] { this: Actor with ActorLogging =>
+
+  implicit def ec: ExecutionContext
+
+  implicit def materializer: ActorMaterializer
+
+  private val decider: Supervision.Decider = _ => Supervision.Resume
+  
+  private val replySink = Sink.foreach[(Any, T)] {
+    case (response, commandContext) =>
+      val reply = commandContext.clientContext map { (_, response) } getOrElse response
+      commandContext.replyTo ! reply
+  }
+
+  protected def actorReceive: Receive
+  
+  protected def streamSource: Source[(Any, T), SourceQueueWithComplete[T]]
+
+  override def receive = streamReceive.orElse(actorReceive)
+
+  private [actor] lazy val stream = {
+    streamSource
+      .to(replySink)
+      .withAttributes(ActorAttributes.supervisionStrategy(decider))
+      .run()
+  }
+
+  override def preStart(): Unit = {
+    stream.watchCompletion() onComplete {
+      case Success(_) => self ! StreamCompleted
+      case Failure(failure) => self ! StreamFailed(failure)
+    }
+  }
+
+  def sendToStream(commandContext: T) = {
+    val enqueue = stream offer commandContext map {
+      case Enqueued => EnqueueResponse(Enqueued, commandContext)
+      case other => EnqueueResponse(other, commandContext)
+    } recoverWith {
+      case t => Future.successful(FailedToEnqueue(t, commandContext))
+    }
+
+    pipe(enqueue) to self
+    ()
+  }
+  
+  private def backpressure(commandContext: StreamContext) = {
+    val originalRequest = commandContext.clientContext map { _ -> commandContext.request } getOrElse commandContext.request
+    commandContext.replyTo ! BackPressure(originalRequest)
+  }
+
+  private def streamReceive: Receive = {
+    case ShutdownCommand => stream.complete()
+    case EnqueueResponse(Enqueued, _: T @unchecked) => // Good !
+    case EnqueueResponse(Dropped, commandContext) => backpressure(commandContext)
+      
+      // In any of the cases below, the stream is in a failed state, which will be caught by the watchCompletion hook and the
+      // actor will be restarted
+    case EnqueueResponse(QueueClosed, commandContext) => backpressure(commandContext)
+    case EnqueueResponse(QueueOfferResult.Failure(_), commandContext) => backpressure(commandContext)
+    case FailedToEnqueue(_, commandContext) => backpressure(commandContext)
+      
+    case StreamCompleted => context stop self
+    case StreamFailed(failure) => restart(failure)
+  }
+
+  /** Throw the exception to force the actor to restart so it can be back in business
+    * IMPORTANT: Make sure the supervision strategy for this actor is Restart
+    */
+  private def restart(throwable: Throwable) = {
+    throw new ActorRestartException(throwable)
+  }
+}
diff --git a/core/src/main/scala/cromwell/core/actor/StreamIntegration.scala b/core/src/main/scala/cromwell/core/actor/StreamIntegration.scala
new file mode 100644
index 000000000..6b7b51371
--- /dev/null
+++ b/core/src/main/scala/cromwell/core/actor/StreamIntegration.scala
@@ -0,0 +1,15 @@
+package cromwell.core.actor
+
+import akka.actor.ActorRef
+import akka.stream.QueueOfferResult
+
+object StreamIntegration {
+  trait StreamContext {
+    def replyTo: ActorRef
+    def request: Any
+    def clientContext: Option[Any] = None
+  }
+  case class EnqueueResponse(response: QueueOfferResult, request: StreamContext)
+  case class BackPressure(request: Any)
+  case class FailedToEnqueue(failure: Throwable, request: StreamContext)
+}
diff --git a/core/src/main/scala/cromwell/core/callcaching/CallCachingEligibility.scala b/core/src/main/scala/cromwell/core/callcaching/CallCachingEligibility.scala
deleted file mode 100644
index f39d82e19..000000000
--- a/core/src/main/scala/cromwell/core/callcaching/CallCachingEligibility.scala
+++ /dev/null
@@ -1,22 +0,0 @@
-package cromwell.core.callcaching
-
-sealed trait CallCachingEligibility
-case object CallCachingEligible extends CallCachingEligibility
-sealed trait CallCachingIneligible extends CallCachingEligibility {
-  def message: String
-}
-  
-case class FloatingDockerTagWithHash(hash: String) extends CallCachingIneligible {
-  override val message = s"""You are using a floating docker tag in this task. Cromwell does not consider tasks with floating tags to be eligible for call caching.
-        |If you want this task to be eligible for call caching in the future, use a docker runtime attribute with a digest instead.
-        |This is the exact docker image that was used for this job: $hash
-        |You can replace the docker runtime attribute in your task with the above value to make this task eligible for call caching.""".stripMargin
-}
-
-case object FloatingDockerTagWithoutHash extends CallCachingIneligible {
-  override val message = s"""You are using a floating docker tag in this task. Cromwell does not consider tasks with floating tags to be eligible for call caching.
-         |If you want this task to be eligible for call caching in the future, use a docker runtime attribute with a digest instead.
-         |Cromwell attempted to retrieve the current hash for this docker image but failed.
-         |This is not necessarily a cause for concern as Cromwell is currently only able to retrieve hashes for Dockerhub and GCR images.
-         |The job will be dispatched to the appropriate backend that will attempt to run it.""".stripMargin
-}
diff --git a/core/src/main/scala/cromwell/core/callcaching/HashResultMessage.scala b/core/src/main/scala/cromwell/core/callcaching/HashResultMessage.scala
index 581326c16..9233061cd 100644
--- a/core/src/main/scala/cromwell/core/callcaching/HashResultMessage.scala
+++ b/core/src/main/scala/cromwell/core/callcaching/HashResultMessage.scala
@@ -1,6 +1,14 @@
 package cromwell.core.callcaching
 
-case class HashKey(key: String, checkForHitOrMiss: Boolean = true)
+
+object HashKey {
+  def apply(keyComponents: String*) = new HashKey(true, keyComponents.toList)
+  def apply(checkForHitOrMiss: Boolean, keyComponents: String*) = new HashKey(checkForHitOrMiss, keyComponents.toList)
+}
+
+case class HashKey(checkForHitOrMiss: Boolean, keyComponents: List[String]) {
+  val key = keyComponents.mkString(": ")
+}
 case class HashValue(value: String)
 case class HashResult(hashKey: HashKey, hashValue: HashValue)
 
@@ -8,4 +16,6 @@ sealed trait HashResultMessage
 trait SuccessfulHashResultMessage extends HashResultMessage {
   def hashes: Set[HashResult]
 }
-case class HashingFailedMessage(key: HashKey, reason: Throwable) extends HashResultMessage
\ No newline at end of file
+case class MultiHashingFailedMessage(keys: Set[HashKey], reason: Throwable) extends HashResultMessage
+case class HashingFailedMessage(file: String, reason: Throwable) extends HashResultMessage
+case object HashingServiceUnvailable extends HashResultMessage
diff --git a/core/src/main/scala/cromwell/core/callcaching/MaybeCallCachingEligible.scala b/core/src/main/scala/cromwell/core/callcaching/MaybeCallCachingEligible.scala
new file mode 100644
index 000000000..c2460e1b8
--- /dev/null
+++ b/core/src/main/scala/cromwell/core/callcaching/MaybeCallCachingEligible.scala
@@ -0,0 +1,19 @@
+package cromwell.core.callcaching
+
+sealed trait MaybeCallCachingEligible {
+  def dockerHash: Option[String]
+}
+
+sealed trait CallCachingEligible extends MaybeCallCachingEligible
+sealed trait CallCachingIneligible extends MaybeCallCachingEligible
+
+case object NoDocker extends CallCachingEligible {
+  override def dockerHash: Option[String] = None
+}
+case class DockerWithHash(dockerAttribute: String) extends CallCachingEligible {
+  override def dockerHash: Option[String] = Option(dockerAttribute)
+}
+
+case class FloatingDockerTagWithoutHash(dockerTag: String) extends CallCachingIneligible {
+  override def dockerHash: Option[String] = None
+}
diff --git a/core/src/main/scala/cromwell/core/callcaching/docker/DockerHashResult.scala b/core/src/main/scala/cromwell/core/callcaching/docker/DockerHashResult.scala
deleted file mode 100644
index fbfd18de9..000000000
--- a/core/src/main/scala/cromwell/core/callcaching/docker/DockerHashResult.scala
+++ /dev/null
@@ -1,17 +0,0 @@
-package cromwell.core.callcaching.docker
-
-object DockerHashResult {
-  // See https://docs.docker.com/registry/spec/api/#/content-digests
-  val DigestRegex = """([a-zA-Z0-9_+.-]+):([a-zA-Z0-9]+)""".r
-  
-  def apply(str: String): DockerHashResult = {
-    str match {
-      case DigestRegex(alg, hash) => new DockerHashResult(alg, hash)
-      case _ => throw new IllegalArgumentException(s"Hash value $str does not have the expected 'algorithm:hash' syntax")
-    }
-  }
-}
-
-case class DockerHashResult(hashAlgorithm: String, hashValue: String) {
-  val algorithmAndHash = s"$hashAlgorithm:$hashValue"
-}
diff --git a/core/src/main/scala/cromwell/core/callcaching/docker/DockerImageIdentifier.scala b/core/src/main/scala/cromwell/core/callcaching/docker/DockerImageIdentifier.scala
deleted file mode 100644
index 9e40ab15d..000000000
--- a/core/src/main/scala/cromwell/core/callcaching/docker/DockerImageIdentifier.scala
+++ /dev/null
@@ -1,83 +0,0 @@
-package cromwell.core.callcaching.docker
-
-import scala.util.{Failure, Success, Try}
-
-sealed trait DockerImageIdentifier {
-  def host: Option[String]
-  def repository: String
-  def image: String
-  def reference: String
-
-  lazy val name = s"$repository/$image"
-  lazy val hostAsString = host map { h => s"$h/" } getOrElse ""
-  lazy val fullName = s"$hostAsString$repository/$image:$reference"
-}
-
-case class DockerImageIdentifierWithoutHash(host: Option[String], repository: String, image: String, reference: String) extends DockerImageIdentifier {
-  def withHash(hash: DockerHashResult) =  DockerImageIdentifierWithHash(host, repository, image, reference, hash)
-}
-
-case class DockerImageIdentifierWithHash(host: Option[String], repository: String, image: String, reference: String, hash: DockerHashResult) extends DockerImageIdentifier {
-  override lazy val fullName: String = s"$hostAsString$repository/$image@${hash.algorithmAndHash}"
-}
-
-object DockerImageIdentifier {
-  // See https://github.com/docker-library/official-images/tree/master/library
-  private val DefaultDockerRepo = "library"
-  private val DefaultDockerTag = "latest"
-  private val TagSeparator = ":"
-  private val DigestSeparator = "@"
-  
-  private val DockerStringRegex =
-    s"""
-       (?x)                                     # Turn on comments and whitespace insensitivity
-       
-       (                                        # Begin capturing group for name
-          [a-z0-9]+(?:[._-][a-z0-9]+)*          # API v2 name component regex - see https://docs.docker.com/registry/spec/api/#/overview
-          (?:/[a-z0-9]+(?:[._-][a-z0-9]+)*)*    # Optional additional name components separated by /
-       )                                        # End capturing group for name
-       
-       (?:   
-          (                                     # Begin capturing group for tag separator
-            :|@                                 # Tag separator. ':' is followed by a tag, '@' is followed by a digest
-          )                                     # End capturing group for tag separator
-       
-          (                                     # Begin capturing group for reference 
-            [A-Za-z0-9]+(?:[.:_-][A-Za-z0-9]+)* # Reference
-          )                                     # End capturing group for reference  
-       )?                                      
-       """.trim.r
-  
-  def fromString(dockerString: String): Try[DockerImageIdentifier] = {
-    dockerString match {
-        // Just a name means latest tag implicitly
-      case DockerStringRegex(name, null, null) => buildId(name, TagSeparator,  DefaultDockerTag)
-      case DockerStringRegex(name, tagSeparator, reference) => buildId(name, tagSeparator, reference)
-      case _ => Failure(new IllegalArgumentException(s"Docker image $dockerString has an invalid syntax."))
-    }
-  }
-  
-  private def isRegistryHostName(str: String) = str.contains('.')
-  
-  private def buildId(name: String, tagSeparator: String, reference: String) = {
-    val (dockerHost, dockerRepo, dockerImage) = name.split('/').toList match {
-      // If just one component (e.g ubuntu), assume default repo
-      case image :: Nil => (None, DefaultDockerRepo, image)
-      // If repo/image (e.g broadinstitute/cromwell) without host
-      case repo :: image :: Nil if !isRegistryHostName(repo) => (None, repo, image)
-      // If host/image (e.g index.docker.io/ubuntu), assume default repo
-      case host :: image :: Nil if isRegistryHostName(host) => (Option(host), DefaultDockerRepo, image)
-      // Not a host followed more than one components
-      case nothost :: rest if !isRegistryHostName(nothost) => (None, s"$nothost/${rest.init.mkString("/")}", rest.last)
-      // A host followed more than one components (e.g gcr.io/google-containers/alpine-with-bash)
-      case host :: rest if isRegistryHostName(host) => (Option(host), rest.init.mkString("/"), rest.last)
-    }
-    
-    tagSeparator match {
-      case DigestSeparator => Try(DockerHashResult(reference)) map { hash => DockerImageIdentifierWithHash(dockerHost, dockerRepo, dockerImage, reference, hash) }
-      case TagSeparator => Success(DockerImageIdentifierWithoutHash(dockerHost, dockerRepo, dockerImage, reference))
-        // Should have been caught by the regex, but in case..
-      case other => Failure(new IllegalArgumentException(s"Invalid separator between image and tag in docker attribute: $other"))
-    }
-  }
-}
diff --git a/core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/HttpFlowWithRetry.scala b/core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/HttpFlowWithRetry.scala
deleted file mode 100644
index 6509d1027..000000000
--- a/core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/HttpFlowWithRetry.scala
+++ /dev/null
@@ -1,120 +0,0 @@
-package cromwell.core.callcaching.docker.registryv2.flows
-
-import akka.NotUsed
-import akka.http.scaladsl.model.{HttpRequest, HttpResponse, StatusCodes}
-import akka.stream.scaladsl.{Flow, GraphDSL, Merge, Partition}
-import akka.stream.{FanOutShape2, OverflowStrategy}
-import cromwell.core.callcaching.docker.registryv2.flows.FlowUtils._
-import cromwell.core.callcaching.docker.registryv2.flows.HttpFlowWithRetry._
-
-import scala.util.Try
-
-object HttpFlowWithRetry {
-  def defaultIsRetryable(response: HttpResponse) = {
-    response.status match {
-      case StatusCodes.TooManyRequests => true
-      case StatusCodes.InternalServerError => true
-      case _ => false
-    }
-  }
-
-  /**
-    * In order to allow for retries, the http context object needs to encapsulate the original request,
-    * so that it can be re-submitted if necessary.
-    * This type provides this encapsulation by creating a pair of (T, HttpRequest) where T is the custom type that
-    * the user wishes to use as a context for their specific use case.
-    */
-  type ContextWithRequest[T] = (T, HttpRequest)
-
-  /**
-    * Type for a http flow using ContextWithRequest as context type.
-    */
-  type RetryableHttpFlow[T] = Flow[(HttpRequest, ContextWithRequest[T]), (Try[HttpResponse], ContextWithRequest[T]), NotUsed]
-}
-
-/**
-  * Adds additional retry logic around an existing akka http flow.
-  * @param httpClientFlow The akka http flow to use underneath
-  * @param retryBufferSize size of the buffer for requests to be retried. When the buffer is full, requests WILL BE DROPPED
-  *                        Act accordingly by resubmitting requests after a certain amount of time with no response for example.
-  * @param isRetryable function to determine weather or not an http request should be retried based on the http response                        
-  * @tparam T Type of the context data to be passed along with the request
-  * @return A http flow that will retry on request failures and on failed responses as seen fit by the isRetryable method
-  */
-case class HttpFlowWithRetry[T](
-                            httpClientFlow: RetryableHttpFlow[T],
-                            retryBufferSize: Int = 1000,
-                            isRetryable: HttpResponse => Boolean = defaultIsRetryable
-                          ) {
-
-  lazy val flow = GraphDSL.create() { implicit builder =>
-    import GraphDSL.Implicits._
-
-    // Http shape
-    val http = builder.add(httpClientFlow)
-    
-    // Wrap the user context into a pair of (flowContext, httpRequest) to allow for retries
-    val source = builder.add(Flow[(HttpRequest, T)] map {
-      case (httpRequest, flowContext) => (httpRequest, (flowContext, httpRequest))
-    })
-    
-    // Partition Try[HttpResponse] into Success and Failure
-    val partitionResponseTry = builder.add(fanOutTry[HttpResponse, (T, HttpRequest)])
-    
-    val requestSuccessful = partitionResponseTry.out0
-    // Extract the request from the failed try so it can be resubmitted
-    val requestFailed = partitionResponseTry.out1.log("request failed") map toRetryableRequest
-    
-    // Merges requests coming from 3 input ports:
-    // - in(0) takes requests from the outside
-    // - in(1) takes retryable failed requests
-    // - in(2) takes retryable failed responses (with non successful and retryable status code)
-    val mergeRequests = builder.add(Merge[(HttpRequest, (T, HttpRequest))](3))
-    
-    // Splits http response into 3 categories:
-    // - out(0) emits successful responses
-    // - out(1) emits failed but retryable responses
-    // - out(2) emits failed, non-retryable responses
-    val partitionHttpResponse = builder.add(Partition[(HttpResponse, (T, HttpRequest))](3, {
-      // Successful return code
-      case (httpResponse, flowContext) if httpResponse.status.isSuccess() => 0
-      // Failed return code but retryable
-      case (httpResponse, flowContext) if isRetryable(httpResponse) => 1
-      // Failed return code and non retryable
-      case (httpResponse, flowContext) => 2
-    }))
-
-    val responseSuccessful = partitionHttpResponse.out(0) map toTerminalResponse
-    val responseRetryable = partitionHttpResponse.out(1) map toRetryableRequest
-    val responseFailed = partitionHttpResponse.out(2) map toTerminalResponse
-    
-    // Buffer for retryable requests. Will DROP requests when full
-    // Will also delay re submission
-    val retryBuffer = Flow[(HttpRequest, (T, HttpRequest))]
-      .buffer(retryBufferSize, OverflowStrategy.dropHead)
-
-    // Submit outside requests to the first port of the mergeRequests merger 
-    source ~> mergeRequests.in(0)
-    
-    // Submit request to underlying http flow  -  Partition responses: Failure | Success
-    mergeRequests     ~>     http      ~>         partitionResponseTry.in
-                                                  // Success -> analyze response and partition into 3 (see above)
-                                                  requestSuccessful ~> partitionHttpResponse
-    // Retry failed requests (Try[HttpResponse] was a Failure)
-    mergeRequests.in(1)     <~ retryBuffer <~     requestFailed.outlet
-    // Retry retryable failed responses (the HttpResponse had a non successful return code and was deemed retryable)
-    mergeRequests.in(2)        <~        retryBuffer         <~        responseRetryable.outlet
-
-    new FanOutShape2(source.in, responseSuccessful.outlet, responseFailed.outlet)
-  }
-
-  // create a re-submittable request from a failed retryable
-  private def toRetryableRequest(value: (Any, (T, HttpRequest))) = value match {
-    case (_, (flowContext, request)) => (request, (flowContext, request))
-  }
-
-  // extract only the response and the context so it can be emitted through the output port
-  private def toTerminalResponse(value: (HttpResponse, (T, HttpRequest))) = value match {
-    case (response, (flowContext, _)) => (response, flowContext)
-  }
-}
diff --git a/core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/gcr/GcrAbstractFlow.scala b/core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/gcr/GcrAbstractFlow.scala
deleted file mode 100644
index 09e946412..000000000
--- a/core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/gcr/GcrAbstractFlow.scala
+++ /dev/null
@@ -1,35 +0,0 @@
-package cromwell.core.callcaching.docker.registryv2.flows.gcr
-
-import akka.http.scaladsl.model.headers.{Authorization, OAuth2BearerToken}
-import akka.stream.ActorMaterializer
-import com.google.api.client.auth.oauth2.Credential
-import cromwell.core.callcaching.docker.DockerHashActor.DockerHashContext
-import cromwell.core.callcaching.docker.registryv2.DockerRegistryV2AbstractFlow
-import cromwell.core.callcaching.docker.registryv2.DockerRegistryV2AbstractFlow.HttpDockerFlow
-
-import scala.concurrent.ExecutionContext
-import scala.concurrent.duration._
-
-abstract class GcrAbstractFlow(httpClientFlow: HttpDockerFlow, host: String)(implicit ec: ExecutionContext, materializer: ActorMaterializer) extends DockerRegistryV2AbstractFlow(httpClientFlow)(ec, materializer) {
-  
-  private val AccessTokenAcceptableTTL = 1.minute.toSeconds
-  
-  override val registryHostName = host
-  override val authorizationServerHostName = s"$host/v2"
-  
-  /**
-    * Builds the list of headers for the token request
-    */
-   def buildTokenRequestHeaders(dockerHashContext: DockerHashContext) = {
-    dockerHashContext.credentials collect {
-      case credential: Credential => Authorization(OAuth2BearerToken(freshAccessToken(credential)))
-    }
-  }
-  
-  private def freshAccessToken(credential: Credential) = {
-    if (credential.getExpiresInSeconds < AccessTokenAcceptableTTL) {
-      credential.refreshToken()
-    }
-    credential.getAccessToken
-  }
-}
diff --git a/core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/gcr/GcrEuFlow.scala b/core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/gcr/GcrEuFlow.scala
deleted file mode 100644
index de7856c60..000000000
--- a/core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/gcr/GcrEuFlow.scala
+++ /dev/null
@@ -1,8 +0,0 @@
-package cromwell.core.callcaching.docker.registryv2.flows.gcr
-
-import akka.stream.ActorMaterializer
-import cromwell.core.callcaching.docker.registryv2.DockerRegistryV2AbstractFlow.HttpDockerFlow
-
-import scala.concurrent.ExecutionContext
-
-private class GcrEuFlow(httpClientFlow: HttpDockerFlow)(implicit ec: ExecutionContext, materializer: ActorMaterializer) extends GcrAbstractFlow(httpClientFlow, "eu.gcr.io")
diff --git a/core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/gcr/GcrFlow.scala b/core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/gcr/GcrFlow.scala
deleted file mode 100644
index 4bc27a4d0..000000000
--- a/core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/gcr/GcrFlow.scala
+++ /dev/null
@@ -1,8 +0,0 @@
-package cromwell.core.callcaching.docker.registryv2.flows.gcr
-
-import akka.stream.ActorMaterializer
-import cromwell.core.callcaching.docker.registryv2.DockerRegistryV2AbstractFlow.HttpDockerFlow
-
-import scala.concurrent.ExecutionContext
-
-private class GcrFlow(httpClientFlow: HttpDockerFlow)(implicit ec: ExecutionContext, materializer: ActorMaterializer) extends GcrAbstractFlow(httpClientFlow, "gcr.io")
diff --git a/core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/gcr/GcrUsFlow.scala b/core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/gcr/GcrUsFlow.scala
deleted file mode 100644
index 57a93f753..000000000
--- a/core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/gcr/GcrUsFlow.scala
+++ /dev/null
@@ -1,8 +0,0 @@
-package cromwell.core.callcaching.docker.registryv2.flows.gcr
-
-import akka.stream.ActorMaterializer
-import cromwell.core.callcaching.docker.registryv2.DockerRegistryV2AbstractFlow.HttpDockerFlow
-
-import scala.concurrent.ExecutionContext
-
-private class GcrUsFlow(httpClientFlow: HttpDockerFlow)(implicit ec: ExecutionContext, materializer: ActorMaterializer) extends GcrAbstractFlow(httpClientFlow, "us.gcr.io")
diff --git a/core/src/main/scala/cromwell/core/core.scala b/core/src/main/scala/cromwell/core/core.scala
index 8cfe7c2f1..7a5831bbe 100644
--- a/core/src/main/scala/cromwell/core/core.scala
+++ b/core/src/main/scala/cromwell/core/core.scala
@@ -2,7 +2,7 @@ package cromwell.core
 
 import cromwell.core.path.Path
 import lenthall.exception.ThrowableAggregation
-import wdl4s.values.WdlValue
+import wdl4s.wdl.values.WdlValue
 
 
 case class CallContext(root: Path, stdout: String, stderr: String)
diff --git a/core/src/main/scala/cromwell/core/io/AsyncIo.scala b/core/src/main/scala/cromwell/core/io/AsyncIo.scala
new file mode 100644
index 000000000..37537bdba
--- /dev/null
+++ b/core/src/main/scala/cromwell/core/io/AsyncIo.scala
@@ -0,0 +1,78 @@
+package cromwell.core.io
+
+import akka.actor.{Actor, ActorLogging, ActorRef}
+import cromwell.core.path.BetterFileMethods.OpenOptions
+import cromwell.core.path.Path
+
+import scala.concurrent.duration._
+import scala.concurrent.{Future, Promise}
+import scala.language.postfixOps
+
+
+trait AsyncIo extends IoClientHelper { this: Actor with ActorLogging with IoCommandBuilder =>
+  
+  protected val ioTimeout = 3 minutes
+  
+  override private [core] def ioResponseReceive: Receive = {
+    case (promise: Promise[_], ack: IoAck[Any] @unchecked) =>
+      cancelTimeout(promise -> ack.command)
+      // This is not typesafe. 
+      // However the sendIoCommand method ensures that the command and the promise have the same generic type
+      // Which means as long as only the sendIoCommand method is used to send requests, and the ioActor honors his contract
+      // and send back the right context with the right response, the types are virtually guaranteed to match.
+      promise.asInstanceOf[Promise[Any]].complete(ack.toTry)
+      ()
+  }
+  
+  /**
+    * IMPORTANT: This loads the entire content of the file into memory !
+    * Only use for small files !
+    */
+  def contentAsStringAsync(path: Path): Future[String] = {
+    val promise = Promise[String]
+    sendIoCommandWithPromise(contentAsStringCommand(path), promise)
+    promise.future
+  }
+  
+  def writeAsync(path: Path, content: String, options: OpenOptions): Future[Unit] = {
+    val promise = Promise[Unit]
+    sendIoCommandWithPromise(writeCommand(path, content, options), promise)
+    promise.future
+  }
+
+  def sizeAsync(path: Path): Future[Long] = {
+    val promise = Promise[Long]
+    sendIoCommandWithPromise(sizeCommand(path), promise)
+    promise.future
+  }
+
+  def hashAsync(path: Path): Future[String] = {
+    val promise = Promise[String]
+    sendIoCommandWithPromise(hashCommand(path), promise)
+    promise.future
+  }
+
+  def deleteAsync(path: Path, swallowIoExceptions: Boolean = false): Future[Unit] = {
+    val promise = Promise[Unit]
+    sendIoCommandWithPromise(deleteCommand(path, swallowIoExceptions), promise)
+    promise.future
+  }
+
+  def copyAsync(src: Path, dest: Path, overwrite: Boolean = true): Future[Unit] = {
+    val promise = Promise[Unit]
+    // Allow for a much larger timeout for copies, as large files can take a while (even on gcs, if they are in different locations...)
+    sendIoCommandWithPromise(copyCommand(src, dest, overwrite), promise, 1 hour)
+    promise.future
+  }
+
+  private def sendIoCommandWithPromise[T](command: IoCommand[T], promise: Promise[T], timeout: FiniteDuration = ioTimeout) = {
+    sendIoCommandWithContext(command, promise, timeout)
+  }
+
+  override def onTimeout(message: Any, to: ActorRef): Unit = message match {
+    case (promise: Promise[_], ioAck: IoAck[_]) => 
+      promise.tryFailure(IoTimeout(ioAck.command))
+      ()
+    case _ =>
+  }
+}
diff --git a/core/src/main/scala/cromwell/core/io/DefaultIoCommand.scala b/core/src/main/scala/cromwell/core/io/DefaultIoCommand.scala
new file mode 100644
index 000000000..a05961b4a
--- /dev/null
+++ b/core/src/main/scala/cromwell/core/io/DefaultIoCommand.scala
@@ -0,0 +1,25 @@
+package cromwell.core.io
+
+import better.files.File.OpenOptions
+import cromwell.core.path.Path
+
+object DefaultIoCommand {
+  case class DefaultIoCopyCommand(override val source: Path,
+                                  override val destination: Path,
+                                  override val overwrite: Boolean) extends IoCopyCommand(
+    source, destination, overwrite
+  )
+  case class DefaultIoContentAsStringCommand(override val file: Path) extends IoContentAsStringCommand(file)
+  case class DefaultIoSizeCommand(override val file: Path) extends IoSizeCommand(file)
+  case class DefaultIoWriteCommand(override val file: Path,
+                                   override val content: String,
+                                   override val openOptions: OpenOptions) extends IoWriteCommand(
+    file, content, openOptions
+  )
+  case class DefaultIoDeleteCommand(override val file: Path,
+                                    override val swallowIOExceptions: Boolean) extends IoDeleteCommand(
+    file, swallowIOExceptions
+  )
+  case class DefaultIoHashCommand(override val file: Path) extends IoHashCommand(file)
+  case class DefaultIoTouchCommand(override val file: Path) extends IoTouchCommand(file)
+}
diff --git a/core/src/main/scala/cromwell/core/io/IoAck.scala b/core/src/main/scala/cromwell/core/io/IoAck.scala
new file mode 100644
index 000000000..c2b48a031
--- /dev/null
+++ b/core/src/main/scala/cromwell/core/io/IoAck.scala
@@ -0,0 +1,27 @@
+package cromwell.core.io
+
+import scala.util.{Failure, Success, Try}
+
+/**
+  * Generic trait for values returned after a command is executed. Can be Success or Failure.
+  *
+  * @tparam T type of the returned value if success
+  */
+sealed trait IoAck[T] {
+  /**
+    * Original command
+    */
+  def command: IoCommand[T]
+  def toTry: Try[T]
+}
+
+case class IoSuccess[T](command: IoCommand[T], result: T) extends IoAck[T] {
+  override def toTry = Success(result)
+}
+case class IoFailure[T](command: IoCommand[T], failure: Throwable) extends IoAck[T] {
+  override def toTry = Failure(failure)
+}
+
+case class IoRetry[T](command: IoCommand[T], failure: Throwable) extends IoAck[T] {
+  override def toTry = Failure(failure)
+}
diff --git a/core/src/main/scala/cromwell/core/io/IoClientHelper.scala b/core/src/main/scala/cromwell/core/io/IoClientHelper.scala
new file mode 100644
index 000000000..b6f13b50e
--- /dev/null
+++ b/core/src/main/scala/cromwell/core/io/IoClientHelper.scala
@@ -0,0 +1,33 @@
+package cromwell.core.io
+
+import akka.actor.{Actor, ActorLogging, ActorRef}
+import cromwell.core.actor.RobustClientHelper
+
+import scala.concurrent.duration.FiniteDuration
+
+trait IoClientHelper extends RobustClientHelper { this: Actor with ActorLogging with IoCommandBuilder =>
+  def ioActor: ActorRef
+
+  private [core] def ioResponseReceive: Receive = {
+    case ack: IoAck[_] if hasTimeout(ack.command) =>
+      cancelTimeout(ack.command)
+      receive.apply(ack)
+    case (context: Any, ack: IoAck[_]) if hasTimeout(context -> ack.command) =>
+      cancelTimeout(context -> ack.command)
+      receive.apply(context -> ack)
+  }
+  
+  def ioReceive = robustReceive orElse ioResponseReceive
+  
+  def sendIoCommand(ioCommand: IoCommand[_]) = {
+    sendIoCommandWithCustomTimeout(ioCommand, RobustClientHelper.DefaultRequestLostTimeout)
+  }
+
+  def sendIoCommandWithCustomTimeout(ioCommand: IoCommand[_], timeout: FiniteDuration) = {
+    robustSend(ioCommand, ioActor, timeout)
+  }
+
+  def sendIoCommandWithContext[T](ioCommand: IoCommand[_], context: T, timeout: FiniteDuration = RobustClientHelper.DefaultRequestLostTimeout) = {
+    robustSend(context -> ioCommand, ioActor, timeout)
+  }
+}
diff --git a/core/src/main/scala/cromwell/core/io/IoCommand.scala b/core/src/main/scala/cromwell/core/io/IoCommand.scala
new file mode 100644
index 000000000..fcc2d72a3
--- /dev/null
+++ b/core/src/main/scala/cromwell/core/io/IoCommand.scala
@@ -0,0 +1,87 @@
+package cromwell.core.io
+
+import better.files.File.OpenOptions
+import com.google.api.client.util.ExponentialBackOff
+import cromwell.core.path.Path
+import cromwell.core.retry.SimpleExponentialBackoff
+
+import scala.concurrent.duration.{FiniteDuration, _}
+import scala.language.postfixOps
+
+object IoCommand {
+  def defaultGoogleBackoff = new ExponentialBackOff.Builder()
+    .setInitialIntervalMillis((1 second).toMillis.toInt)
+    .setMaxIntervalMillis((5 minutes).toMillis.toInt)
+    .setMultiplier(3L)
+    .setRandomizationFactor(0.2D)
+    .setMaxElapsedTimeMillis((10 minutes).toMillis.toInt)
+    .build()
+  
+  def defaultBackoff = SimpleExponentialBackoff(defaultGoogleBackoff)
+  
+  type RetryCommand[T] = (FiniteDuration, IoCommand[T])
+}
+
+trait IoCommand[+T] {
+  /**
+    * Completes the command successfully
+    * @return a message to be sent back to the sender, if needed
+    */
+  def success[S >: T](value: S): IoSuccess[S] = IoSuccess(this, value)
+  
+  /**
+    * Fail the command with an exception
+    */
+  def fail[S >: T](failure: Throwable): IoFailure[S] = IoFailure(this, failure)
+}
+
+/**
+  * Copy source -> destination
+  * Will create the destination directory if it doesn't exist.
+  */
+abstract class IoCopyCommand(val source: Path, val destination: Path, val overwrite: Boolean) extends IoCommand[Unit] {
+  override def toString = s"copy ${source.pathAsString} to ${destination.pathAsString} with overwrite = $overwrite"
+}
+
+/**
+  * Read file as a string (load the entire content in memory)
+  */
+abstract class IoContentAsStringCommand(val file: Path) extends IoCommand[String] {
+  override def toString = s"read content of ${file.pathAsString}"
+}
+
+/**
+  * Return the size of file
+  */
+abstract class IoSizeCommand(val file: Path) extends IoCommand[Long] {
+  override def toString = s"get size of ${file.pathAsString}"
+}
+
+/**
+  * Write content in file
+  * Will create the destination directory if it doesn't exist.
+  */
+abstract class IoWriteCommand(val file: Path, val content: String, val openOptions: OpenOptions) extends IoCommand[Unit] {
+  override def toString = s"write to ${file.pathAsString}"
+}
+
+/**
+  * Delete file
+  */
+abstract class IoDeleteCommand(val file: Path, val swallowIOExceptions: Boolean) extends IoCommand[Unit] {
+  override def toString = s"delete ${file.pathAsString}"
+}
+
+/**
+  * Get Hash value for file
+  */
+abstract class IoHashCommand(val file: Path) extends IoCommand[String] {
+  override def toString = s"get hash of ${file.pathAsString}"
+}
+
+/**
+  * Touch a file
+  */
+abstract class IoTouchCommand(val file: Path) extends IoCommand[Unit] {
+  override def toString = s"touch ${file.pathAsString}"
+}
diff --git a/core/src/main/scala/cromwell/core/io/IoCommandBuilder.scala b/core/src/main/scala/cromwell/core/io/IoCommandBuilder.scala
new file mode 100644
index 000000000..ad26449b4
--- /dev/null
+++ b/core/src/main/scala/cromwell/core/io/IoCommandBuilder.scala
@@ -0,0 +1,25 @@
+package cromwell.core.io
+
+import cromwell.core.io.DefaultIoCommand._
+import cromwell.core.path.BetterFileMethods.OpenOptions
+import cromwell.core.path.Path
+
+trait IoCommandBuilder {
+  def contentAsStringCommand(path: Path): IoContentAsStringCommand
+  def writeCommand(path: Path, content: String, options: OpenOptions): IoWriteCommand
+  def sizeCommand(path: Path): IoSizeCommand
+  def deleteCommand(path: Path, swallowIoExceptions: Boolean): IoDeleteCommand
+  def copyCommand(src: Path, dest: Path, overwrite: Boolean): IoCopyCommand
+  def hashCommand(file: Path): IoHashCommand
+  def touchCommand(file: Path): IoTouchCommand
+}
+
+trait DefaultIoCommandBuilder extends IoCommandBuilder {
+  def contentAsStringCommand(path: Path): IoContentAsStringCommand = DefaultIoContentAsStringCommand(path)
+  def writeCommand(path: Path, content: String, options: OpenOptions): IoWriteCommand = DefaultIoWriteCommand(path, content, options)
+  def sizeCommand(path: Path): IoSizeCommand = DefaultIoSizeCommand(path)
+  def deleteCommand(path: Path, swallowIoExceptions: Boolean): IoDeleteCommand = DefaultIoDeleteCommand(path, swallowIoExceptions)
+  def copyCommand(src: Path, dest: Path, overwrite: Boolean): IoCopyCommand = DefaultIoCopyCommand(src, dest, overwrite)
+  def hashCommand(file: Path): IoHashCommand = DefaultIoHashCommand(file)
+  def touchCommand(file: Path): IoTouchCommand = DefaultIoTouchCommand(file)
+}
diff --git a/core/src/main/scala/cromwell/core/io/IoTimeout.scala b/core/src/main/scala/cromwell/core/io/IoTimeout.scala
new file mode 100644
index 000000000..76716f346
--- /dev/null
+++ b/core/src/main/scala/cromwell/core/io/IoTimeout.scala
@@ -0,0 +1,5 @@
+package cromwell.core.io
+
+import java.util.concurrent.TimeoutException
+
+case class IoTimeout(command: IoCommand[_]) extends TimeoutException(s"The I/O operation $command timed out")
diff --git a/core/src/main/scala/cromwell/core/io/Throttle.scala b/core/src/main/scala/cromwell/core/io/Throttle.scala
new file mode 100644
index 000000000..f59d13895
--- /dev/null
+++ b/core/src/main/scala/cromwell/core/io/Throttle.scala
@@ -0,0 +1,5 @@
+package cromwell.core.io
+
+import scala.concurrent.duration.FiniteDuration
+
+case class Throttle(elements: Int, per: FiniteDuration, maximumBurst: Int)
diff --git a/core/src/main/scala/cromwell/core/labels/Label.scala b/core/src/main/scala/cromwell/core/labels/Label.scala
index 4fc1e2959..cee9ac91a 100644
--- a/core/src/main/scala/cromwell/core/labels/Label.scala
+++ b/core/src/main/scala/cromwell/core/labels/Label.scala
@@ -5,69 +5,40 @@ import cats.data.Validated._
 import cats.syntax.cartesian._
 import cats.syntax.validated._
 
+import scala.util.matching.Regex
+
 sealed abstract case class Label(key: String, value: String)
 
 object Label {
 
-  // Yes, 63. Not a typo for 64.
-  // See 'labels' in https://cloud.google.com/genomics/reference/rpc/google.genomics.v1alpha2#google.genomics.v1alpha2.RunPipelineArgs
-  private val MaxLabelLength = 63
-  val LabelRegexPattern = "[a-z]([-a-z0-9]*[a-z0-9])?"
+  val MaxLabelLength = 63
+  val LabelKeyRegex = "[a-z]([-a-z0-9]*[a-z0-9])?"
+  val LabelValueRegex = "([a-z0-9]*[-a-z0-9]*[a-z0-9])?"
 
-  def validateName(s: String): ErrorOr[String] = {
-    if (LabelRegexPattern.r.pattern.matcher(s).matches) {
-      if (s.length <= MaxLabelLength) s.validNel else s"Invalid label: $s was ${s.length} characters. The maximum is $MaxLabelLength".invalidNel
-    } else {
-      s"Invalid label: $s did not match the regex $LabelRegexPattern".invalidNel
-    }
-  }
+  val LabelExpectationsMessage =
+    s"A Label key must match the pattern `$LabelKeyRegex` and a label value must match the pattern `$LabelValueRegex`."
 
-  def validateLabel(key: String, value: String): ErrorOr[Label] = {
-    val validatedKey = validateName(key)
-    val validatedValue = validateName(value)
-
-    (validatedKey |@| validatedValue) map { case (k, v) => new Label(k, v) {} }
+  def validateLabelRegex(s: String, regexAllowed: Regex): ErrorOr[String] = {
+    (regexAllowed.pattern.matcher(s).matches, s.length <= MaxLabelLength) match {
+      case (true, true) => s.validNel
+      case (false, false) => s"Invalid label: `$s` did not match regex $regexAllowed and it is ${s.length} characters. The maximum is $MaxLabelLength.".invalidNel
+      case (false, _) => s"Invalid label: `$s` did not match the regex $regexAllowed.".invalidNel
+      case (_, false) => s"Invalid label: `$s` is ${s.length} characters. The maximum is $MaxLabelLength.".invalidNel
+    }
   }
 
-  /**
-    * Change to meet the constraint:
-    *  - To match the regex LabelRegexPattern
-    *  - To be between 1 and MaxLabelLength characters total
-    */
-  def safeName(mainText: String): String = {
+  def validateLabelKey(key: String): ErrorOr[String] = validateLabelRegex(key, LabelKeyRegex.r)
 
-    validateName(mainText) match {
-      case Valid(labelText) => labelText
-      case _ =>
-        def appendSafe(current: String, nextChar: Char): String = {
-          nextChar match {
-            case c if c.isLetterOrDigit || c == '-' => current + c.toLower
-            case _ => current + '-'
-          }
-        }
+  def validateLabelValue(key: String): ErrorOr[String] = validateLabelRegex(key, LabelValueRegex.r)
 
-        val foldResult = mainText.toCharArray.foldLeft("")(appendSafe)
-
-        val startsValid = foldResult.headOption.exists(_.isLetter)
-        val endsValid = foldResult.lastOption.exists(_.isLetterOrDigit)
-
-        val validStart = if (startsValid) foldResult else "x--" + foldResult
-        val validStartAndEnd = if (endsValid) validStart else validStart + "--x"
-
-        val length = validStartAndEnd.length
-        val tooLong = length > MaxLabelLength
+  def validateLabel(key: String, value: String): ErrorOr[Label] = {
+    val validatedKey = validateLabelKey(key)
+    val validatedValue = validateLabelValue(value)
 
-        if (tooLong) {
-          val middleSeparator = "---"
-          val subSectionLength = (MaxLabelLength - middleSeparator.length) / 2
-          validStartAndEnd.substring(0, subSectionLength) + middleSeparator + validStartAndEnd.substring(length - subSectionLength, length)
-        } else {
-          validStartAndEnd
-        }
-    }
+    (validatedKey |@| validatedValue) map { case (k, v) => new Label(k, v) {} }
   }
 
-  def safeLabel(key: String, value: String): Label = {
-    new Label(safeName(key), safeName(value)) {}
+  def apply(key: String, value: String) = {
+    new Label(key, value) {}
   }
 }
diff --git a/core/src/main/scala/cromwell/core/labels/Labels.scala b/core/src/main/scala/cromwell/core/labels/Labels.scala
index 8e891fb51..e5fe3f575 100644
--- a/core/src/main/scala/cromwell/core/labels/Labels.scala
+++ b/core/src/main/scala/cromwell/core/labels/Labels.scala
@@ -1,18 +1,31 @@
 package cromwell.core.labels
 
+import cats.data.Validated._
+import cats.instances.vector._
+import cats.syntax.traverse._
+import lenthall.validation.ErrorOr
+import lenthall.validation.ErrorOr.ErrorOr
+
 import scala.collection.JavaConverters._
 
 case class Labels(value: Vector[Label]) {
 
-  def asJesLabels = (value map { label => label.key -> label.value }).toMap.asJava
+  def asTuple: Vector[(String, String)] = value.map(label => label.key -> label.value)
+
+  def asMap: Map[String, String] = asTuple.toMap
+
+  def asJavaMap = asMap.asJava
 
   def ++(that: Labels) = Labels(value ++ that.value)
 }
 
 object Labels {
   def apply(values: (String, String)*): Labels = {
-    val kvps: Seq[(String, String)] = values.toSeq
-    Labels((kvps map { case (k, v) => Label.safeLabel(k, v) }).to[Vector])
+    Labels(values.toVector map (Label.apply _).tupled)
+  }
+
+  def validateMapOfLabels(labels: Map[String, String]): ErrorOr[Labels] = {
+    labels.toVector traverse { Label.validateLabel _ }.tupled map Labels.apply
   }
 
   def empty = Labels(Vector.empty)
diff --git a/core/src/main/scala/cromwell/core/package.scala b/core/src/main/scala/cromwell/core/package.scala
index def878003..e94dbc809 100644
--- a/core/src/main/scala/cromwell/core/package.scala
+++ b/core/src/main/scala/cromwell/core/package.scala
@@ -1,13 +1,34 @@
 package cromwell
 
-import wdl4s.values.WdlValue
+import cats.data.Validated._
+import cats.syntax.validated._
+import lenthall.validation.ErrorOr.ErrorOr
+import wdl4s.wdl.values.WdlValue
+
+import scala.util.{Failure, Success, Try}
 
 package object core {
   type LocallyQualifiedName = String
   type FullyQualifiedName = String
   type WorkflowOutputs = Map[FullyQualifiedName, JobOutput]
   type WorkflowOptionsJson = String
+  type WorkflowType = String
+  type WorkflowTypeVersion = String
   type CallOutputs = Map[LocallyQualifiedName, JobOutput]
   type HostInputs = Map[String, WdlValue]
   type EvaluatedRuntimeAttributes = Map[String, WdlValue]
+
+  implicit class toErrorOr[A](val trySomething: Try[A]) {
+    def tryToErrorOr: ErrorOr[A] = trySomething match {
+      case Success(options) => options.validNel
+      case Failure(err) => err.getMessage.invalidNel
+    }
+  }
+
+  implicit class toTry[A](val validatedSomething: ErrorOr[A]) {
+    def errorOrToTry: Try[A] = validatedSomething match {
+      case Valid(options) => Success(options)
+      case Invalid(err) => Failure(new RuntimeException(s"Error(s): ${err.toList.mkString(",")}"))
+    }
+  }
 }
diff --git a/core/src/main/scala/cromwell/core/path/BetterFileMethods.scala b/core/src/main/scala/cromwell/core/path/BetterFileMethods.scala
index b6c09bf7a..94c780a84 100644
--- a/core/src/main/scala/cromwell/core/path/BetterFileMethods.scala
+++ b/core/src/main/scala/cromwell/core/path/BetterFileMethods.scala
@@ -144,9 +144,14 @@ trait BetterFileMethods {
     betterFile.append(text)(openOptions, codec)
     this
   }
+  
+  final def appendText(text: String)(implicit openOptions: OpenOptions = OpenOptions.append, codec: Codec): this.type = {
+    betterFile.appendText(text)(openOptions, codec)
+    this
+  }
 
-  final def append(bytes: Array[Byte])(implicit openOptions: OpenOptions): this.type = {
-    betterFile.append(bytes)(openOptions)
+  final def appendByteArray(bytes: Array[Byte])(implicit openOptions: OpenOptions): this.type = {
+    betterFile.appendByteArray(bytes)(openOptions)
     this
   }
 
@@ -155,8 +160,8 @@ trait BetterFileMethods {
     this
   }
 
-  final def write(bytes: Array[Byte])(implicit openOptions: OpenOptions): this.type = {
-    betterFile.write(bytes)(openOptions)
+  final def writeByteArray(bytes: Array[Byte])(implicit openOptions: OpenOptions): this.type = {
+    betterFile.writeByteArray(bytes)(openOptions)
     this
   }
 
@@ -165,6 +170,11 @@ trait BetterFileMethods {
     this
   }
 
+  final def writeText(text: String)(implicit openOptions: OpenOptions = OpenOptions.default, codec: Codec): this.type = {
+    betterFile.writeText(text)(openOptions, codec)
+    this
+  }
+  
   final def write(text: String)(implicit openOptions: OpenOptions = OpenOptions.default, codec: Codec): this.type = {
     betterFile.write(text)(openOptions, codec)
     this
@@ -204,11 +214,11 @@ trait BetterFileMethods {
     betterFile.bufferedReader(codec)
 
   final def newBufferedWriter(implicit codec: Codec, openOptions: OpenOptions = OpenOptions.default): BufferedWriter =
-    betterFile.newBufferedWriter(codec)
+    betterFile.newBufferedWriter(codec, openOptions)
 
   final def bufferedWriter(implicit codec: Codec,
                            openOptions: OpenOptions = OpenOptions.default): ManagedResource[BufferedWriter] =
-    betterFile.bufferedWriter(codec)
+    betterFile.bufferedWriter(codec, openOptions)
 
   final def newFileReader: FileReader = betterFile.newFileReader
 
diff --git a/core/src/main/scala/cromwell/core/path/DefaultPathBuilderFactory.scala b/core/src/main/scala/cromwell/core/path/DefaultPathBuilderFactory.scala
index 5339fae3c..234a19d79 100644
--- a/core/src/main/scala/cromwell/core/path/DefaultPathBuilderFactory.scala
+++ b/core/src/main/scala/cromwell/core/path/DefaultPathBuilderFactory.scala
@@ -3,6 +3,8 @@ package cromwell.core.path
 import akka.actor.ActorSystem
 import cromwell.core.WorkflowOptions
 
+import scala.concurrent.{ExecutionContext, Future}
+
 case object DefaultPathBuilderFactory extends PathBuilderFactory {
-  override def withOptions(options: WorkflowOptions)(implicit actorSystem: ActorSystem) = DefaultPathBuilder
+  override def withOptions(options: WorkflowOptions)(implicit actorSystem: ActorSystem, ec: ExecutionContext) = Future.successful(DefaultPathBuilder)
 }
diff --git a/core/src/main/scala/cromwell/core/path/EvenBetterPathMethods.scala b/core/src/main/scala/cromwell/core/path/EvenBetterPathMethods.scala
index 1fc6bf4df..3b71d0712 100644
--- a/core/src/main/scala/cromwell/core/path/EvenBetterPathMethods.scala
+++ b/core/src/main/scala/cromwell/core/path/EvenBetterPathMethods.scala
@@ -38,7 +38,7 @@ trait EvenBetterPathMethods {
 
   // betterFile.symbolicLink calls Files.readSymbolicLink, but then implicitly converts the java.nio.Path returned to a better.File
   // which calls toAbsolutePath. Consequently, if the path was relative, the current directory is used to make it absolute.
-  // This is not the desired behaviour to be able to follow relative symbolic links, so bypass better files method and directly use the java one.
+  // This is not the desired behavior to be able to follow relative symbolic links, so bypass better files method and directly use the java one.
   final def symbolicLinkRelative: Option[Path] = {
     if (betterFile.isSymbolicLink) {
       Option(newPath(Files.readSymbolicLink(betterFile.path)))
diff --git a/core/src/main/scala/cromwell/core/path/PathBuilderFactory.scala b/core/src/main/scala/cromwell/core/path/PathBuilderFactory.scala
index 7ee20eb2d..63a91e02b 100644
--- a/core/src/main/scala/cromwell/core/path/PathBuilderFactory.scala
+++ b/core/src/main/scala/cromwell/core/path/PathBuilderFactory.scala
@@ -3,9 +3,11 @@ package cromwell.core.path
 import akka.actor.ActorSystem
 import cromwell.core.WorkflowOptions
 
+import scala.concurrent.{ExecutionContext, Future}
+
 /**
   * Provide a method that can instantiate a path builder with the specified workflow options.
   */
 trait PathBuilderFactory {
-  def withOptions(options: WorkflowOptions)(implicit actorSystem: ActorSystem): PathBuilder
+  def withOptions(options: WorkflowOptions)(implicit as: ActorSystem, ec: ExecutionContext): Future[PathBuilder]
 }
diff --git a/core/src/main/scala/cromwell/core/path/proxy/FileSystemProxy.scala b/core/src/main/scala/cromwell/core/path/proxy/FileSystemProxy.scala
deleted file mode 100644
index f9e9b5817..000000000
--- a/core/src/main/scala/cromwell/core/path/proxy/FileSystemProxy.scala
+++ /dev/null
@@ -1,25 +0,0 @@
-package cromwell.core.path.proxy
-
-import java.lang.Iterable
-import java.nio.file._
-import java.nio.file.attribute.UserPrincipalLookupService
-import java.nio.file.spi.FileSystemProvider
-import java.util
-
-class FileSystemProxy(delegate: FileSystem, injectedProvider: FileSystemProvider) extends FileSystem {
-
-  override def provider(): FileSystemProvider = injectedProvider
-
-  /* delegated */
-  override def supportedFileAttributeViews(): util.Set[String] = delegate.supportedFileAttributeViews()
-  override def getSeparator: String = delegate.getSeparator
-  override def getRootDirectories: Iterable[Path] = delegate.getRootDirectories
-  override def newWatchService(): WatchService = delegate.newWatchService()
-  override def getFileStores: Iterable[FileStore] = delegate.getFileStores
-  override def isReadOnly: Boolean = delegate.isReadOnly
-  override def getPath(first: String, more: String*): Path = new PathProxy(delegate.getPath(first, more: _*), this)
-  override def isOpen: Boolean = delegate.isOpen
-  override def close(): Unit = delegate.close()
-  override def getPathMatcher(syntaxAndPattern: String): PathMatcher = delegate.getPathMatcher(syntaxAndPattern)
-  override def getUserPrincipalLookupService: UserPrincipalLookupService = delegate.getUserPrincipalLookupService
-}
diff --git a/core/src/main/scala/cromwell/core/path/proxy/PathProxy.scala b/core/src/main/scala/cromwell/core/path/proxy/PathProxy.scala
deleted file mode 100644
index 28428e0a3..000000000
--- a/core/src/main/scala/cromwell/core/path/proxy/PathProxy.scala
+++ /dev/null
@@ -1,44 +0,0 @@
-package cromwell.core.path.proxy
-
-import java.io.File
-import java.net.URI
-import java.nio.file.WatchEvent.{Kind, Modifier}
-import java.nio.file._
-import java.util
-
-import scala.util.Try
-
-class PathProxy(delegate: Path, injectedFileSystem: FileSystem) extends Path {
-  def unbox[T](clazz: Class[T]): Try[T] = Try {
-    clazz.cast(delegate)
-  }
-
-  override def getFileSystem: FileSystem = injectedFileSystem
-
-  /* delegated */
-  override def subpath(beginIndex: Int, endIndex: Int): Path = delegate.subpath(beginIndex, endIndex)
-  override def toFile: File = delegate.toFile
-  override def resolveSibling(other: Path): Path = delegate.resolveSibling(other)
-  override def resolveSibling(other: String): Path = delegate.resolveSibling(other)
-  override def isAbsolute: Boolean = delegate.isAbsolute
-  override def getName(index: Int): Path = delegate.getName(index)
-  override def getParent: Path = delegate.getParent
-  override def toAbsolutePath: Path = delegate.toAbsolutePath
-  override def relativize(other: Path): Path = delegate.relativize(other)
-  override def getNameCount: Int = delegate.getNameCount
-  override def toUri: URI = delegate.toUri
-  override def compareTo(other: Path): Int = delegate.compareTo(other)
-  override def register(watcher: WatchService, events: Array[Kind[_]], modifiers: Modifier*): WatchKey = delegate.register(watcher, events, modifiers: _*)
-  override def register(watcher: WatchService, events: Kind[_]*): WatchKey = delegate.register(watcher, events: _*)
-  override def getFileName: Path = delegate.getFileName
-  override def getRoot: Path = delegate.getRoot
-  override def iterator(): util.Iterator[Path] = delegate.iterator()
-  override def normalize(): Path = delegate.normalize()
-  override def endsWith(other: Path): Boolean = delegate.endsWith(other)
-  override def endsWith(other: String): Boolean = delegate.endsWith(other)
-  override def resolve(other: Path): Path = delegate.resolve(other)
-  override def resolve(other: String): Path = delegate.resolve(other)
-  override def startsWith(other: Path): Boolean = delegate.startsWith(other)
-  override def startsWith(other: String): Boolean = delegate.startsWith(other)
-  override def toRealPath(options: LinkOption*): Path = delegate.toRealPath(options: _*)
-}
diff --git a/core/src/main/scala/cromwell/core/path/proxy/RetryableFileSystemProviderProxy.scala b/core/src/main/scala/cromwell/core/path/proxy/RetryableFileSystemProviderProxy.scala
deleted file mode 100644
index 4d2a481c3..000000000
--- a/core/src/main/scala/cromwell/core/path/proxy/RetryableFileSystemProviderProxy.scala
+++ /dev/null
@@ -1,59 +0,0 @@
-package cromwell.core.path.proxy
-
-import java.net.URI
-import java.nio.channels.SeekableByteChannel
-import java.nio.file.DirectoryStream.Filter
-import java.nio.file._
-import java.nio.file.attribute.{BasicFileAttributes, FileAttribute, FileAttributeView}
-import java.nio.file.spi.FileSystemProvider
-import java.util
-
-import akka.actor.ActorSystem
-import cromwell.core.path.CustomRetryParams
-import cromwell.core.retry.Retry
-
-import scala.concurrent.{Await, Future}
-
-class RetryableFileSystemProviderProxy[T <: FileSystemProvider](delegate: T, retryParams: CustomRetryParams = CustomRetryParams.Default)(implicit actorSystem: ActorSystem) extends FileSystemProvider {
-  private val iOExecutionContext = actorSystem.dispatchers.lookup("akka.dispatchers.io-dispatcher")
-
-  // the nio interface is synchronous so we need to wait for the result
-  def withRetry[U](f: () => U): U = Await.result(
-    Retry.withRetry(
-      () => Future(f())(iOExecutionContext),
-      retryParams.maxRetries,
-      retryParams.backoff,
-      retryParams.isTransient,
-      retryParams.isFatal
-    ),
-    retryParams.timeout
-  )
-
-  override def getPath(uri: URI): Path = {
-    val path = delegate.getPath(uri)
-    new PathProxy(path, new FileSystemProxy(path.getFileSystem, this))
-  }
-  override def newFileSystem(uri: URI, env: util.Map[String, _]): FileSystem = {
-    new FileSystemProxy(delegate.newFileSystem(uri, env), this)
-  }
-  override def getScheme: String = delegate.getScheme
-  override def getFileSystem(uri: URI): FileSystem = {
-    new FileSystemProxy(delegate.getFileSystem(uri), this)
-  }
-  override def getFileStore(path: Path): FileStore = delegate.getFileStore(path)
-
-  /* retried operations */
-  override def move(source: Path, target: Path, options: CopyOption*): Unit = withRetry { () => delegate.move(source, target, options: _*) }
-  override def checkAccess(path: Path, modes: AccessMode*): Unit = withRetry { () => delegate.checkAccess(path, modes: _*) }
-  override def createDirectory(dir: Path, attrs: FileAttribute[_]*): Unit = withRetry { () => delegate.createDirectory(dir, attrs: _*) }
-  override def newByteChannel(path: Path, options: util.Set[_ <: OpenOption], attrs: FileAttribute[_]*): SeekableByteChannel = withRetry { () => delegate.newByteChannel(path, options, attrs: _*) }
-  override def isHidden(path: Path): Boolean = withRetry { () => delegate.isHidden(path) }
-  override def copy(source: Path, target: Path, options: CopyOption*): Unit = withRetry { () => delegate.copy(source, target, options: _*) }
-  override def delete(path: Path): Unit = withRetry { () => delegate.delete(path) }
-  override def newDirectoryStream(dir: Path, filter: Filter[_ >: Path]): DirectoryStream[Path] = withRetry { () => delegate.newDirectoryStream(dir, filter) }
-  override def setAttribute(path: Path, attribute: String, value: scala.Any, options: LinkOption*): Unit = withRetry { () => delegate.setAttribute(path, attribute, value, options: _*) }
-  override def readAttributes[A <: BasicFileAttributes](path: Path, `type`: Class[A], options: LinkOption*): A = withRetry { () => delegate.readAttributes(path, `type`, options: _*) }
-  override def readAttributes(path: Path, attributes: String, options: LinkOption*): util.Map[String, AnyRef] = withRetry { () => delegate.readAttributes(path, attributes, options: _*) }
-  override def isSameFile(path: Path, path2: Path): Boolean = withRetry { () => delegate.isSameFile(path, path2) }
-  override def getFileAttributeView[V <: FileAttributeView](path: Path, `type`: Class[V], options: LinkOption*): V = withRetry { () => delegate.getFileAttributeView(path, `type`, options: _*) }
-}
diff --git a/core/src/main/scala/cromwell/core/retry/Retry.scala b/core/src/main/scala/cromwell/core/retry/Retry.scala
index a2788cbb8..c6d1681ba 100644
--- a/core/src/main/scala/cromwell/core/retry/Retry.scala
+++ b/core/src/main/scala/cromwell/core/retry/Retry.scala
@@ -33,15 +33,16 @@ object Retry {
     implicit val ec: ExecutionContext = actorSystem.dispatcher
     val delay = backoff.backoffMillis.millis
 
-    if (maxRetries.forall(_ > 0)) {
-      f() recoverWith {
-        case throwable if isFatal(throwable) => Future.failed(CromwellFatalException(throwable))
-        case throwable if !isFatal(throwable) =>
-          val retriesLeft = if (isTransient(throwable)) maxRetries else maxRetries map { _ - 1 }
+    f() recoverWith {
+      case throwable if isFatal(throwable) => Future.failed(CromwellFatalException(throwable))
+      case throwable if !isFatal(throwable) =>
+        val retriesLeft = if (isTransient(throwable)) maxRetries else maxRetries map { _ - 1 }
+        
+        if (retriesLeft.forall(_ > 0)) {
           after(delay, actorSystem.scheduler)(withRetry(f, backoff = backoff, maxRetries = retriesLeft, isTransient = isTransient, isFatal = isFatal))
-      }
-    } else f() recoverWith {
-      case e: Exception => Future.failed(new CromwellFatalException(e))
+        } else {
+          Future.failed(new CromwellFatalException(throwable))
+        }
     }
   }
 
diff --git a/core/src/main/scala/cromwell/core/simpleton/WdlValueBuilder.scala b/core/src/main/scala/cromwell/core/simpleton/WdlValueBuilder.scala
index 774c1b5ce..238553f2a 100644
--- a/core/src/main/scala/cromwell/core/simpleton/WdlValueBuilder.scala
+++ b/core/src/main/scala/cromwell/core/simpleton/WdlValueBuilder.scala
@@ -1,12 +1,12 @@
 package cromwell.core.simpleton
 
-import wdl4s.TaskOutput
-import wdl4s.types._
-import wdl4s.values.{WdlArray, WdlMap, WdlOptionalValue, WdlPair, WdlValue}
+import cromwell.core.simpleton.WdlValueSimpleton._
+import cromwell.core.{CallOutputs, JobOutput}
+import wdl4s.wdl.TaskOutput
+import wdl4s.wdl.types._
+import wdl4s.wdl.values.{WdlArray, WdlMap, WdlOptionalValue, WdlPair, WdlValue}
 
 import scala.language.postfixOps
-import cromwell.core.{CallOutputs, JobOutput}
-import cromwell.core.simpleton.WdlValueSimpleton._
 
 
 /**
@@ -88,7 +88,7 @@ object WdlValueBuilder {
 
     // Group tuples by key using a Map with key type `K`.
     def group[K](tuples: Traversable[(K, SimpletonComponent)]): Map[K, Traversable[SimpletonComponent]] = {
-      tuples groupBy { case (i, _) => i } mapValues { _ map { case (i, s) => s} }
+      tuples groupBy { case (i, _) => i } mapValues { _ map { case (_, s) => s} }
     }
 
     outputType match {
diff --git a/core/src/main/scala/cromwell/core/simpleton/WdlValueSimpleton.scala b/core/src/main/scala/cromwell/core/simpleton/WdlValueSimpleton.scala
index 1f5e04375..aa5397d55 100644
--- a/core/src/main/scala/cromwell/core/simpleton/WdlValueSimpleton.scala
+++ b/core/src/main/scala/cromwell/core/simpleton/WdlValueSimpleton.scala
@@ -1,6 +1,6 @@
 package cromwell.core.simpleton
 
-import wdl4s.values._
+import wdl4s.wdl.values._
 
 case class WdlValueSimpleton(simpletonKey: String, simpletonValue: WdlPrimitive)
 
diff --git a/core/src/main/scala/cromwell/util/GracefulShutdownHelper.scala b/core/src/main/scala/cromwell/util/GracefulShutdownHelper.scala
new file mode 100644
index 000000000..5ed66fb5e
--- /dev/null
+++ b/core/src/main/scala/cromwell/util/GracefulShutdownHelper.scala
@@ -0,0 +1,34 @@
+package cromwell.util
+
+import akka.actor.{Actor, ActorLogging, ActorRef, Terminated}
+import akka.pattern.GracefulStopSupport
+import cats.data.NonEmptyList
+import cromwell.util.GracefulShutdownHelper.ShutdownCommand
+
+object GracefulShutdownHelper {
+  case object ShutdownCommand
+}
+
+trait GracefulShutdownHelper extends GracefulStopSupport { this: Actor with ActorLogging =>
+  private var shuttingDown: Boolean = false
+  private var shutdownList: Set[ActorRef] = Set.empty
+  
+  def isShuttingDown: Boolean = shuttingDown
+
+  def waitForActorsAndShutdown(actorsLists: NonEmptyList[ActorRef]): Unit = {
+    if (shuttingDown) {
+      log.error("Programmer error, this actor has already initiated its shutdown. Only call this once per actor !")
+    } else {
+      shuttingDown = true
+      shutdownList = actorsLists.toList.toSet
+      shutdownList foreach context.watch
+      shutdownList foreach { _ ! ShutdownCommand }
+      
+      context become {
+        case Terminated(actor) if shuttingDown && shutdownList.contains(actor) =>
+          shutdownList = shutdownList - actor
+          if (shutdownList.isEmpty) context stop self
+      }
+    }
+  }
+}
diff --git a/core/src/main/scala/cromwell/util/JsonFormatting/WdlValueJsonFormatter.scala b/core/src/main/scala/cromwell/util/JsonFormatting/WdlValueJsonFormatter.scala
index dc7f55fe5..53bdd4293 100644
--- a/core/src/main/scala/cromwell/util/JsonFormatting/WdlValueJsonFormatter.scala
+++ b/core/src/main/scala/cromwell/util/JsonFormatting/WdlValueJsonFormatter.scala
@@ -1,9 +1,10 @@
 package cromwell.util.JsonFormatting
 
 import spray.json._
-import wdl4s.WdlExpression
-import wdl4s.types.{WdlArrayType, WdlMapType, WdlStringType}
-import wdl4s.values._
+import wdl4s.wdl.WdlExpression
+import wdl4s.wdl.types._
+import wdl4s.wdl.values._
+import wdl4s.wdl.values.{WdlBoolean, WdlFloat, WdlInteger, WdlString, WdlValue}
 
 object WdlValueJsonFormatter extends DefaultJsonProtocol {
   implicit object WdlValueJsonFormat extends RootJsonFormat[WdlValue] {
diff --git a/core/src/main/scala/cromwell/util/PromiseActor.scala b/core/src/main/scala/cromwell/util/PromiseActor.scala
index 58aea267a..bd5efa5b0 100644
--- a/core/src/main/scala/cromwell/util/PromiseActor.scala
+++ b/core/src/main/scala/cromwell/util/PromiseActor.scala
@@ -1,7 +1,7 @@
 package cromwell.util
 
 import akka.actor._
-
+import cromwell.core.Dispatcher.EngineDispatcher
 import scala.concurrent.{Future, Promise}
 
 private class PromiseActor(promise: Promise[Any], sendTo: ActorRef, msg: Any) extends Actor with ActorLogging {
@@ -42,7 +42,7 @@ object PromiseActor {
     promise.future
   }
 
-  def props(promise: Promise[Any], sendTo: ActorRef, msg: Any): Props = Props(new PromiseActor(promise, sendTo, msg))
+  def props(promise: Promise[Any], sendTo: ActorRef, msg: Any): Props = Props(new PromiseActor(promise, sendTo, msg)).withDispatcher(EngineDispatcher)
 
   implicit class EnhancedActorRef(val actorRef: ActorRef) extends AnyVal {
     def askNoTimeout(message: Any)(implicit actorRefFactory: ActorRefFactory): Future[Any] = {
diff --git a/core/src/test/resources/application.conf b/core/src/test/resources/application.conf
index 1ef667470..f45d06517 100644
--- a/core/src/test/resources/application.conf
+++ b/core/src/test/resources/application.conf
@@ -22,7 +22,7 @@ database.db.connectionTimeout = 3000
 database-test-mysql {
   # Run the following to (optionally) drop and (re-)create the database:
   # mysql -utravis -e "DROP DATABASE IF EXISTS cromwell_test" && mysql -utravis -e "CREATE DATABASE cromwell_test"
-  driver = "slick.driver.MySQLDriver$"
+  profile = "slick.jdbc.MySQLProfile$"
   db {
     driver = "com.mysql.jdbc.Driver"
     url = "jdbc:mysql://localhost/cromwell_test?useSSL=false"
diff --git a/core/src/test/scala/cromwell/core/FailIoActor.scala b/core/src/test/scala/cromwell/core/FailIoActor.scala
new file mode 100644
index 000000000..c27e5f3e4
--- /dev/null
+++ b/core/src/test/scala/cromwell/core/FailIoActor.scala
@@ -0,0 +1,19 @@
+package cromwell.core
+
+import akka.actor.{Actor, Props}
+import cromwell.core.FailIoActor._
+import cromwell.core.io._
+
+object FailIoActor {
+  def props() = Props(new FailIoActor())
+  val failure = new Exception("Io failure - part of test flow")
+}
+
+class FailIoActor() extends Actor {
+  override def receive = {
+    case command: IoCommand[_] => sender() ! IoFailure(command, failure)
+
+    // With context
+    case (requestContext: Any, command: IoCommand[_]) => sender() ! (requestContext -> IoFailure(command, failure))
+  }
+}
diff --git a/core/src/test/scala/cromwell/core/MockIoActor.scala b/core/src/test/scala/cromwell/core/MockIoActor.scala
new file mode 100644
index 000000000..6d525b5bf
--- /dev/null
+++ b/core/src/test/scala/cromwell/core/MockIoActor.scala
@@ -0,0 +1,25 @@
+package cromwell.core
+
+import akka.actor.{Actor, Props}
+import cromwell.core.io._
+
+object MockIoActor {
+  def props(returnCode: String = "0", stderrSize: Long = 0L) = Props(new MockIoActor(returnCode, stderrSize))
+}
+
+class MockIoActor(returnCode: String, stderrSize: Long) extends Actor {
+  override def receive = {
+    case command: IoCopyCommand => sender() ! IoSuccess(command, ())
+    case command: IoWriteCommand => sender() ! IoSuccess(command, ())
+    case command: IoDeleteCommand => sender() ! IoSuccess(command, ())
+    case command: IoSizeCommand => sender() ! IoSuccess(command, 0L)
+    case command: IoContentAsStringCommand => sender() ! IoSuccess(command, "0")
+      
+      // With context
+    case (requestContext: Any, command: IoCopyCommand) => sender() ! (requestContext -> IoSuccess(command, ()))
+    case (requestContext: Any, command: IoWriteCommand) => sender() ! (requestContext -> IoSuccess(command, ()))
+    case (requestContext: Any, command: IoDeleteCommand) => sender() ! (requestContext -> IoSuccess(command, ()))
+    case (requestContext: Any, command: IoSizeCommand) => sender() ! (requestContext -> IoSuccess(command, stderrSize))
+    case (requestContext: Any, command: IoContentAsStringCommand) => sender() ! (requestContext -> IoSuccess(command, returnCode))
+  }
+}
diff --git a/core/src/test/scala/cromwell/core/SimpleIoActor.scala b/core/src/test/scala/cromwell/core/SimpleIoActor.scala
new file mode 100644
index 000000000..2daaa2b95
--- /dev/null
+++ b/core/src/test/scala/cromwell/core/SimpleIoActor.scala
@@ -0,0 +1,100 @@
+package cromwell.core
+
+import akka.actor.{Actor, Props}
+import cromwell.core.io._
+
+import scala.io.Codec
+import scala.util.{Failure, Success, Try}
+
+object SimpleIoActor {
+  def props = Props(new SimpleIoActor)
+}
+
+class SimpleIoActor extends Actor {
+  
+  override def receive = {
+    case command: IoCopyCommand =>
+      
+      Try(command.source.copyTo(command.destination, command.overwrite)) match {
+        case Success(_) => sender() ! IoSuccess(command, ())
+        case Failure(failure) => sender() ! IoFailure(command, failure)
+      }
+      
+    case command: IoWriteCommand =>
+      
+      Try(command.file.write(command.content)(command.openOptions, Codec.UTF8)) match {
+        case Success(_) => sender() ! IoSuccess(command, ())
+        case Failure(failure) => sender() ! IoFailure(command, failure)
+      }
+      
+    case command: IoDeleteCommand =>
+      
+      Try(command.file.delete(command.swallowIOExceptions)) match {
+        case Success(_) => sender() ! IoSuccess(command, ())
+        case Failure(failure) => sender() ! IoFailure(command, failure)
+      }
+      
+    case command: IoSizeCommand =>
+      
+      Try(command.file.size) match {
+        case Success(size) => sender() ! IoSuccess(command, size)
+        case Failure(failure) => sender() ! IoFailure(command, failure)
+      }
+      
+    case command: IoContentAsStringCommand =>
+      
+      Try(command.file.contentAsString) match {
+        case Success(content) => sender() ! IoSuccess(command, content)
+        case Failure(failure) => sender() ! IoFailure(command, failure)
+      }
+      
+    case command: IoHashCommand =>
+      Try(command.file.md5) match {
+        case Success(hash) => sender() ! IoSuccess(command, hash)
+        case Failure(failure) => sender() ! IoFailure(command, failure)
+      }
+
+    // With context
+    case (requestContext: Any, command: IoCopyCommand) =>
+      
+      Try(command.source.copyTo(command.destination, command.overwrite)) match {
+        case Success(_) => sender() ! (requestContext -> IoSuccess(command, ()))
+        case Failure(failure) => sender() ! (requestContext -> IoFailure(command, failure))
+      }
+      
+    case (requestContext: Any, command: IoWriteCommand) =>
+
+      Try(command.file.write(command.content)) match {
+        case Success(_) => sender() ! (requestContext -> IoSuccess(command, ()))
+        case Failure(failure) => sender() ! (requestContext -> IoFailure(command, failure))
+      }
+      
+    case (requestContext: Any, command: IoDeleteCommand) =>
+
+      Try(command.file.delete(command.swallowIOExceptions)) match {
+        case Success(_) => sender() ! (requestContext -> IoSuccess(command, ()))
+        case Failure(failure) => sender() ! (requestContext -> IoFailure(command, failure))
+      }
+      
+    case (requestContext: Any, command: IoSizeCommand) =>
+      
+      Try(command.file.size) match {
+        case Success(size) => sender() ! (requestContext -> IoSuccess(command, size))
+        case Failure(failure) => sender() ! (requestContext -> IoFailure(command, failure))
+      }
+      
+    case (requestContext: Any, command: IoContentAsStringCommand) =>
+
+      Try(command.file.contentAsString) match {
+        case Success(content) => sender() ! (requestContext -> IoSuccess(command, content))
+        case Failure(failure) => sender() ! (requestContext -> IoFailure(command, failure))
+      }
+      
+    case (requestContext: Any, command: IoHashCommand) =>
+      
+      Try(command.file.md5) match {
+        case Success(hash) => sender() ! (requestContext -> IoSuccess(command, hash))
+        case Failure(failure) => sender() ! (requestContext -> IoFailure(command, failure))
+      }
+  }
+}
diff --git a/core/src/test/scala/cromwell/core/TestKitSuite.scala b/core/src/test/scala/cromwell/core/TestKitSuite.scala
index 751949a7f..282765e9f 100644
--- a/core/src/test/scala/cromwell/core/TestKitSuite.scala
+++ b/core/src/test/scala/cromwell/core/TestKitSuite.scala
@@ -22,6 +22,8 @@ abstract class TestKitSuite(actorSystemName: String = TestKitSuite.randomName,
   }
 
   val emptyActor = system.actorOf(Props.empty)
+  val mockIoActor = system.actorOf(MockIoActor.props())
+  val failIoActor = system.actorOf(FailIoActor.props())
 }
 
 object TestKitSuite {
@@ -53,7 +55,7 @@ object TestKitSuite {
       |    }
       |
       |    # A dispatcher for engine actors
-      |    # Because backends behaviour is unpredictable (potentially blocking, slow) the engine runs
+      |    # Because backends behavior is unpredictable (potentially blocking, slow) the engine runs
       |    # on its own dispatcher to prevent backends from affecting its performance.
       |    engine-dispatcher {
       |      type = Dispatcher
diff --git a/core/src/test/scala/cromwell/core/actor/RobustClientHelperSpec.scala b/core/src/test/scala/cromwell/core/actor/RobustClientHelperSpec.scala
new file mode 100644
index 000000000..e3f73a6ac
--- /dev/null
+++ b/core/src/test/scala/cromwell/core/actor/RobustClientHelperSpec.scala
@@ -0,0 +1,187 @@
+package cromwell.core.actor
+
+import akka.actor.{Actor, ActorLogging, ActorRef}
+import akka.testkit.{ImplicitSender, TestActorRef, TestProbe}
+import cromwell.core.TestKitSuite
+import cromwell.core.actor.StreamIntegration.BackPressure
+import org.scalatest.{FlatSpecLike, Matchers}
+
+import scala.concurrent.duration._
+import scala.language.postfixOps
+
+class RobustClientHelperSpec extends TestKitSuite with FlatSpecLike with Matchers with ImplicitSender {
+  behavior of "RobustClientHelper"
+  
+  it should "handle Backpressure responses" in {
+    val remoteActor = TestProbe()
+    val delegateActor = TestProbe()
+    
+    val margin = 1 second
+    val backpressureTimeout = 1 second
+    val noResponseTimeout = 10 seconds
+    val testActor = TestActorRef(new TestActor(delegateActor.ref, backpressureTimeout, noResponseTimeout))
+    
+    val messageToSend = TestActor.TestMessage("hello")
+    
+    //send message
+    testActor.underlyingActor.sendMessage(messageToSend, remoteActor.ref)
+    
+    // remote actor receives message
+    remoteActor.expectMsg(messageToSend)
+    
+    // remote actor sends a backpressure message
+    remoteActor.reply(BackPressure(messageToSend))
+    
+    // remote actor expects request again after backpressureTimeout
+    remoteActor.expectMsg(backpressureTimeout + margin, messageToSend)
+    
+    // remote actor replies
+    remoteActor.reply("world")
+    
+    // delegate actor receives response
+    delegateActor.expectMsg("world")
+    
+    // remote actor doesn't receives new messages
+    remoteActor.expectNoMsg()
+    
+    // Wait long enough that to make sure that we won't receive a ServiceUnreachable message, meaning the timeout timer
+    // has been cancelled. Note that it is the responsibility of the actor to cancel it, the RobustClientHelper does not
+    // handle that part.
+    delegateActor.expectNoMsg(8 seconds)
+  }
+
+  it should "handle a successful response" in {
+    val remoteActor = TestProbe()
+    val delegateActor = TestProbe()
+
+    val backpressureTimeout = 1 second
+    val noResponseTimeout = 20 seconds
+    val testActor = TestActorRef(new TestActor(delegateActor.ref, backpressureTimeout, noResponseTimeout))
+
+    val messageToSend = TestActor.TestMessage("hello")
+
+    // send message
+    testActor.underlyingActor.sendMessage(messageToSend, remoteActor.ref)
+
+    // remote actor receives message
+    remoteActor.expectMsg(messageToSend)
+    
+    // remote actor replies
+    remoteActor.reply("world")
+    
+    // delegate receives response
+    delegateActor.expectMsg("world")
+    
+    // remote actor doesn't receives new messages
+    remoteActor.expectNoMsg()
+    delegateActor.expectNoMsg()
+  }
+
+  it should "timeout if no response" in {
+    val remoteActor = TestProbe()
+    val delegateActor = TestProbe()
+
+    val backpressureTimeout = 1 second
+    val noResponseTimeout = 2 seconds
+    val testActor = TestActorRef(new TestActor(delegateActor.ref, backpressureTimeout, noResponseTimeout))
+
+    val messageToSend = TestActor.TestMessage("hello")
+
+    // send message
+    testActor.underlyingActor.sendMessage(messageToSend, remoteActor.ref)
+
+    // remote actor receives message
+    remoteActor.expectMsg(messageToSend)
+
+    // remote actor does not reply
+
+    // delegate receives ServiceUnreachable message
+    delegateActor.expectMsg(TestActor.ServiceUnreachable)
+
+    // remote actor doesn't receives new messages
+    remoteActor.expectNoMsg()
+    delegateActor.expectNoMsg()
+  }
+
+  it should "reset timeout when backpressured is received" in {
+    val remoteActor = TestProbe()
+    val delegateActor = TestProbe()
+    
+    val margin = 1 second
+    val backpressureTimeout = 1 second
+    val noResponseTimeout = 3 seconds
+    val testActor = TestActorRef(new TestActor(delegateActor.ref, backpressureTimeout, noResponseTimeout))
+
+    val messageToSend = TestActor.TestMessage("hello")
+
+    // send message
+    testActor.underlyingActor.sendMessage(messageToSend, remoteActor.ref)
+
+    // remote actor receives message
+    remoteActor.expectMsg(messageToSend)
+
+    // remote actor sends a backpressure message
+    remoteActor.reply(BackPressure(messageToSend))
+
+    // remote actor expects request again after backpressureTimeout
+    remoteActor.expectMsg(backpressureTimeout + margin, messageToSend)
+
+    // remote actor replies
+    remoteActor.reply("world")
+
+    // delegate receives ServiceUnreachable message
+    delegateActor.expectMsg("world")
+
+    // remote actor doesn't receives new messages
+    remoteActor.expectNoMsg()
+    // ensure that no ServiceUnreachable message was sent
+    delegateActor.expectNoMsg(4 seconds)
+  }
+  
+  it should "randomize backpressure timings" in {
+    val delegateActor = TestProbe()
+    val backpressureTimeout = 100 seconds
+    val noResponseTimeout = 3 seconds
+    val randomizeFactor = 0.5D
+    
+    val testActor = TestActorRef(new TestActor(delegateActor.ref, backpressureTimeout, noResponseTimeout, randomizeFactor)).underlyingActor
+    
+    val randomBackpressures = 0 until 10 map { _ =>
+      val time = testActor.generateBackpressureTime
+      time.gt(50.seconds) shouldBe true
+      time.lt(150.seconds) shouldBe true
+      time
+    }
+    
+    // They should all be different
+    randomBackpressures.distinct.size > 1 shouldBe true
+  }
+  
+  private [actor] object TestActor {
+    case class TestMessage(v: String)
+    case object ServiceUnreachable
+  }
+  private class TestActor(delegateTo: ActorRef,
+                          override val backpressureTimeout: FiniteDuration,
+                          noResponseTimeout: FiniteDuration,
+                          override val backpressureRandomizerFactor: Double = 0.5D) extends Actor with ActorLogging with RobustClientHelper {
+    
+    context.become(robustReceive orElse receive)
+    var messageSent: Any = _
+    
+    override def receive: Receive = {
+      case message =>
+        cancelTimeout(messageSent)
+        delegateTo ! message
+    }
+    
+    def sendMessage(message: Any, to: ActorRef) = {
+      messageSent = message
+      robustSend(message, to, noResponseTimeout)
+    }
+
+    override protected def onTimeout(message: Any, to: ActorRef): Unit = {
+      delegateTo ! TestActor.ServiceUnreachable
+    }
+  }
+}
diff --git a/core/src/test/scala/cromwell/core/actor/StreamActorHelperSpec.scala b/core/src/test/scala/cromwell/core/actor/StreamActorHelperSpec.scala
new file mode 100644
index 000000000..4fcc1fc98
--- /dev/null
+++ b/core/src/test/scala/cromwell/core/actor/StreamActorHelperSpec.scala
@@ -0,0 +1,74 @@
+package cromwell.core.actor
+
+import akka.actor.{Actor, ActorLogging, ActorRef, Props}
+import akka.stream.QueueOfferResult.Dropped
+import akka.stream.scaladsl.Source
+import akka.stream.{ActorMaterializer, OverflowStrategy}
+import akka.testkit.{ImplicitSender, TestActorRef}
+import cromwell.core.TestKitSuite
+import cromwell.core.actor.StreamIntegration._
+import cromwell.core.actor.TestStreamActor.{TestStreamActorCommand, TestStreamActorContext}
+import org.scalatest.{FlatSpecLike, Matchers}
+
+import scala.concurrent.ExecutionContext
+
+class StreamActorHelperSpec extends TestKitSuite with FlatSpecLike with Matchers with ImplicitSender {
+  behavior of "StreamActorHelper"
+  
+  implicit val materializer = ActorMaterializer()
+
+  it should "catch EnqueueResponse message" in {
+    val actor = TestActorRef(Props(new TestStreamActor(1)))
+    val command = new TestStreamActorCommand
+    actor ! command
+    expectMsg("hello")
+    system stop actor
+  }
+  
+  it should "send a backpressure message when messages are dropped by the queue" in {
+    val actor = TestActorRef(new TestStreamActor(1))
+    val command = new TestStreamActorCommand
+
+    actor ! EnqueueResponse(Dropped, TestStreamActorContext(command, self, None))
+
+    expectMsg(BackPressure(command))
+
+    system stop actor
+  }
+
+  it should "send a backpressure message with context when messages are dropped by the queue" in {
+    val actor = TestActorRef(new TestStreamActor(1))
+    val command = new TestStreamActorCommand
+
+    actor ! EnqueueResponse(Dropped, TestStreamActorContext(command, self, Option("context")))
+
+    expectMsg(BackPressure("context" -> command))
+
+    system stop actor
+  }
+}
+
+
+private object TestStreamActor {
+  class TestStreamActorCommand
+  case class TestStreamActorContext(request: TestStreamActorCommand, replyTo: ActorRef, override val clientContext: Option[Any]) extends StreamContext
+}
+
+private class TestStreamActor(queueSize: Int)(implicit override val materializer: ActorMaterializer) extends Actor with ActorLogging with StreamActorHelper[TestStreamActorContext] {
+  
+  override protected def actorReceive: Receive = {
+    case command: TestStreamActorCommand =>
+      val replyTo = sender()
+      val commandContext = TestStreamActorContext(command, replyTo, None)
+      sendToStream(commandContext)
+    case (userContext: Any, command: TestStreamActorCommand) =>
+      val replyTo = sender()
+      val commandContext = TestStreamActorContext(command, replyTo, Option(userContext))
+      sendToStream(commandContext)
+  }
+
+  override protected val streamSource = Source.queue[TestStreamActorContext](queueSize, OverflowStrategy.dropNew)
+    .map{ ("hello", _) }
+
+  override implicit def ec: ExecutionContext = context.dispatcher
+}
diff --git a/core/src/test/scala/cromwell/core/callcaching/HashKeySpec.scala b/core/src/test/scala/cromwell/core/callcaching/HashKeySpec.scala
new file mode 100644
index 000000000..a25b311a6
--- /dev/null
+++ b/core/src/test/scala/cromwell/core/callcaching/HashKeySpec.scala
@@ -0,0 +1,35 @@
+package cromwell.core.callcaching
+
+import org.scalatest.{FlatSpec, Matchers}
+
+class HashKeySpec extends FlatSpec with Matchers {
+
+  "HashKey" should "produce consistent key value" in {
+    val keys = Set(
+      HashKey("command template"),
+      HashKey("backend name"),
+      HashKey("input count"),
+      HashKey("output count"),
+      HashKey("runtime attribute", "failOnStderr"),
+      HashKey(checkForHitOrMiss = false, "runtime attribute", "cpu"),
+      HashKey("runtime attribute", "continueOnReturnCode"),
+      HashKey("input", "String stringInput"),
+      HashKey("output", "String myOutput"),
+      HashKey("runtime attribute", "docker")
+    )
+    
+    keys map { _.key } should contain theSameElementsAs Set(
+      "command template",
+      "backend name",
+      "input count",
+      "output count",
+      "runtime attribute: failOnStderr",
+      "runtime attribute: cpu",
+      "runtime attribute: continueOnReturnCode",
+      "input: String stringInput",
+      "output: String myOutput",
+      "runtime attribute: docker"
+    )
+  }
+  
+}
diff --git a/core/src/test/scala/cromwell/core/io/AsyncIoSpec.scala b/core/src/test/scala/cromwell/core/io/AsyncIoSpec.scala
new file mode 100644
index 000000000..6cb519079
--- /dev/null
+++ b/core/src/test/scala/cromwell/core/io/AsyncIoSpec.scala
@@ -0,0 +1,116 @@
+package cromwell.core.io
+
+import java.nio.file.{FileAlreadyExistsException, NoSuchFileException}
+import java.util.UUID
+
+import akka.actor.{Actor, ActorLogging, ActorRef}
+import akka.testkit.TestActorRef
+import cromwell.core.path.DefaultPathBuilder
+import cromwell.core.{SimpleIoActor, TestKitSuite}
+import org.scalatest.mockito.MockitoSugar
+import org.scalatest.{AsyncFlatSpecLike, Matchers}
+
+class AsyncIoSpec extends TestKitSuite with AsyncFlatSpecLike with Matchers with MockitoSugar {
+
+  behavior of "AsyncIoSpec"
+  
+  val simpleIoActor = system.actorOf(SimpleIoActor.props)
+  
+  override def afterAll() = {
+    system stop simpleIoActor
+    super.afterAll()
+  } 
+  
+  it should "write asynchronously" in {
+    val testActor = TestActorRef(new AsyncIoTestActor(simpleIoActor))
+
+    val testPath = DefaultPathBuilder.createTempFile()
+    
+    testActor.underlyingActor.writeAsync(testPath, "hello", Seq.empty) map { _ =>
+      assert(testPath.contentAsString == "hello")
+    }
+  }
+
+  it should "read asynchronously" in {
+    val testActor = TestActorRef(new AsyncIoTestActor(simpleIoActor))
+
+    val testPath = DefaultPathBuilder.createTempFile()
+    testPath.write("hello")
+
+    testActor.underlyingActor.contentAsStringAsync(testPath) map { result =>
+      assert(result == "hello")
+    }
+  }
+
+  it should "get size asynchronously" in {
+    val testActor = TestActorRef(new AsyncIoTestActor(simpleIoActor))
+
+    val testPath = DefaultPathBuilder.createTempFile()
+    testPath.write("hello")
+
+    testActor.underlyingActor.sizeAsync(testPath) map { size =>
+      assert(size == 5)
+    }
+  }
+
+  it should "get hash asynchronously" in {
+    val testActor = TestActorRef(new AsyncIoTestActor(simpleIoActor))
+
+    val testPath = DefaultPathBuilder.createTempFile()
+    testPath.write("hello")
+
+    testActor.underlyingActor.hashAsync(testPath) map { hash =>
+      assert(hash == "5D41402ABC4B2A76B9719D911017C592")
+    }
+  }
+
+  it should "copy asynchronously" in {
+    val testActor = TestActorRef(new AsyncIoTestActor(simpleIoActor))
+
+    val testPath = DefaultPathBuilder.createTempFile()
+    val testCopyPath = testPath.sibling(UUID.randomUUID().toString)
+
+    testActor.underlyingActor.copyAsync(testPath, testCopyPath) map { hash =>
+      assert(testCopyPath.exists)
+    }
+
+    testPath.write("new text")
+    
+    // Honor overwrite true
+    testActor.underlyingActor.copyAsync(testPath, testCopyPath, overwrite = true) map { hash =>
+      assert(testCopyPath.exists)
+      assert(testCopyPath.contentAsString == "new text")
+    }
+
+    // Honor overwrite false
+    recoverToSucceededIf[FileAlreadyExistsException] { testActor.underlyingActor.copyAsync(testPath, testCopyPath, overwrite = false) }
+  }
+
+  it should "delete asynchronously" in {
+    val testActor = TestActorRef(new AsyncIoTestActor(simpleIoActor))
+
+    val testPath = DefaultPathBuilder.createTempFile()
+
+    testActor.underlyingActor.deleteAsync(testPath) map { _ =>
+      assert(!testPath.exists)
+    }
+
+    // Honor swallow exception true
+    testActor.underlyingActor.deleteAsync(testPath, swallowIoExceptions = true) map { _ =>
+      assert(!testPath.exists)
+    }
+
+    // Honor swallow exception false
+    recoverToSucceededIf[NoSuchFileException] { testActor.underlyingActor.deleteAsync(testPath, swallowIoExceptions = false) }
+  }
+
+  private class AsyncIoTestActor(override val ioActor: ActorRef) extends Actor with ActorLogging with AsyncIo with DefaultIoCommandBuilder {
+
+    context.become(ioReceive orElse receive)
+
+    override def receive: Receive = {
+      case _ =>
+    }
+  }
+
+}
diff --git a/core/src/test/scala/cromwell/core/io/IoClientHelperSpec.scala b/core/src/test/scala/cromwell/core/io/IoClientHelperSpec.scala
new file mode 100644
index 000000000..1411a47c6
--- /dev/null
+++ b/core/src/test/scala/cromwell/core/io/IoClientHelperSpec.scala
@@ -0,0 +1,107 @@
+package cromwell.core.io
+
+import akka.actor.{Actor, ActorLogging, ActorRef}
+import akka.testkit.{TestActorRef, TestProbe}
+import cromwell.core.TestKitSuite
+import cromwell.core.io.DefaultIoCommand.DefaultIoSizeCommand
+import cromwell.core.path.Path
+import org.scalatest.mockito.MockitoSugar
+import org.scalatest.{FlatSpecLike, Matchers}
+
+import scala.concurrent.duration.{FiniteDuration, _}
+import scala.language.postfixOps
+
+class IoClientHelperSpec extends TestKitSuite with FlatSpecLike with Matchers with MockitoSugar {
+
+  behavior of "IoClientHelperSpec"
+
+  it should "intercept IoAcks and cancel timers" in {
+    val ioActorProbe = TestProbe()
+    val delegateProbe = TestProbe()
+    val backpressureTimeout = 1 second
+    val noResponseTimeout = 3 seconds
+    
+    val testActor = TestActorRef(new IoClientHelperTestActor(ioActorProbe.ref, delegateProbe.ref, backpressureTimeout, noResponseTimeout)) 
+    
+    val command = DefaultIoSizeCommand(mock[Path])
+    val response = IoSuccess(command, 5)
+
+    // Send the command
+    testActor.underlyingActor.sendMessage(command)
+
+    // Io actor receives the command
+    ioActorProbe.expectMsg(command)
+    
+    // Io actor replies
+    ioActorProbe.reply(response)
+    
+    // delegate should receive the response
+    delegateProbe.expectMsg(response)
+    
+    // And nothing else, meaning the timeout timer has been cancelled
+    delegateProbe.expectNoMsg()
+
+    // timeouts map should be empty
+    testActor.underlyingActor.timeouts.isEmpty shouldBe true
+  }
+
+  it should "intercept IoAcks and cancel timers for a command with context" in {
+    val ioActorProbe = TestProbe()
+    val delegateProbe = TestProbe()
+    val backpressureTimeout = 1 second
+    val noResponseTimeout = 3 seconds
+
+    val testActor = TestActorRef(new IoClientHelperTestActor(ioActorProbe.ref, delegateProbe.ref, backpressureTimeout, noResponseTimeout))
+
+    val commandContext = "context"
+    val command = DefaultIoSizeCommand(mock[Path])
+    val response = IoSuccess(command, 5)
+
+    // Send the command
+    testActor.underlyingActor.sendMessageWithContext(commandContext, command)
+
+    // Io actor receives the command
+    ioActorProbe.expectMsg(commandContext -> command)
+
+    // Io actor replies
+    ioActorProbe.reply(commandContext -> response)
+
+    // delegate should receive the response
+    delegateProbe.expectMsgPF(1 second) {
+      case (contextReceived,  responseReceived) if contextReceived == "context" && responseReceived == response =>
+    }
+
+    // And nothing else, meaning the timeout timer has been cancelled
+    delegateProbe.expectNoMsg()
+
+    // timeouts map should be empty
+    testActor.underlyingActor.timeouts.isEmpty shouldBe true
+  }
+
+  private case object ServiceUnreachable
+  
+  private class IoClientHelperTestActor(override val ioActor: ActorRef,
+                                delegateTo: ActorRef,
+                                override val backpressureTimeout: FiniteDuration,
+                                noResponseTimeout: FiniteDuration) extends Actor with ActorLogging with IoClientHelper with DefaultIoCommandBuilder {
+
+    context.become(ioReceive orElse receive)
+
+    override def receive: Receive = {
+      case message => delegateTo ! message
+    }
+
+    def sendMessage(command: IoCommand[_]) = {
+      sendIoCommandWithCustomTimeout(command, noResponseTimeout)
+    }
+
+    def sendMessageWithContext(context: Any, command: IoCommand[_]) = {
+      sendIoCommandWithContext(command, context, noResponseTimeout)
+    }
+
+    override protected def onTimeout(message: Any, to: ActorRef): Unit = {
+      delegateTo ! ServiceUnreachable
+    }
+  }
+
+}
diff --git a/core/src/test/scala/cromwell/core/labels/LabelSpec.scala b/core/src/test/scala/cromwell/core/labels/LabelSpec.scala
index 47060b6d6..52c331bec 100644
--- a/core/src/test/scala/cromwell/core/labels/LabelSpec.scala
+++ b/core/src/test/scala/cromwell/core/labels/LabelSpec.scala
@@ -10,40 +10,44 @@ class LabelSpec extends FlatSpec with Matchers {
   /**
     * In the format 'to validate', 'expected result'
     */
-  val goodLabelStrings = List(
+  val goodLabelKeys = List(
     "cromwell-root-workflow-id",
     "cromwell-11f2468c-39d6-4be3-85c8-32735c01e66b",
     "just-the-right-length-just-the-right-length-just-the-right-leng"
   )
 
-  val badLabelConversions = List(
-    "11f2468c-39d6-4be3-85c8-32735c01e66b" -> "x--11f2468c-39d6-4be3-85c8-32735c01e66b",
-    "0-cromwell-root-workflow-id" -> "x--0-cromwell-root-workflow-id",
-    "" -> "x----x",
-    "cromwell-root-workflow-id-" -> "cromwell-root-workflow-id---x",
-    "0-cromwell-root-workflow-id-" -> "x--0-cromwell-root-workflow-id---x",
-    "Cromwell-root-workflow-id" -> "cromwell-root-workflow-id",
-    "cromwell_root_workflow_id" -> "cromwell-root-workflow-id",
-    "too-long-too-long-too-long-too-long-too-long-too-long-too-long-t" -> "too-long-too-long-too-long-too---g-too-long-too-long-too-long-t",
-    "0-too-long-and-invalid-too-long-and-invalid-too-long-and-invali+" -> "x--0-too-long-and-invalid-too----nvalid-too-long-and-invali---x"
+  val goodLabelValues = List(
+    "11f2468c-39d6-4be3-85c8-32735c01e66b",
+    ""
   )
 
-  goodLabelStrings foreach { label =>
-    it should s"validate the good label string '$label'" in {
-      Label.validateName(label) should be(Valid(label))
+  val badLabelKeys = List(
+    "11f2468c-39d6-4be3-85c8-32735c01e66b",
+    "0-cromwell-root-workflow-id",
+    "",
+    "cromwell-root-workflow-id-",
+    "0-cromwell-root-workflow-id-",
+    "Cromwell-root-workflow-id"
+  )
+
+  goodLabelKeys foreach { key =>
+    it should s"validate a good label key '$key'" in {
+      Label.validateLabelKey(key) should be(Valid(key))
     }
   }
 
-  badLabelConversions foreach { case (label: String, conversion: String) =>
-    it should s"not validate the bad label string '$label'" in {
-      Label.validateName(label) match {
-        case Invalid(_) => // Good!
-        case Valid(_) => fail(s"Label validation succeeded but should have failed.")
-      }
+  goodLabelValues foreach { value =>
+    it should s"validate a good label value '$value'" in {
+      Label.validateLabelValue(value) should be(Valid(value))
     }
+  }
 
-    it should s"convert the bad label string '$label' into the safe label string '$conversion'" in {
-      Label.safeName(label) should be(conversion)
+  badLabelKeys foreach { key =>
+    it should s"not validate a bad label key $key" in {
+      Label.validateLabelKey(key) match {
+        case Invalid(_) => // Good!
+        case Valid(_) => fail(s"Label key validation succeeded but should have failed.")
+      }
     }
   }
 }
diff --git a/core/src/test/scala/cromwell/core/path/proxy/RetryableFileSystemProxySpec.scala b/core/src/test/scala/cromwell/core/path/proxy/RetryableFileSystemProxySpec.scala
deleted file mode 100644
index 71b0a7c96..000000000
--- a/core/src/test/scala/cromwell/core/path/proxy/RetryableFileSystemProxySpec.scala
+++ /dev/null
@@ -1,280 +0,0 @@
-package cromwell.core.path.proxy
-
-import java.io.FileNotFoundException
-import java.nio.channels.SeekableByteChannel
-import java.nio.file.DirectoryStream.Filter
-import java.nio.file.attribute.{BasicFileAttributes, FileAttributeView}
-import java.nio.file.spi.FileSystemProvider
-import java.nio.file.{DirectoryStream, OpenOption, Path, StandardOpenOption}
-import java.util.concurrent.TimeoutException
-
-import cromwell.core.path.CustomRetryParams
-import cromwell.core.retry.Backoff
-import cromwell.core.{CromwellFatalException, TestKitSuite}
-import org.mockito.Matchers._
-import org.mockito.Mockito._
-import org.mockito.invocation.InvocationOnMock
-import org.mockito.stubbing.Answer
-import org.scalatest.{FlatSpecLike, Matchers}
-
-import scala.concurrent.duration._
-import scala.language.postfixOps
-
-class RetryableFileSystemProxySpec extends TestKitSuite with FlatSpecLike with Matchers {
-
-  behavior of "RetryableFileSystemProxySpec"
-
-  case class ThrowParams(exception: Exception, nbTimes: Int)
-
-  abstract class FileSystemAnswer[T](delay: Option[Duration] = None,
-                                     throws: Option[ThrowParams] = None) extends Answer[T] {
-
-    var nbThrows = 0
-
-    def delayAndOrThrow() = {
-      delay foreach { d => Thread.sleep(d.toMillis) }
-      throws foreach { e =>
-        if (nbThrows < e.nbTimes) {
-          nbThrows = nbThrows + 1
-          throw e.exception
-        }
-      }
-    }
-  }
-
-  def mockFileSystem(delay: Option[Duration] = None,
-                     throws: Option[ThrowParams] = None): FileSystemProvider =  {
-
-    val provider = mock(classOf[FileSystemProvider])
-
-    def answerUnit: Answer[Unit] = new FileSystemAnswer[Unit](delay, throws) {
-      override def answer(invocation: InvocationOnMock): Unit = delayAndOrThrow()
-    }
-
-    def answerBoolean: Answer[Boolean] = new FileSystemAnswer[Boolean](delay, throws) {
-      override def answer(invocation: InvocationOnMock): Boolean = {
-        delayAndOrThrow()
-        true
-      }
-    }
-
-    def answerSeekableByteChannel: Answer[SeekableByteChannel] = new FileSystemAnswer[SeekableByteChannel](delay, throws) {
-      override def answer(invocation: InvocationOnMock): SeekableByteChannel = {
-        delayAndOrThrow()
-        mock(classOf[SeekableByteChannel])
-      }
-    }
-
-    def answerDirectoryStream: Answer[DirectoryStream[Path]] = new FileSystemAnswer[DirectoryStream[Path]](delay, throws) {
-      override def answer(invocation: InvocationOnMock): DirectoryStream[Path] = {
-        delayAndOrThrow()
-        mock(classOf[DirectoryStream[Path]])
-      }
-    }
-
-    def answerBasicFileAttributes: Answer[BasicFileAttributes] = new FileSystemAnswer[BasicFileAttributes](delay, throws) {
-      override def answer(invocation: InvocationOnMock): BasicFileAttributes = {
-        delayAndOrThrow()
-        mock(classOf[BasicFileAttributes])
-      }
-    }
-
-    def answerMap: Answer[java.util.Map[String, AnyRef]] = new FileSystemAnswer[java.util.Map[String, AnyRef]](delay, throws) {
-      override def answer(invocation: InvocationOnMock): java.util.Map[String, AnyRef] = {
-        delayAndOrThrow()
-        new java.util.HashMap[String, AnyRef]()
-      }
-    }
-
-    def answerFileAttributeView: Answer[FileAttributeView] = new FileSystemAnswer[FileAttributeView](delay, throws) {
-      override def answer(invocation: InvocationOnMock): FileAttributeView = {
-        delayAndOrThrow()
-        mock(classOf[FileAttributeView])
-      }
-    }
-
-    import java.nio.file.Path
-
-    when(provider.move(any[Path], any[Path])).thenAnswer(answerUnit)
-    when(provider.checkAccess(any[Path])).thenAnswer(answerUnit)
-    when(provider.createDirectory(any[Path])).thenAnswer(answerUnit)
-    when(provider.newByteChannel(any[Path], any[java.util.Set[OpenOption]])).thenAnswer(answerSeekableByteChannel)
-    when(provider.isHidden(any[Path])).thenAnswer(answerBoolean)
-    when(provider.copy(any[Path], any[Path])).thenAnswer(answerUnit)
-    when(provider.delete(any[Path])).thenAnswer(answerUnit)
-    when(provider.newDirectoryStream(any[Path], any[Filter[Path]]())).thenAnswer(answerDirectoryStream)
-    when(provider.setAttribute(any[Path], any[String], any[Object])).thenAnswer(answerUnit)
-    when(provider.readAttributes(any[Path], any[String])).thenAnswer(answerMap)
-    when(provider.readAttributes(any[Path], any[Class[BasicFileAttributes]])).thenAnswer(answerBasicFileAttributes)
-    when(provider.isSameFile(any[Path], any[Path])).thenAnswer(answerBoolean)
-    when(provider.getFileAttributeView(any[Path], any[Class[FileAttributeView]])).thenAnswer(answerFileAttributeView)
-
-    provider
-  }
-
-  val testRetryParams = CustomRetryParams.Default.copy(backoff = new Backoff {
-    override def next: Backoff = this
-    override def backoffMillis: Long = 0
-  })
-
-  val pathMock = mock(classOf[Path])
-
-  it should "timeout if the operation takes too long" ignore {
-    val retryParams = testRetryParams.copy(timeout = 100 millis)
-    val mockFs = mockFileSystem(delay = Option(200 millis))
-    val retryableFs = new RetryableFileSystemProviderProxy(mockFs, retryParams)(system)
-
-    a[TimeoutException] shouldBe thrownBy(retryableFs.move(pathMock, pathMock))
-    a[TimeoutException] shouldBe thrownBy(retryableFs.checkAccess(pathMock))
-    a[TimeoutException] shouldBe thrownBy(retryableFs.createDirectory(pathMock))
-    a[TimeoutException] shouldBe thrownBy(retryableFs.newByteChannel(pathMock, mock(classOf[java.util.Set[StandardOpenOption]])))
-    a[TimeoutException] shouldBe thrownBy(retryableFs.isHidden(pathMock))
-    a[TimeoutException] shouldBe thrownBy(retryableFs.copy(pathMock, pathMock))
-    a[TimeoutException] shouldBe thrownBy(retryableFs.delete(pathMock))
-    a[TimeoutException] shouldBe thrownBy(retryableFs.newDirectoryStream(pathMock, mock(classOf[Filter[Path]])))
-    a[TimeoutException] shouldBe thrownBy(retryableFs.setAttribute(pathMock, "", ""))
-    a[TimeoutException] shouldBe thrownBy(retryableFs.readAttributes(pathMock, classOf[BasicFileAttributes]))
-    a[TimeoutException] shouldBe thrownBy(retryableFs.readAttributes(pathMock, ""))
-    a[TimeoutException] shouldBe thrownBy(retryableFs.isSameFile(pathMock, pathMock))
-    a[TimeoutException] shouldBe thrownBy(retryableFs.getFileAttributeView(pathMock, classOf[FileAttributeView]))
-  }
-
-  it should "retry on failure and finally succeed if under retry max" in {
-    val retryParams = testRetryParams.copy(maxRetries = Option(4))
-    val mockFs = mockFileSystem(throws = Option(ThrowParams(new Exception(), nbTimes = 2)))
-    val retryableFs = new RetryableFileSystemProviderProxy(mockFs, retryParams)(system)
-
-    retryableFs.move(pathMock, pathMock)
-    retryableFs.checkAccess(pathMock)
-    retryableFs.createDirectory(pathMock)
-    retryableFs.newByteChannel(pathMock, mock(classOf[java.util.Set[StandardOpenOption]]))
-    retryableFs.isHidden(pathMock)
-    retryableFs.copy(pathMock, pathMock)
-    retryableFs.delete(pathMock)
-    retryableFs.newDirectoryStream(pathMock, mock(classOf[Filter[Path]]))
-    retryableFs.setAttribute(pathMock, "", "")
-    retryableFs.readAttributes(pathMock, classOf[BasicFileAttributes])
-    retryableFs.readAttributes(pathMock, "")
-    retryableFs.isSameFile(pathMock, pathMock)
-    retryableFs.getFileAttributeView(pathMock, classOf[FileAttributeView])
-
-    verify(mockFs, times(3)).move(any[Path], any[Path])
-    verify(mockFs, times(3)).checkAccess(any[Path])
-    verify(mockFs, times(3)).createDirectory(any[Path])
-    verify(mockFs, times(3)).newByteChannel(any[Path], any[java.util.Set[OpenOption]])
-    verify(mockFs, times(3)).isHidden(any[Path])
-    verify(mockFs, times(3)).copy(any[Path], any[Path])
-    verify(mockFs, times(3)).delete(any[Path])
-    verify(mockFs, times(3)).newDirectoryStream(any[Path], any[Filter[Path]]())
-    verify(mockFs, times(3)).setAttribute(any[Path], any[String], any[Object])
-    verify(mockFs, times(3)).readAttributes(any[Path], any[String])
-    verify(mockFs, times(3)).readAttributes(any[Path], any[Class[BasicFileAttributes]])
-    verify(mockFs, times(3)).isSameFile(any[Path], any[Path])
-    verify(mockFs, times(3)).getFileAttributeView(any[Path], any[Class[FileAttributeView]])
-  }
-
-  it should "retry on failure and fail if over retry max" in {
-    val retryParams = testRetryParams.copy(maxRetries = Option(2))
-    val mockFs = mockFileSystem(throws = Option(ThrowParams(new IllegalArgumentException(), nbTimes = 3)))
-    val retryableFs = new RetryableFileSystemProviderProxy(mockFs, retryParams)(system)
-
-    (the [CromwellFatalException] thrownBy retryableFs.move(pathMock, pathMock)).getCause shouldBe a[IllegalArgumentException]
-    (the [CromwellFatalException] thrownBy retryableFs.checkAccess(pathMock)).getCause shouldBe a[IllegalArgumentException]
-    (the [CromwellFatalException] thrownBy retryableFs.createDirectory(pathMock)).getCause shouldBe a[IllegalArgumentException]
-    (the [CromwellFatalException] thrownBy retryableFs.newByteChannel(pathMock, mock(classOf[java.util.Set[StandardOpenOption]]))).getCause shouldBe a[IllegalArgumentException]
-    (the [CromwellFatalException] thrownBy retryableFs.isHidden(pathMock)).getCause shouldBe a[IllegalArgumentException]
-    (the [CromwellFatalException] thrownBy retryableFs.copy(pathMock, pathMock)).getCause shouldBe a[IllegalArgumentException]
-    (the [CromwellFatalException] thrownBy retryableFs.delete(pathMock)).getCause shouldBe a[IllegalArgumentException]
-    (the [CromwellFatalException] thrownBy retryableFs.newDirectoryStream(pathMock, mock(classOf[Filter[Path]]))).getCause shouldBe a[IllegalArgumentException]
-    (the [CromwellFatalException] thrownBy retryableFs.setAttribute(pathMock, "", "")).getCause shouldBe a[IllegalArgumentException]
-    (the [CromwellFatalException] thrownBy retryableFs.readAttributes(pathMock, classOf[BasicFileAttributes])).getCause shouldBe a[IllegalArgumentException]
-    (the [CromwellFatalException] thrownBy retryableFs.readAttributes(pathMock, "")).getCause shouldBe a[IllegalArgumentException]
-    (the [CromwellFatalException] thrownBy retryableFs.isSameFile(pathMock, pathMock)).getCause shouldBe a[IllegalArgumentException]
-    (the [CromwellFatalException] thrownBy retryableFs.getFileAttributeView(pathMock, classOf[FileAttributeView])).getCause shouldBe a[IllegalArgumentException]
-
-    verify(mockFs, times(3)).move(any[Path], any[Path])
-    verify(mockFs, times(3)).checkAccess(any[Path])
-    verify(mockFs, times(3)).createDirectory(any[Path])
-    verify(mockFs, times(3)).newByteChannel(any[Path], any[java.util.Set[OpenOption]])
-    verify(mockFs, times(3)).isHidden(any[Path])
-    verify(mockFs, times(3)).copy(any[Path], any[Path])
-    verify(mockFs, times(3)).delete(any[Path])
-    verify(mockFs, times(3)).newDirectoryStream(any[Path], any[Filter[Path]]())
-    verify(mockFs, times(3)).setAttribute(any[Path], any[String], any[Object])
-    verify(mockFs, times(3)).readAttributes(any[Path], any[String])
-    verify(mockFs, times(3)).readAttributes(any[Path], any[Class[BasicFileAttributes]])
-    verify(mockFs, times(3)).isSameFile(any[Path], any[Path])
-    verify(mockFs, times(3)).getFileAttributeView(any[Path], any[Class[FileAttributeView]])
-  }
-
-  it should "ignore transient exceptions" in {
-    def isTransient(t: Throwable) = t.isInstanceOf[FileNotFoundException]
-    val retryParams = testRetryParams.copy(maxRetries = Option(1), isTransient = isTransient)
-    val mockFs = mockFileSystem(throws = Option(ThrowParams(new FileNotFoundException(), nbTimes = 2)))
-    val retryableFs = new RetryableFileSystemProviderProxy(mockFs, retryParams)(system)
-
-    retryableFs.move(pathMock, pathMock)
-    retryableFs.checkAccess(pathMock)
-    retryableFs.createDirectory(pathMock)
-    retryableFs.newByteChannel(pathMock, mock(classOf[java.util.Set[StandardOpenOption]]))
-    retryableFs.isHidden(pathMock)
-    retryableFs.copy(pathMock, pathMock)
-    retryableFs.delete(pathMock)
-    retryableFs.newDirectoryStream(pathMock, mock(classOf[Filter[Path]]))
-    retryableFs.setAttribute(pathMock, "", "")
-    retryableFs.readAttributes(pathMock, classOf[BasicFileAttributes])
-    retryableFs.readAttributes(pathMock, "")
-    retryableFs.isSameFile(pathMock, pathMock)
-    retryableFs.getFileAttributeView(pathMock, classOf[FileAttributeView])
-
-    verify(mockFs, times(3)).move(any[Path], any[Path])
-    verify(mockFs, times(3)).checkAccess(any[Path])
-    verify(mockFs, times(3)).createDirectory(any[Path])
-    verify(mockFs, times(3)).newByteChannel(any[Path], any[java.util.Set[OpenOption]])
-    verify(mockFs, times(3)).isHidden(any[Path])
-    verify(mockFs, times(3)).copy(any[Path], any[Path])
-    verify(mockFs, times(3)).delete(any[Path])
-    verify(mockFs, times(3)).newDirectoryStream(any[Path], any[Filter[Path]]())
-    verify(mockFs, times(3)).setAttribute(any[Path], any[String], any[Object])
-    verify(mockFs, times(3)).readAttributes(any[Path], any[String])
-    verify(mockFs, times(3)).readAttributes(any[Path], any[Class[BasicFileAttributes]])
-    verify(mockFs, times(3)).isSameFile(any[Path], any[Path])
-    verify(mockFs, times(3)).getFileAttributeView(any[Path], any[Class[FileAttributeView]])
-  }
-
-  it should "fail immediately on fatal exceptions" in {
-    def isFatal(t: Throwable) = t.isInstanceOf[FileNotFoundException]
-    val retryParams = testRetryParams.copy(maxRetries = Option(5), isFatal = isFatal)
-    val mockFs = mockFileSystem(throws = Option(ThrowParams(new FileNotFoundException(), nbTimes = 3)))
-    val retryableFs = new RetryableFileSystemProviderProxy(mockFs, retryParams)(system)
-
-    (the [CromwellFatalException] thrownBy retryableFs.move(pathMock, pathMock)).getCause shouldBe a[FileNotFoundException]
-    (the [CromwellFatalException] thrownBy retryableFs.checkAccess(pathMock)).getCause shouldBe a[FileNotFoundException]
-    (the [CromwellFatalException] thrownBy retryableFs.createDirectory(pathMock)).getCause shouldBe a[FileNotFoundException]
-    (the [CromwellFatalException] thrownBy retryableFs.newByteChannel(pathMock, mock(classOf[java.util.Set[StandardOpenOption]]))).getCause shouldBe a[FileNotFoundException]
-    (the [CromwellFatalException] thrownBy retryableFs.isHidden(pathMock)).getCause shouldBe a[FileNotFoundException]
-    (the [CromwellFatalException] thrownBy retryableFs.copy(pathMock, pathMock)).getCause shouldBe a[FileNotFoundException]
-    (the [CromwellFatalException] thrownBy retryableFs.delete(pathMock)).getCause shouldBe a[FileNotFoundException]
-    (the [CromwellFatalException] thrownBy retryableFs.newDirectoryStream(pathMock, mock(classOf[Filter[Path]]))).getCause shouldBe a[FileNotFoundException]
-    (the [CromwellFatalException] thrownBy retryableFs.setAttribute(pathMock, "", "")).getCause shouldBe a[FileNotFoundException]
-    (the [CromwellFatalException] thrownBy retryableFs.readAttributes(pathMock, classOf[BasicFileAttributes])).getCause shouldBe a[FileNotFoundException]
-    (the [CromwellFatalException] thrownBy retryableFs.readAttributes(pathMock, "")).getCause shouldBe a[FileNotFoundException]
-    (the [CromwellFatalException] thrownBy retryableFs.isSameFile(pathMock, pathMock)).getCause shouldBe a[FileNotFoundException]
-    (the [CromwellFatalException] thrownBy retryableFs.getFileAttributeView(pathMock, classOf[FileAttributeView])).getCause shouldBe a[FileNotFoundException]
-
-    verify(mockFs, times(1)).move(any[Path], any[Path])
-    verify(mockFs, times(1)).checkAccess(any[Path])
-    verify(mockFs, times(1)).createDirectory(any[Path])
-    verify(mockFs, times(1)).newByteChannel(any[Path], any[java.util.Set[OpenOption]])
-    verify(mockFs, times(1)).isHidden(any[Path])
-    verify(mockFs, times(1)).copy(any[Path], any[Path])
-    verify(mockFs, times(1)).delete(any[Path])
-    verify(mockFs, times(1)).newDirectoryStream(any[Path], any[Filter[Path]]())
-    verify(mockFs, times(1)).setAttribute(any[Path], any[String], any[Object])
-    verify(mockFs, times(1)).readAttributes(any[Path], any[String])
-    verify(mockFs, times(1)).readAttributes(any[Path], any[Class[BasicFileAttributes]])
-    verify(mockFs, times(1)).isSameFile(any[Path], any[Path])
-    verify(mockFs, times(1)).getFileAttributeView(any[Path], any[Class[FileAttributeView]])
-  }
-
-}
diff --git a/core/src/test/scala/cromwell/core/retry/RetrySpec.scala b/core/src/test/scala/cromwell/core/retry/RetrySpec.scala
index 27f24076c..f62b49d47 100644
--- a/core/src/test/scala/cromwell/core/retry/RetrySpec.scala
+++ b/core/src/test/scala/cromwell/core/retry/RetrySpec.scala
@@ -34,7 +34,7 @@ class RetrySpec extends TestKitSuite("retry-spec") with FlatSpecLike with Matche
                        isFatal: Throwable => Boolean = Retry.throwableToFalse): Future[Int] = {
 
     withRetry(
-      f = work.doIt,
+      f = () => work.doIt(),
       maxRetries = Option(retries),
       isTransient = isTransient,
       isFatal = isFatal
diff --git a/core/src/test/scala/cromwell/core/simpleton/WdlValueBuilderSpec.scala b/core/src/test/scala/cromwell/core/simpleton/WdlValueBuilderSpec.scala
index 1e558ceb1..514051e22 100644
--- a/core/src/test/scala/cromwell/core/simpleton/WdlValueBuilderSpec.scala
+++ b/core/src/test/scala/cromwell/core/simpleton/WdlValueBuilderSpec.scala
@@ -4,9 +4,9 @@ import cromwell.core.simpleton.WdlValueBuilderSpec._
 import org.scalatest.{FlatSpec, Matchers}
 import org.specs2.mock.Mockito
 import wdl4s.parser.WdlParser.Ast
-import wdl4s.types.{WdlArrayType, WdlIntegerType, WdlMapType, WdlStringType}
-import wdl4s.values.{WdlArray, WdlInteger, WdlMap, WdlPair, WdlString, WdlValue}
-import wdl4s.{TaskOutput, WdlExpression}
+import wdl4s.wdl.types.{WdlArrayType, WdlIntegerType, WdlMapType, WdlStringType}
+import wdl4s.wdl.values.{WdlArray, WdlInteger, WdlMap, WdlPair, WdlString, WdlValue}
+import wdl4s.wdl.{TaskOutput, WdlExpression}
 
 object WdlValueBuilderSpec {
   // WdlValueBuilder doesn't care about this expression, but something needs to be passed to the TaskOutput constructor.
@@ -115,9 +115,9 @@ class WdlValueBuilderSpec extends FlatSpec with Matchers with Mockito {
 
   it should "round trip everything together with no losses" in {
 
-    val wdlValues = (simpletonConversions map { case SimpletonConversion(name, wdlValue, simpletons) => name -> wdlValue }).toMap
+    val wdlValues = (simpletonConversions map { case SimpletonConversion(name, wdlValue, _) => name -> wdlValue }).toMap
     val taskOutputs = wdlValues map { case (k, wv) => TaskOutput(k, wv.wdlType, IgnoredExpression, mock[Ast], None) }
-    val allSimpletons = simpletonConversions flatMap { case SimpletonConversion(name, wdlValue, simpletons) => simpletons }
+    val allSimpletons = simpletonConversions flatMap { case SimpletonConversion(_, _, simpletons) => simpletons }
 
     import WdlValueSimpleton._
 
diff --git a/core/src/test/scala/cromwell/util/AkkaTestUtil.scala b/core/src/test/scala/cromwell/util/AkkaTestUtil.scala
index 10b05dc2b..1633a7d1d 100644
--- a/core/src/test/scala/cromwell/util/AkkaTestUtil.scala
+++ b/core/src/test/scala/cromwell/util/AkkaTestUtil.scala
@@ -8,7 +8,7 @@ object AkkaTestUtil {
   implicit class EnhancedTestProbe(probe: TestProbe) {
     def props = Props(new Actor with ActorLogging {
       def receive = {
-        case outbound if sender == probe.ref =>
+        case outbound @ _ if sender == probe.ref =>
           val msg = "Unexpected outbound message from Probe. You're doing something wrong!"
           log.error(msg)
           throw new RuntimeException(msg)
diff --git a/core/src/test/scala/cromwell/util/GracefulShutdownHelperSpec.scala b/core/src/test/scala/cromwell/util/GracefulShutdownHelperSpec.scala
new file mode 100644
index 000000000..4d93073dc
--- /dev/null
+++ b/core/src/test/scala/cromwell/util/GracefulShutdownHelperSpec.scala
@@ -0,0 +1,42 @@
+package cromwell.util
+
+import akka.actor.{Actor, ActorLogging, Props}
+import akka.testkit.TestProbe
+import cats.data.NonEmptyList
+import cromwell.core.TestKitSuite
+import cromwell.util.GracefulShutdownHelper.ShutdownCommand
+import org.scalatest.{FlatSpecLike, Matchers}
+
+class GracefulShutdownHelperSpec extends TestKitSuite with FlatSpecLike with Matchers {
+  behavior of "GracefulShutdownHelper"
+  
+  it should "send ShutdownCommand to actors, wait for them to shutdown, then shut itself down" in {
+    val testProbeA = TestProbe()
+    val testProbeB = TestProbe()
+    
+    val testActor = system.actorOf(Props(new Actor with GracefulShutdownHelper with ActorLogging {
+      override def receive: Receive = {
+        case ShutdownCommand => waitForActorsAndShutdown(NonEmptyList.of(testProbeA.ref, testProbeB.ref))
+      }
+    }))
+    
+    watch(testActor)
+
+    testActor ! ShutdownCommand
+
+    testProbeA.expectMsg(ShutdownCommand)
+    testProbeB.expectMsg(ShutdownCommand)
+
+    // Make sure it's still alive
+    expectNoMsg()
+
+    system stop testProbeA.ref
+
+    // Make sure it's still alive
+    expectNoMsg()
+
+    system stop testProbeB.ref
+    
+    expectTerminated(testActor)
+  }
+}
diff --git a/core/src/test/scala/cromwell/util/SampleWdl.scala b/core/src/test/scala/cromwell/util/SampleWdl.scala
index 5a3940b81..b7d07fee9 100644
--- a/core/src/test/scala/cromwell/util/SampleWdl.scala
+++ b/core/src/test/scala/cromwell/util/SampleWdl.scala
@@ -4,18 +4,29 @@ import java.util.UUID
 
 import cromwell.core.WorkflowSourceFilesWithoutImports
 import cromwell.core.path.{DefaultPathBuilder, Path}
-import cromwell.core.WorkflowSourceFilesWithoutImports
 import spray.json._
-import wdl4s._
-import wdl4s.types.{WdlArrayType, WdlStringType}
-import wdl4s.values._
+import wdl4s.wdl.types.{WdlArrayType, WdlStringType}
+import wdl4s.wdl.values._
+import wdl4s.wdl.{WorkflowJson, WorkflowRawInputs, WorkflowSource}
 
 import scala.language.postfixOps
 
 trait SampleWdl extends TestFileUtil {
-  def wdlSource(runtime: String = ""): WdlSource
-  def asWorkflowSources(runtime: String = "", workflowOptions: String = "{}", labels: String = "{}") =
-    WorkflowSourceFilesWithoutImports(wdlSource = wdlSource(runtime), inputsJson = wdlJson, workflowOptionsJson = workflowOptions, labelsJson = labels)
+  def workflowSource(runtime: String = ""): WorkflowSource
+  def asWorkflowSources(runtime: String = "",
+                        workflowOptions: String = "{}",
+                        labels: String = "{}",
+                        workflowType: Option[String] = Option("WDL"),
+                        workflowTypeVersion: Option[String] = None) = {
+    WorkflowSourceFilesWithoutImports(
+      workflowSource = workflowSource(runtime),
+      inputsJson = workflowJson,
+      workflowOptionsJson = workflowOptions,
+      labelsJson = labels,
+      workflowType = workflowType,
+      workflowTypeVersion = workflowTypeVersion)
+  }
+
   val rawInputs: WorkflowRawInputs
 
   def name = getClass.getSimpleName.stripSuffix("$")
@@ -54,7 +65,7 @@ trait SampleWdl extends TestFileUtil {
     def read(value: JsValue) = throw new NotImplementedError(s"Reading JSON not implemented: $value")
   }
 
-  def wdlJson: WdlJson = rawInputs.toJson.prettyPrint
+  def workflowJson: WorkflowJson = rawInputs.toJson.prettyPrint
 
   def deleteFile(path: Path) = path.delete()
 }
@@ -62,7 +73,7 @@ trait SampleWdl extends TestFileUtil {
 object SampleWdl {
 
   object HelloWorld extends SampleWdl {
-    override def wdlSource(runtime: String = "") =
+    override def workflowSource(runtime: String = "") =
       s"""
         |task hello {
         |  String addressee
@@ -87,7 +98,7 @@ object SampleWdl {
   }
 
   object HelloWorldWithoutWorkflow extends SampleWdl {
-    override def wdlSource(runtime: String = "") =
+    override def workflowSource(runtime: String = "") =
       s"""
         |task hello {
         |  String addressee
@@ -107,7 +118,7 @@ object SampleWdl {
   }
 
   object GoodbyeWorld extends SampleWdl {
-    override def wdlSource(runtime: String = "") =
+    override def workflowSource(runtime: String = "") =
       """
         |task goodbye {
         |  command {
@@ -128,7 +139,7 @@ object SampleWdl {
   }
 
   object EmptyString extends SampleWdl {
-    override def wdlSource(runtime: String = "") =
+    override def workflowSource(runtime: String = "") =
       s"""
         |task hello {
         |  command {
@@ -170,13 +181,13 @@ object SampleWdl {
 
 
   object EmptyWorkflow extends SampleWdl {
-    override def wdlSource(runtime: String = "") = "workflow empty_workflow {}"
+    override def workflowSource(runtime: String = "") = "workflow empty_workflow {}"
 
     val rawInputs = Map.empty[String, Any]
   }
 
   object CoercionNotDefined extends SampleWdl {
-    override def wdlSource(runtime: String = "") = {
+    override def workflowSource(runtime: String = "") = {
       s"""
         |task summary {
         |  String bfile
@@ -207,7 +218,7 @@ object SampleWdl {
   }
 
   trait ThreeStepTemplate extends SampleWdl {
-    override def wdlSource(runtime: String = "") = sourceString().replaceAll("RUNTIME", runtime)
+    override def workflowSource(runtime: String = "") = sourceString().replaceAll("RUNTIME", runtime)
     private val outputSectionPlaceholder = "OUTPUTSECTIONPLACEHOLDER"
     def sourceString(outputsSection: String = "") = {
       val withPlaceholders =
@@ -268,7 +279,7 @@ object SampleWdl {
   object ThreeStep extends ThreeStepTemplate
 
   object ThreeStepWithOutputsSection extends ThreeStepTemplate {
-    override def wdlSource(runtime: String = "") = sourceString(outputsSection =
+    override def workflowSource(runtime: String = "") = sourceString(outputsSection =
       """
         |output {
         | cgrep.count
@@ -278,7 +289,7 @@ object SampleWdl {
   }
 
   object ThreeStepWithInputsInTheOutputsSection extends ThreeStepTemplate {
-    override def wdlSource(runtime: String = "") = sourceString(outputsSection =
+    override def workflowSource(runtime: String = "") = sourceString(outputsSection =
       """
         |output {
         | cgrep.pattern
@@ -293,7 +304,7 @@ object SampleWdl {
   object WorkflowOutputsWithFiles extends SampleWdl {
     // ASCII art from http://www.chris.com/ascii/joan/www.geocities.com/SoHo/7373/flag.html with pipes
     // replaced by exclamation points to keep stripMargin from removing the flagpole.
-    override def wdlSource(runtime: String = "") =
+    override def workflowSource(runtime: String = "") =
       """
         task A {
           command {
@@ -360,7 +371,7 @@ object SampleWdl {
   }
 
   object WorkflowScatterOutputsWithFileArrays extends SampleWdl {
-    override def wdlSource(runtime: String = "") =
+    override def workflowSource(runtime: String = "") =
       """
         |task A {
         |  command {
@@ -387,7 +398,7 @@ object SampleWdl {
 
 
   object DeclarationsWorkflow extends SampleWdl {
-    override def wdlSource(runtime: String): WdlSource =
+    override def workflowSource(runtime: String): WorkflowSource =
       s"""
         |task cat {
         |  File file
@@ -441,7 +452,7 @@ object SampleWdl {
   }
 
   trait ZeroOrMorePostfixQuantifier extends SampleWdl {
-    override def wdlSource(runtime: String): WdlSource =
+    override def workflowSource(runtime: String): WorkflowSource =
       s"""
         |task hello {
         |  Array[String] person
@@ -472,7 +483,7 @@ object SampleWdl {
   }
 
   trait OneOrMorePostfixQuantifier extends SampleWdl {
-    override def wdlSource(runtime: String): WdlSource =
+    override def workflowSource(runtime: String): WorkflowSource =
       s"""
         |task hello {
         |  Array[String]+ person
@@ -499,7 +510,7 @@ object SampleWdl {
   }
 
   object CurrentDirectory extends SampleWdl {
-    override def wdlSource(runtime: String): String =
+    override def workflowSource(runtime: String): String =
       """
         |task whereami {
         |  command {
@@ -520,7 +531,7 @@ object SampleWdl {
   }
 
   object ArrayIO extends SampleWdl {
-    override def wdlSource(runtime: String = "") =
+    override def workflowSource(runtime: String = "") =
       s"""
         |task serialize {
         |  Array[String] strs
@@ -547,7 +558,7 @@ object SampleWdl {
     createFileArray(catRootDir)
     def cleanup() = cleanupFileArray(catRootDir)
 
-    override def wdlSource(runtime: String = "") =
+    override def workflowSource(runtime: String = "") =
       s"""
         |task cat {
         |  Array[File]+ files
@@ -572,7 +583,7 @@ object SampleWdl {
     createFileArray(catRootDir)
     def cleanup() = cleanupFileArray(catRootDir)
 
-    override def wdlSource(runtime: String = "") =
+    override def workflowSource(runtime: String = "") =
       s"""
         |task write_map {
         |  Map[File, String] file_to_name
@@ -661,7 +672,7 @@ object SampleWdl {
       |}
     """.stripMargin
 
-    override def wdlSource(runtime: String = "") =
+    override def workflowSource(runtime: String = "") =
       s"""$tasks
         |
         |workflow w {
@@ -679,7 +690,7 @@ object SampleWdl {
   }
 
   object SiblingsScatterWdl extends ScatterWdl {
-    override def wdlSource(runtime: String = "") =
+    override def workflowSource(runtime: String = "") =
       s"""$tasks
         |
         |workflow w {
@@ -700,7 +711,7 @@ object SampleWdl {
   }
 
   object SimpleScatterWdl extends SampleWdl {
-    override def wdlSource(runtime: String = "") =
+    override def workflowSource(runtime: String = "") =
       s"""task echo_int {
         |  Int int
         |  command {echo $${int}}
@@ -723,7 +734,7 @@ object SampleWdl {
   }
 
   object SimpleScatterWdlWithOutputs extends SampleWdl {
-    override def wdlSource(runtime: String = "") =
+    override def workflowSource(runtime: String = "") =
       s"""task echo_int {
         |  Int int
         |  command {echo $${int}}
@@ -748,7 +759,7 @@ object SampleWdl {
   }
 
   case class PrepareScatterGatherWdl(salt: String = UUID.randomUUID().toString) extends SampleWdl {
-    override def wdlSource(runtime: String = "") = {
+    override def workflowSource(runtime: String = "") = {
       s"""
         |#
         |# Goal here is to split up the input file into files of 1 line each (in the prepare) then in parallel call wc -w on each newly created file and count the words into another file then in the gather, sum the results of each parallel call to come up with
@@ -816,7 +827,7 @@ object SampleWdl {
   }
 
   object FileClobber extends SampleWdl {
-    override def wdlSource(runtime: String = "") =
+    override def workflowSource(runtime: String = "") =
       s"""task read_line {
         |  File in
         |  command { cat $${in} }
@@ -841,7 +852,7 @@ object SampleWdl {
   }
 
   object FilePassingWorkflow extends SampleWdl {
-    override def wdlSource(runtime: String): WdlSource =
+    override def workflowSource(runtime: String): WorkflowSource =
       s"""task a {
         |  File in
         |  String out_name = "out"
@@ -881,7 +892,7 @@ object SampleWdl {
     *               different
     */
   case class CallCachingWorkflow(salt: String) extends SampleWdl {
-    override def wdlSource(runtime: String): WdlSource =
+    override def workflowSource(runtime: String): WorkflowSource =
       s"""task a {
         |  File in
         |  String out_name = "out"
@@ -933,7 +944,7 @@ object SampleWdl {
         |k3\tv3
       """.stripMargin.trim
 
-    override def wdlSource(runtime: String): WdlSource =
+    override def workflowSource(runtime: String): WorkflowSource =
       s"""
         |task a {
         |  Array[String] array
@@ -969,7 +980,7 @@ object SampleWdl {
   }
 
   object ArrayOfArrays extends SampleWdl {
-    override def wdlSource(runtime: String = "") =
+    override def workflowSource(runtime: String = "") =
       s"""task subtask {
         |  Array[File] a
         |  command {
@@ -1009,7 +1020,7 @@ object SampleWdl {
   }
 
   object CallCachingHashingWdl extends SampleWdl {
-    override def wdlSource(runtime: String): WdlSource =
+    override def workflowSource(runtime: String): WorkflowSource =
       s"""task t {
         |  Int a
         |  Float b
@@ -1047,7 +1058,7 @@ object SampleWdl {
   }
 
   object ExpressionsInInputs extends SampleWdl {
-    override def wdlSource(runtime: String = "") =
+    override def workflowSource(runtime: String = "") =
       s"""task echo {
         |  String inString
         |  command {
@@ -1077,7 +1088,7 @@ object SampleWdl {
   }
 
   object WorkflowFailSlow extends SampleWdl {
-    override def wdlSource(runtime: String = "") =
+    override def workflowSource(runtime: String = "") =
       s"""
 task shouldCompleteFast {
         |    Int a
diff --git a/core/src/test/scala/cromwell/util/TestFileUtil.scala b/core/src/test/scala/cromwell/util/TestFileUtil.scala
index a6bedd490..6f0d08a1f 100644
--- a/core/src/test/scala/cromwell/util/TestFileUtil.scala
+++ b/core/src/test/scala/cromwell/util/TestFileUtil.scala
@@ -3,7 +3,7 @@ package cromwell.util
 import java.nio.file.attribute.PosixFilePermission
 
 import cromwell.core.path.{DefaultPathBuilder, Path}
-import wdl4s.values._
+import wdl4s.wdl.values._
 
 trait TestFileUtil {
   def createCannedFile(prefix: String, contents: String, dir: Option[Path] = None): Path = {
diff --git a/core/src/test/scala/cromwell/util/WdlValueJsonFormatterSpec.scala b/core/src/test/scala/cromwell/util/WdlValueJsonFormatterSpec.scala
index 91d678c01..7924f30ab 100644
--- a/core/src/test/scala/cromwell/util/WdlValueJsonFormatterSpec.scala
+++ b/core/src/test/scala/cromwell/util/WdlValueJsonFormatterSpec.scala
@@ -1,14 +1,10 @@
 package cromwell.util
 
-import scala.Vector
-
-import org.scalatest.FlatSpec
-import org.scalatest.Matchers
-
-import JsonFormatting.WdlValueJsonFormatter.WdlValueJsonFormat
-import spray.json.{ JsObject, pimpString }
-import wdl4s.types.{ WdlArrayType, WdlStringType }
-import wdl4s.values.{ WdlArray, WdlPair, WdlString }
+import cromwell.util.JsonFormatting.WdlValueJsonFormatter.WdlValueJsonFormat
+import org.scalatest.{FlatSpec, Matchers}
+import spray.json.{JsObject, pimpString}
+import wdl4s.wdl.types.{WdlArrayType, WdlStringType}
+import wdl4s.wdl.values.{WdlArray, WdlPair, WdlString}
 
 class WdlValueJsonFormatterSpec extends FlatSpec with Matchers {
 
diff --git a/cromwellApiClient/src/main/scala/cromwell/api/CromwellClient.scala b/cromwellApiClient/src/main/scala/cromwell/api/CromwellClient.scala
new file mode 100644
index 000000000..71c0d813f
--- /dev/null
+++ b/cromwellApiClient/src/main/scala/cromwell/api/CromwellClient.scala
@@ -0,0 +1,173 @@
+package cromwell.api
+
+import java.net.URL
+
+import akka.http.scaladsl.Http
+import akka.actor.ActorSystem
+import akka.http.scaladsl.coding.{Deflate, Gzip, NoCoding}
+import akka.http.scaladsl.model.{HttpEntity, _}
+import akka.http.scaladsl.model.headers.HttpEncodings
+import akka.http.scaladsl.unmarshalling.{Unmarshal, Unmarshaller}
+import akka.stream.ActorMaterializer
+import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._
+import akka.util.ByteString
+import cromwell.api.model._
+import spray.json._
+
+import scala.concurrent.{ExecutionContext, Future}
+import cromwell.api.CromwellClient._
+
+import scala.util.{Failure, Success, Try}
+
+class CromwellClient(val cromwellUrl: URL, val apiVersion: String)(implicit actorSystem: ActorSystem, materializer: ActorMaterializer) {
+
+  lazy val engineEndpoint = s"$cromwellUrl/engine/$apiVersion"
+  lazy val submitEndpoint = s"$cromwellUrl/api/workflows/$apiVersion"
+  // Everything else is a suffix off the submit endpoint:
+  lazy val batchSubmitEndpoint = s"$submitEndpoint/batch"
+  private def workflowSpecificEndpoint(workflowId: WorkflowId, endpoint: String) = s"$submitEndpoint/$workflowId/$endpoint"
+  def abortEndpoint(workflowId: WorkflowId): String = workflowSpecificEndpoint(workflowId, "abort")
+  def statusEndpoint(workflowId: WorkflowId): String = workflowSpecificEndpoint(workflowId, "status")
+  def metadataEndpoint(workflowId: WorkflowId): String = workflowSpecificEndpoint(workflowId, "metadata")
+  def outputsEndpoint(workflowId: WorkflowId): String = workflowSpecificEndpoint(workflowId, "outputs")
+  def logsEndpoint(workflowId: WorkflowId): String = workflowSpecificEndpoint(workflowId, "logs")
+  def diffEndpoint(workflowA: WorkflowId, callA: String, indexA: ShardIndex, workflowB: WorkflowId, callB: String, indexB: ShardIndex): String = {
+    def shardParam(aOrB: String, s: ShardIndex) = s.index.map(i => s"&index$aOrB=$i.toString").getOrElse("")
+    s"$submitEndpoint/callcaching/diff?workflowA=$workflowA&callA=$callA&workflowB=$workflowB&callB=$callB${shardParam("A", indexA)}${shardParam("B", indexB)}"
+  }
+  lazy val backendsEndpoint = s"$submitEndpoint/backends"
+  lazy val versionEndpoint = s"$engineEndpoint/version"
+
+  import model.CromwellStatusJsonSupport._
+  import model.WorkflowOutputsJsonSupport._
+  import model.WorkflowLogsJsonSupport._
+  import model.CromwellBackendsJsonSupport._
+  import model.CromwellVersionJsonSupport._
+  import model.CallCacheDiffJsonSupport._
+
+  private def requestEntityForSubmit(workflowSubmission: WorkflowSubmission) = {
+    import cromwell.api.model.LabelsJsonFormatter._
+
+    val sourceBodyParts = Map(
+      "workflowSource" -> Option(workflowSubmission.wdl),
+      "workflowType" -> workflowSubmission.workflowType,
+      "workflowTypeVersion" -> workflowSubmission.workflowTypeVersion,
+      "workflowInputs" -> workflowSubmission.inputsJson,
+      "workflowOptions" -> insertSecrets(workflowSubmission.options, workflowSubmission.refreshToken),
+      "customLabels" -> Option(workflowSubmission.customLabels.toJson.toString)
+    ) collect { case (name, Some(source: String)) => Multipart.FormData.BodyPart(name, HttpEntity(MediaTypes.`application/json`, ByteString(source))) }
+    val zipBodyParts = Map(
+      "workflowDependencies" -> workflowSubmission.zippedImports
+    ) collect { case (name, Some(file)) => Multipart.FormData.BodyPart.fromPath(name, MediaTypes.`application/zip`, file.path) }
+
+    val multipartFormData = Multipart.FormData((sourceBodyParts ++ zipBodyParts).toSeq : _*)
+    multipartFormData.toEntity()
+  }
+
+  def submit(workflow: WorkflowSubmission)(implicit ec: ExecutionContext): Future[SubmittedWorkflow] = {
+    val requestEntity = requestEntityForSubmit(workflow)
+
+    makeRequest[CromwellStatus](HttpRequest(HttpMethods.POST, submitEndpoint, List.empty[HttpHeader], requestEntity)) map { status =>
+      SubmittedWorkflow(WorkflowId.fromString(status.id), cromwellUrl, workflow)
+    }
+  }
+
+  def submitBatch(workflow: WorkflowBatchSubmission)(implicit ec: ExecutionContext): Future[List[SubmittedWorkflow]] = {
+    import DefaultJsonProtocol._
+
+    val requestEntity = requestEntityForSubmit(workflow)
+
+    // Make a set of submissions that represent the batch (so we can zip with the results later):
+    val submissionSet = workflow.inputsBatch.map(inputs => WorkflowSingleSubmission(
+      wdl = workflow.wdl,
+      workflowType = workflow.workflowType,
+      workflowTypeVersion = workflow.workflowTypeVersion,
+      inputsJson = Option(inputs),
+      options = workflow.options,
+      customLabels = workflow.customLabels,
+      zippedImports = workflow.zippedImports,
+      refreshToken = workflow.refreshToken))
+
+    makeRequest[List[CromwellStatus]](HttpRequest(HttpMethods.POST, batchSubmitEndpoint, List.empty[HttpHeader], requestEntity)) map { statuses =>
+      val zipped = submissionSet.zip(statuses)
+      zipped map { case (submission, status) =>
+        SubmittedWorkflow(WorkflowId.fromString(status.id), cromwellUrl, submission)
+      }
+    }
+  }
+
+  def abort(workflowId: WorkflowId)(implicit ec: ExecutionContext): Future[WorkflowStatus] = simpleRequest[CromwellStatus](uri = abortEndpoint(workflowId), method = HttpMethods.POST) map WorkflowStatus.apply
+  def status(workflowId: WorkflowId)(implicit ec: ExecutionContext): Future[WorkflowStatus] = simpleRequest[CromwellStatus](statusEndpoint(workflowId)) map WorkflowStatus.apply
+  def metadata(workflowId: WorkflowId)(implicit ec: ExecutionContext): Future[WorkflowMetadata] = simpleRequest[String](metadataEndpoint(workflowId)) map WorkflowMetadata
+  def outputs(workflowId: WorkflowId)(implicit ec: ExecutionContext): Future[WorkflowOutputs] = simpleRequest[WorkflowOutputs](outputsEndpoint(workflowId))
+  def logs(workflowId: WorkflowId)(implicit ec: ExecutionContext): Future[WorkflowLogs] = simpleRequest[WorkflowLogsStruct](outputsEndpoint(workflowId)) map WorkflowLogs.apply
+  def callCacheDiff(workflowA: WorkflowId, callA: String, shardIndexA: ShardIndex, workflowB: WorkflowId, callB: String, shardIndexB: ShardIndex)(implicit ec: ExecutionContext): Future[CallCacheDiff] =
+    simpleRequest[CallCacheDiff](diffEndpoint(workflowA, callA, shardIndexA, workflowB, callB, shardIndexB))
+  def backends(implicit ec: ExecutionContext): Future[CromwellBackends] = simpleRequest[CromwellBackends](backendsEndpoint)
+  def version(implicit ec: ExecutionContext): Future[CromwellVersion] = simpleRequest[CromwellVersion](versionEndpoint)
+
+  private [api] def executeRequest(request: HttpRequest) = Http().singleRequest(request)
+
+  /**
+    *
+    * @tparam A The type of response expected. Must be supported by an implicit unmarshaller from ResponseEntity.
+    */
+  private def makeRequest[A](request: HttpRequest)(implicit um: Unmarshaller[ResponseEntity, A], ec: ExecutionContext): Future[A] = for {
+    response <- executeRequest(request)
+    decoded <- Future.fromTry(decodeResponse(response))
+    entity <- Future.fromTry(decoded.toEntity)
+    unmarshalled <- unmarshall(response, entity)(um, ec)
+  } yield unmarshalled
+
+  private def unmarshall[A](response: HttpResponse, entity: Unmarshal[ResponseEntity])(implicit um: Unmarshaller[ResponseEntity, A], ec: ExecutionContext): Future[A] = {
+    import CromwellFailedResponseExceptionJsonSupport._
+
+    if (response.status.isSuccess()) entity.to[A]
+    else entity.to[CromwellFailedResponseException] flatMap Future.failed
+  }
+
+  private def simpleRequest[A](uri: String, method: HttpMethod = HttpMethods.GET)(implicit um: Unmarshaller[ResponseEntity, A], ec: ExecutionContext): Future[A] = makeRequest[A](HttpRequest(uri = uri, method = method))
+
+  private val decoders = Map(
+    HttpEncodings.gzip -> Gzip,
+    HttpEncodings.deflate -> Deflate,
+    HttpEncodings.identity -> NoCoding
+  )
+
+  private def decodeResponse(response: HttpResponse): Try[HttpResponse] = {
+    decoders.get(response.encoding) map { decoder =>
+      Try(decoder.decodeMessage(response))
+    } getOrElse Failure(UnsuccessfulRequestException(s"No decoder for ${response.encoding}", response))
+  }
+}
+
+object CromwellClient {
+  final implicit class EnhancedHttpResponse(val response: HttpResponse) extends AnyVal {
+
+    def toEntity: Try[Unmarshal[ResponseEntity]] = response match {
+      case HttpResponse(_: StatusCodes.Success, _, entity, _) => Success(Unmarshal(entity))
+      case HttpResponse(_: StatusCodes.ServerError, _, entity, _) => Success(Unmarshal(entity))
+      case other => Failure(UnsuccessfulRequestException("Unmarshalling error", other))
+    }
+  }
+
+  final case class UnsuccessfulRequestException(message: String, httpResponse: HttpResponse) extends Exception {
+    override def getMessage: String = message + ": " + httpResponse.toString
+  }
+
+  private[api] def insertSecrets(options: Option[String], refreshToken: Option[String]): Option[String] = {
+    import DefaultJsonProtocol._
+    val tokenKey = "refresh_token"
+
+    val secretOptions = for {
+      refreshTokenValue <- refreshToken
+      optionsValue <- options
+      optionsMap = optionsValue.parseJson.asJsObject.convertTo[Map[String, JsValue]]
+      if optionsMap.contains(tokenKey)
+      secretMap = optionsMap.updated(tokenKey, JsString(refreshTokenValue))
+      secretValue = secretMap.toJson.toString
+    } yield secretValue
+
+    secretOptions orElse options
+  }
+}
diff --git a/cromwellApiClient/src/main/scala/cromwell/api/model/CallCacheDiff.scala b/cromwellApiClient/src/main/scala/cromwell/api/model/CallCacheDiff.scala
new file mode 100644
index 000000000..fa4e7fb91
--- /dev/null
+++ b/cromwellApiClient/src/main/scala/cromwell/api/model/CallCacheDiff.scala
@@ -0,0 +1,15 @@
+package cromwell.api.model
+
+import ShardIndexFormatter._
+import WorkflowIdJsonFormatter._
+import spray.json.DefaultJsonProtocol
+
+case class CallCacheDiffCallDescription(executionStatus: String, allowResultReuse: Boolean, callFqn: String, jobIndex: ShardIndex, workflowId: WorkflowId)
+case class HashDifference(hashKey: String, callA: Option[String], callB: Option[String])
+case class CallCacheDiff(callA: CallCacheDiffCallDescription, callB: CallCacheDiffCallDescription, hashDifferential: List[HashDifference])
+
+object CallCacheDiffJsonSupport extends DefaultJsonProtocol {
+  implicit val CallCacheDiffCallDescriptionFormat = jsonFormat5(CallCacheDiffCallDescription)
+  implicit val HashDifferenceFormat = jsonFormat3(HashDifference)
+  implicit val CallCacheDiffFormat = jsonFormat3(CallCacheDiff)
+}
diff --git a/cromwellApiClient/src/main/scala/cromwell/api/model/CromwellBackends.scala b/cromwellApiClient/src/main/scala/cromwell/api/model/CromwellBackends.scala
new file mode 100644
index 000000000..007723310
--- /dev/null
+++ b/cromwellApiClient/src/main/scala/cromwell/api/model/CromwellBackends.scala
@@ -0,0 +1,9 @@
+package cromwell.api.model
+
+import spray.json.DefaultJsonProtocol
+
+object CromwellBackendsJsonSupport extends DefaultJsonProtocol {
+  implicit val CromwellBackendsFormat = jsonFormat2(CromwellBackends)
+}
+
+final case class CromwellBackends(defaultBackend: String, supportedBackends: List[String])
diff --git a/cromwellApiClient/src/main/scala/cromwell/api/model/CromwellFailedResponseException.scala b/cromwellApiClient/src/main/scala/cromwell/api/model/CromwellFailedResponseException.scala
new file mode 100644
index 000000000..6f58fe3bd
--- /dev/null
+++ b/cromwellApiClient/src/main/scala/cromwell/api/model/CromwellFailedResponseException.scala
@@ -0,0 +1,9 @@
+package cromwell.api.model
+
+import spray.json.DefaultJsonProtocol
+
+object CromwellFailedResponseExceptionJsonSupport extends DefaultJsonProtocol {
+  implicit val CromwellFailedResponseExceptionFormat = jsonFormat2(CromwellFailedResponseException)
+}
+
+case class CromwellFailedResponseException(status: String, message: String) extends Exception(message)
diff --git a/cromwellApiClient/src/main/scala/cromwell/api/model/CromwellQueryResult.scala b/cromwellApiClient/src/main/scala/cromwell/api/model/CromwellQueryResult.scala
new file mode 100644
index 000000000..7fcb0dcba
--- /dev/null
+++ b/cromwellApiClient/src/main/scala/cromwell/api/model/CromwellQueryResult.scala
@@ -0,0 +1,15 @@
+package cromwell.api.model
+
+import java.time.OffsetDateTime
+import spray.json.DefaultJsonProtocol
+import cromwell.api.model.WorkflowIdJsonFormatter._
+import cromwell.api.model.WorkflowStatusJsonFormatter._
+
+case class CromwellQueryResults(results: Seq[CromwellQueryResult])
+
+case class CromwellQueryResult(name: String, id: WorkflowId, status: WorkflowStatus, end: OffsetDateTime, start: OffsetDateTime)
+
+object CromwellQueryResultJsonFormatter extends DefaultJsonProtocol {
+  implicit val CromwellQueryResultJsonFormat = jsonFormat5(CromwellQueryResult)
+  implicit val CromwellQueryResultsJsonFormat = jsonFormat1(CromwellQueryResults)
+}
diff --git a/cromwellApiClient/src/main/scala/cromwell/api/model/CromwellStatus.scala b/cromwellApiClient/src/main/scala/cromwell/api/model/CromwellStatus.scala
new file mode 100644
index 000000000..31433e5c7
--- /dev/null
+++ b/cromwellApiClient/src/main/scala/cromwell/api/model/CromwellStatus.scala
@@ -0,0 +1,9 @@
+package cromwell.api.model
+
+import spray.json.DefaultJsonProtocol
+
+object CromwellStatusJsonSupport extends DefaultJsonProtocol {
+  implicit val CromwellStatusFormat = jsonFormat2(CromwellStatus)
+}
+
+case class CromwellStatus(id: String, status: String)
diff --git a/cromwellApiClient/src/main/scala/cromwell/api/model/CromwellVersion.scala b/cromwellApiClient/src/main/scala/cromwell/api/model/CromwellVersion.scala
new file mode 100644
index 000000000..d6c71a065
--- /dev/null
+++ b/cromwellApiClient/src/main/scala/cromwell/api/model/CromwellVersion.scala
@@ -0,0 +1,9 @@
+package cromwell.api.model
+
+import spray.json.DefaultJsonProtocol
+
+object CromwellVersionJsonSupport extends DefaultJsonProtocol {
+  implicit val CromwellVersionFormat = jsonFormat1(CromwellVersion)
+}
+
+case class CromwellVersion(cromwell: String)
diff --git a/cromwellApiClient/src/main/scala/cromwell/api/model/FailedWorkflowSubmission.scala b/cromwellApiClient/src/main/scala/cromwell/api/model/FailedWorkflowSubmission.scala
new file mode 100644
index 000000000..c9e14cb37
--- /dev/null
+++ b/cromwellApiClient/src/main/scala/cromwell/api/model/FailedWorkflowSubmission.scala
@@ -0,0 +1,9 @@
+package cromwell.api.model
+
+import spray.json.DefaultJsonProtocol
+
+object FailedWorkflowSubmissionJsonSupport extends DefaultJsonProtocol {
+  implicit val FailedWorkflowSubmissionFormat = jsonFormat2(FailedWorkflowSubmission)
+}
+
+case class FailedWorkflowSubmission(status: String, message: String)
diff --git a/cromwellApiClient/src/main/scala/cromwell/api/model/Label.scala b/cromwellApiClient/src/main/scala/cromwell/api/model/Label.scala
new file mode 100644
index 000000000..fd9d88d21
--- /dev/null
+++ b/cromwellApiClient/src/main/scala/cromwell/api/model/Label.scala
@@ -0,0 +1,16 @@
+package cromwell.api.model
+
+import spray.json.{DefaultJsonProtocol, JsObject, JsString, JsValue, RootJsonFormat}
+import scala.language.postfixOps
+
+object LabelsJsonFormatter extends DefaultJsonProtocol {
+  implicit object LabelJsonFormat extends RootJsonFormat[List[Label]] {
+    def write(l: List[Label]) = JsObject(l map { label => label.key -> JsString(label.value)} :_* )
+    def read(value: JsValue) = value.asJsObject.fields map {
+      case (k, JsString(v)) => Label(k, v)
+      case other => throw new UnsupportedOperationException(s"Cannot deserialize $other to a Label")
+    } toList
+  }
+}
+
+case class Label(key: String, value: String)
diff --git a/cromwellApiClient/src/main/scala/cromwell/api/model/ShardIndex.scala b/cromwellApiClient/src/main/scala/cromwell/api/model/ShardIndex.scala
new file mode 100644
index 000000000..405305b8a
--- /dev/null
+++ b/cromwellApiClient/src/main/scala/cromwell/api/model/ShardIndex.scala
@@ -0,0 +1,18 @@
+package cromwell.api.model
+
+import spray.json.{DefaultJsonProtocol, JsNumber, JsValue, RootJsonFormat}
+
+case class ShardIndex(index: Option[Int]) extends AnyVal {
+  override def toString: String = index.getOrElse(-1).toString
+}
+
+object ShardIndexFormatter extends DefaultJsonProtocol {
+  implicit object ShardIndexJsonFormat extends RootJsonFormat[ShardIndex] {
+    def write(si: ShardIndex) = JsNumber(si.index.getOrElse(-1))
+    def read(value: JsValue) = value match {
+      case JsNumber(i) if i.equals(-1) => ShardIndex(None)
+      case JsNumber(i) if i.isValidInt && i.intValue > 0 => ShardIndex(Option(i.intValue()))
+      case other => throw new UnsupportedOperationException(s"Cannot deserialize $other into a ShardIndex")
+    }
+  }
+}
diff --git a/cromwellApiClient/src/main/scala/cromwell/api/model/SubmittedWorkflow.scala b/cromwellApiClient/src/main/scala/cromwell/api/model/SubmittedWorkflow.scala
new file mode 100644
index 000000000..bcebcd5c9
--- /dev/null
+++ b/cromwellApiClient/src/main/scala/cromwell/api/model/SubmittedWorkflow.scala
@@ -0,0 +1,8 @@
+package cromwell.api.model
+
+import java.net.URL
+
+/**
+  * Represents information which we need to capture about a workflow sent to Cromwell.
+  */
+case class SubmittedWorkflow(id: WorkflowId, cromwellServer: URL, workflow: WorkflowSubmission)
diff --git a/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowId.scala b/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowId.scala
new file mode 100644
index 000000000..f52495136
--- /dev/null
+++ b/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowId.scala
@@ -0,0 +1,33 @@
+package cromwell.api.model
+
+import java.util.UUID
+
+import spray.json.{DefaultJsonProtocol, JsString, JsValue, RootJsonFormat}
+
+// ********* !!!!!!!!!! ********
+//
+// WARNING! This is the Cromwell API version of WorkflowId. If you aren't changing the API client, you probably
+// want cromwell.core.WorkflowId instead!
+//
+// ********* !!!!!!!!!! ********
+
+final case class WorkflowId(id: UUID) extends AnyVal {
+  override def toString = id.toString
+  def shortString = id.toString.split("-")(0)
+}
+
+object WorkflowId {
+  def fromString(id: String): WorkflowId = new WorkflowId(UUID.fromString(id))
+  def randomId() = WorkflowId(UUID.randomUUID())
+}
+
+object WorkflowIdJsonFormatter extends DefaultJsonProtocol {
+  implicit object WorkflowIdJsonFormat extends RootJsonFormat[WorkflowId] {
+    def write(id: WorkflowId) = JsString(id.id.toString)
+    def read(value: JsValue) = value match {
+      case JsString(s) => WorkflowId.fromString(s)
+      case other => throw new UnsupportedOperationException(s"Cannot deserialize $other into a ShardIndex")
+    }
+  }
+}
+
diff --git a/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowLogs.scala b/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowLogs.scala
new file mode 100644
index 000000000..b08d6299a
--- /dev/null
+++ b/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowLogs.scala
@@ -0,0 +1,39 @@
+package cromwell.api.model
+
+import spray.json.DefaultJsonProtocol
+import cromwell.api.model.ShardIndexFormatter._
+
+private[api] case class CallLogStruct(stdout: String, stderr: String, backendLogs: Map[String, String], shardIndex: ShardIndex, attempt: Int)
+private[api] case class WorkflowLogsStruct(calls: Map[String, List[CallLogStruct]], id: String)
+
+
+object WorkflowLogsJsonSupport extends DefaultJsonProtocol {
+  implicit val CallLogStructFormat = jsonFormat5(CallLogStruct)
+  implicit val WorkflowLogsStructFormat = jsonFormat2(WorkflowLogsStruct)
+}
+
+/**
+  * @param logs Mapping from shard index and attempt
+  */
+case class CallLogs(logs: Map[JobLogsKey, JobLogs])
+case class JobLogsKey(shardIndex: ShardIndex, attempt: Int)
+case class JobLogs(stdout: String, stderr: String, backendLogs: Map[String, String])
+
+/**
+  * @param logs Mapping from call name to all logs for that call (including all shards and attempts)
+  */
+case class WorkflowLogs(logs: Map[String, CallLogs])
+
+object WorkflowLogs {
+  def callStructsToCallLogs(structs: List[CallLogStruct]): CallLogs = {
+    val callLogs = structs map { struct =>
+      JobLogsKey(struct.shardIndex, struct.attempt) -> JobLogs(struct.stdout, struct.stderr, struct.backendLogs)
+    }
+    CallLogs(callLogs.toMap)
+  }
+
+  def apply(struct: WorkflowLogsStruct): WorkflowLogs = {
+    val workflowLogs = struct.calls map { case (callName, structs) => callName -> callStructsToCallLogs(structs)}
+    WorkflowLogs(workflowLogs)
+  }
+}
diff --git a/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowMetadata.scala b/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowMetadata.scala
new file mode 100644
index 000000000..5b7ff88ab
--- /dev/null
+++ b/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowMetadata.scala
@@ -0,0 +1,3 @@
+package cromwell.api.model
+
+case class WorkflowMetadata(value: String) extends AnyVal
diff --git a/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowOutputs.scala b/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowOutputs.scala
new file mode 100644
index 000000000..4eb4cd0c8
--- /dev/null
+++ b/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowOutputs.scala
@@ -0,0 +1,9 @@
+package cromwell.api.model
+
+import spray.json.DefaultJsonProtocol
+
+object WorkflowOutputsJsonSupport extends DefaultJsonProtocol {
+  implicit val OutputResponseFormat = jsonFormat2(WorkflowOutputs)
+}
+
+case class WorkflowOutputs(id: String, outputs: Map[String, String])
diff --git a/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowStatus.scala b/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowStatus.scala
new file mode 100644
index 000000000..6da1282d2
--- /dev/null
+++ b/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowStatus.scala
@@ -0,0 +1,51 @@
+package cromwell.api.model
+
+import spray.json.{DefaultJsonProtocol, JsString, JsValue, RootJsonFormat}
+
+// ********* !!!!!!!!!! ********
+//
+// WARNING! This is a Cromwell API class. If you aren't changing the API client, you probably
+// want to look elsewhere (maybe in cromwell.core?)
+//
+// ********* !!!!!!!!!! ********
+
+/**
+  * ADT tree to describe Cromwell workflow statuses, both terminal and non-terminal
+  */
+sealed trait WorkflowStatus
+
+sealed trait TerminalStatus extends WorkflowStatus
+case object Aborted extends TerminalStatus
+case object Failed extends TerminalStatus
+case object Succeeded extends TerminalStatus
+
+sealed trait NonTerminalStatus extends WorkflowStatus
+case object Submitted extends NonTerminalStatus
+case object Running extends NonTerminalStatus
+case object Aborting extends NonTerminalStatus
+
+object WorkflowStatus {
+  def apply(status: String): WorkflowStatus = {
+    status match {
+      case "Submitted" => Submitted
+      case "Running" => Running
+      case "Aborting" => Aborting
+      case "Aborted" => Aborted
+      case "Failed" => Failed
+      case "Succeeded" => Succeeded
+      case bad => throw new IllegalArgumentException(s"No such status: $bad")
+    }
+  }
+
+  def apply(workflowStatus: CromwellStatus): WorkflowStatus = apply(workflowStatus.status)
+}
+
+object WorkflowStatusJsonFormatter extends DefaultJsonProtocol {
+  implicit object WorkflowStatusJsonFormat extends RootJsonFormat[WorkflowStatus] {
+    def write(status: WorkflowStatus) = new JsString(status.toString)
+    def read(value: JsValue) = value match {
+      case JsString(string) => WorkflowStatus(string)
+      case other => throw new UnsupportedOperationException(s"Cannot deserialize $other into a WorkflowStatus")
+    }
+  }
+}
diff --git a/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowSubmission.scala b/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowSubmission.scala
new file mode 100644
index 000000000..f20a7aa15
--- /dev/null
+++ b/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowSubmission.scala
@@ -0,0 +1,35 @@
+package cromwell.api.model
+
+import better.files.File
+
+sealed trait WorkflowSubmission {
+  val wdl: String
+  val workflowType: Option[String]
+  val workflowTypeVersion: Option[String]
+  val inputsJson: Option[String]
+  val options: Option[String]
+  val customLabels: Option[List[Label]]
+  val zippedImports: Option[File]
+  val refreshToken: Option[String]
+}
+
+final case class WorkflowSingleSubmission(wdl: String,
+                                          workflowType: Option[String],
+                                          workflowTypeVersion: Option[String],
+                                          inputsJson: Option[String],
+                                          options: Option[String],
+                                          customLabels: Option[List[Label]],
+                                          zippedImports: Option[File],
+                                          refreshToken: Option[String]) extends WorkflowSubmission
+
+final case class WorkflowBatchSubmission(wdl: String,
+                                         workflowType: Option[String],
+                                         workflowTypeVersion: Option[String],
+                                         inputsBatch: List[String],
+                                         options: Option[String],
+                                         customLabels: Option[List[Label]],
+                                         zippedImports: Option[File],
+                                         refreshToken: Option[String]) extends WorkflowSubmission {
+
+  override val inputsJson: Option[String] = Option(inputsBatch.mkString(start = "[", sep = ",", end = "]"))
+}
diff --git a/cromwellApiClient/src/main/scala/cromwell/api/model/package.scala b/cromwellApiClient/src/main/scala/cromwell/api/model/package.scala
new file mode 100644
index 000000000..5dd6bffb1
--- /dev/null
+++ b/cromwellApiClient/src/main/scala/cromwell/api/model/package.scala
@@ -0,0 +1,20 @@
+package cromwell.api
+
+import java.time.OffsetDateTime
+
+import spray.json.{DefaultJsonProtocol, JsString, JsValue, RootJsonFormat}
+
+package object model {
+
+  implicit val OffsetDateTimeJsonFormat = OffsetDateTimeJsonFormatter.OffsetDateTimeFormat
+
+  object OffsetDateTimeJsonFormatter extends DefaultJsonProtocol {
+    object OffsetDateTimeFormat extends RootJsonFormat[OffsetDateTime] {
+      def write(odt: OffsetDateTime) = new JsString(odt.toString)
+      def read(value: JsValue) = value match {
+        case JsString(string) => OffsetDateTime.parse(string)
+        case other => throw new UnsupportedOperationException(s"Cannot deserialize $other into an OffsetDateTime")
+      }
+    }
+  }
+}
diff --git a/cromwellApiClient/src/test/scala/cromwell/api/CromwellClientSpec.scala b/cromwellApiClient/src/test/scala/cromwell/api/CromwellClientSpec.scala
new file mode 100644
index 000000000..5c0b43628
--- /dev/null
+++ b/cromwellApiClient/src/test/scala/cromwell/api/CromwellClientSpec.scala
@@ -0,0 +1,36 @@
+package cromwell.api
+
+import org.scalatest.prop.TableDrivenPropertyChecks
+import org.scalatest.{FlatSpec, Matchers}
+import spray.json.JsonParser.ParsingException
+
+class CromwellClientSpec extends FlatSpec with Matchers with TableDrivenPropertyChecks {
+  behavior of "CromwellClient"
+
+  val table = Table(
+    ("description", "optionsOption", "refreshTokenOption", "expected"),
+    ("ignore bad json when refresh token not provided", Option("{"), None, Option("{")),
+    ("not format json when refresh token key not found", Option("{   }"), Option("myToken"), Option("{   }")),
+    ("replace token when found", Option("""{"refresh_token" : "replace_me"}"""), Option("myToken"),
+      Option("""{"refresh_token":"myToken"}""")),
+  )
+
+  forAll(table) { (description, optionsOption, refreshTokenOption, expected) =>
+    it should description in {
+      val actual = CromwellClient.insertSecrets(optionsOption, refreshTokenOption)
+      actual should be(expected)
+    }
+  }
+
+  it should "throw an exception when inserting a refresh token into bad json" in {
+    val optionsOption = Option("{")
+    val refreshTokenOption = Option("myToken")
+    val actual = intercept[ParsingException](CromwellClient.insertSecrets(optionsOption, refreshTokenOption))
+    actual.summary should be("""Unexpected end-of-input at input index 1 (line 1, position 2), expected '"'""")
+    actual.detail should be(
+      """|
+         |{
+         | ^
+         |""".stripMargin)
+  }
+}
diff --git a/cromwellApiClient/src/test/scala/cromwell/api/CromwellResponseFailedSpec.scala b/cromwellApiClient/src/test/scala/cromwell/api/CromwellResponseFailedSpec.scala
new file mode 100644
index 000000000..d8b2dd917
--- /dev/null
+++ b/cromwellApiClient/src/test/scala/cromwell/api/CromwellResponseFailedSpec.scala
@@ -0,0 +1,41 @@
+package cromwell.api
+
+import java.net.URL
+
+import akka.actor.ActorSystem
+import akka.http.scaladsl.model._
+import akka.stream.ActorMaterializer
+import akka.testkit.TestKit
+import cromwell.api.model.CromwellFailedResponseException
+import org.scalatest.{AsyncFlatSpecLike, BeforeAndAfterAll, Matchers}
+
+import scala.concurrent.duration._
+import scala.concurrent.{Await, Future}
+import scala.language.postfixOps
+
+class CromwellResponseFailedSpec extends TestKit(ActorSystem()) with AsyncFlatSpecLike with Matchers with BeforeAndAfterAll {
+  override def afterAll(): Unit = {
+    Await.ready(system.terminate(), 1 second)
+    super.afterAll()
+  }
+  
+  implicit val materializer = ActorMaterializer()
+  
+  "CromwellAPIClient" should "try to fail the Future with a CromwellFailedResponseException if the HttpResponse is unsuccessful" in {
+    val client = new CromwellClient(new URL("http://fakeurl"), "v1") {
+      override def executeRequest(request: HttpRequest): Future[HttpResponse] = Future.successful(
+        new HttpResponse(StatusCodes.ServiceUnavailable, List.empty[HttpHeader], HttpEntity(ContentTypes.`application/json`,
+          """{
+            |  "status": "fail",
+            |  "message": "Cromwell service shutting down"
+            |}
+          """.stripMargin), HttpProtocols.`HTTP/1.1`)
+      )
+    }
+
+    recoverToExceptionIf[CromwellFailedResponseException] { client.version(scala.concurrent.ExecutionContext.global) } map { exception =>
+      assert(exception.status == "fail")
+      assert(exception.message == "Cromwell service shutting down")
+    }
+  }
+}
diff --git a/cromwellApiClient/src/test/scala/cromwell/api/model/CromwellQueryResultJsonFormatterSpec.scala b/cromwellApiClient/src/test/scala/cromwell/api/model/CromwellQueryResultJsonFormatterSpec.scala
new file mode 100644
index 000000000..f1983e794
--- /dev/null
+++ b/cromwellApiClient/src/test/scala/cromwell/api/model/CromwellQueryResultJsonFormatterSpec.scala
@@ -0,0 +1,45 @@
+package cromwell.api.model
+
+import java.time.OffsetDateTime
+
+import org.scalatest.{FlatSpec, Matchers}
+import spray.json._
+import cromwell.api.model.CromwellQueryResultJsonFormatter._
+
+class CromwellQueryResultJsonFormatterSpec extends FlatSpec with Matchers {
+
+  behavior of "CromwellQueryResultJsonFormat"
+
+  val sampleQueryResult = CromwellQueryResults(results = List(
+    CromwellQueryResult("switcheroo", WorkflowId.fromString("bee51f36-396d-4e22-8a81-33dedff66bf6"), Failed, OffsetDateTime.parse("2017-07-24T14:44:34.010-04:00"), OffsetDateTime.parse("2017-07-24T14:44:33.227-04:00")),
+    CromwellQueryResult("switcheroo", WorkflowId.fromString("0071495e-39eb-478e-bc98-8614b986c91e"), Succeeded, OffsetDateTime.parse("2017-07-24T15:06:45.940-04:00"), OffsetDateTime.parse("2017-07-24T15:04:54.372-04:00"))
+  ))
+
+  val sampleJson =  """|{
+                       |  "results": [
+                       |    {
+                       |      "name": "switcheroo",
+                       |      "id": "bee51f36-396d-4e22-8a81-33dedff66bf6",
+                       |      "status": "Failed",
+                       |      "end": "2017-07-24T14:44:34.010-04:00",
+                       |      "start": "2017-07-24T14:44:33.227-04:00"
+                       |    },
+                       |    {
+                       |      "name": "switcheroo",
+                       |      "id": "0071495e-39eb-478e-bc98-8614b986c91e",
+                       |      "status": "Succeeded",
+                       |      "end": "2017-07-24T15:06:45.940-04:00",
+                       |      "start": "2017-07-24T15:04:54.372-04:00"
+                       |    }
+                       |  ]
+                       |}""".stripMargin.parseJson.asJsObject
+
+  it should "write a query result as a structured JsObject" in {
+
+    sampleQueryResult.toJson shouldEqual sampleJson
+  }
+
+  it should "read a query result as a structured JsObject" in {
+    sampleJson.convertTo[CromwellQueryResults] shouldBe sampleQueryResult
+  }
+}
diff --git a/cromwellApiClient/src/test/scala/cromwell/api/model/LabelsJsonFormatterSpec.scala b/cromwellApiClient/src/test/scala/cromwell/api/model/LabelsJsonFormatterSpec.scala
new file mode 100644
index 000000000..db38ba6a8
--- /dev/null
+++ b/cromwellApiClient/src/test/scala/cromwell/api/model/LabelsJsonFormatterSpec.scala
@@ -0,0 +1,36 @@
+package cromwell.api.model
+
+import spray.json._
+import org.scalatest.FlatSpec
+import org.scalatest.Matchers
+
+class LabelsJsonFormatterSpec extends FlatSpec with Matchers {
+  import cromwell.api.model.LabelsJsonFormatter._
+
+  behavior of "WdlValueJsonFormat"
+
+  val sampleLabels = List(Label("key-1", "value-1"), Label("key-2", "value-2"), Label("key-3", "value-3"))
+  val sampleJson =  """|{
+                       |  "key-1":"value-1",
+                       |  "key-2":"value-2",
+                       |  "key-3":"value-3"
+                       |}""".stripMargin.parseJson.asJsObject
+
+  it should "write a Label as a structured JsObject" in {
+    val label = List(Label("test-key", "test-value"))
+    val expectedJson: JsObject =
+      """|{
+         |  "test-key": "test-value"
+         |}""".stripMargin.parseJson.asJsObject
+
+    label.toJson shouldEqual expectedJson
+  }
+
+  it should "write an optional list of labels as a structured JsObject" in {
+    Option(sampleLabels).toJson shouldEqual sampleJson
+  }
+
+  it should "read a list of labels as a structured JsObject" in {
+    sampleJson.convertTo[List[Label]] shouldBe sampleLabels
+  }
+}
diff --git a/database/migration/src/main/resources/changelog.xml b/database/migration/src/main/resources/changelog.xml
index dc182d106..e005c5ef4 100644
--- a/database/migration/src/main/resources/changelog.xml
+++ b/database/migration/src/main/resources/changelog.xml
@@ -59,6 +59,15 @@
     <include file="changesets/standardize_column_names_patches.xml" relativeToChangelogFile="true" />
     <include file="changesets/nullable_lobs.xml" relativeToChangelogFile="true" />
     <include file="changesets/add_attempt_in_call_caching_entry.xml" relativeToChangelogFile="true" />
+    <include file="changesets/replace_empty_custom_labels.xml" relativeToChangelogFile="true" />
+    <include file="changesets/failure_metadata.xml" relativeToChangelogFile="true" />
+    <include file="changesets/failure_metadata_2.xml" relativeToChangelogFile="true" />
+    <include file="changesets/call_caching_aggregated_hashes.xml" relativeToChangelogFile="true" />
+    <include file="changesets/custom_label_entry.xml" relativeToChangelogFile="true" />
+    <include file="changesets/docker_hash_store.xml" relativeToChangelogFile="true" />
+    <include file="changesets/workflow_store_type_and_version.xml" relativeToChangelogFile="true" />
+    <include file="changesets/preempted_is_a_backend_status.xml" relativeToChangelogFile="true" />
+    <include file="changesets/remove_pre_pbe_tables.xml" relativeToChangelogFile="true" />
 </databaseChangeLog>
 <!--
 
@@ -78,7 +87,7 @@ WHY
     In slick schemas, `index(unique = false)` always creates an index. Depending on the database type,
     `index(unique = true)` sometimes creates a unique _constraint_, not actually an index.
 
-    https://github.com/slick/slick/blob/3.1.1/slick/src/main/scala/slick/driver/HsqldbDriver.scala#L126
+    https://github.com/slick/slick/blob/3.2.0/slick/src/main/scala/slick/jdbc/HsqldbProfile.scala#L127
 
 -*-
 
diff --git a/database/migration/src/main/resources/changesets/call_caching_aggregated_hashes.xml b/database/migration/src/main/resources/changesets/call_caching_aggregated_hashes.xml
new file mode 100644
index 000000000..87a1fb969
--- /dev/null
+++ b/database/migration/src/main/resources/changesets/call_caching_aggregated_hashes.xml
@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
+
+    <changeSet author="tjeandet" id="call_caching_aggregation_entry">
+        <comment>
+            Holds an aggregated version of the hashes for each cache entry.
+            Split into "BaseAggregation" and "InputFilesAggregation"
+            BaseAggregation aggregates all call hashes, except input files.
+            It should be used as a first pass to check if cache hits exist.
+            InputFiles aggregates all hashes from the input files of a call. 
+            It should be used as a second pass to check if cache hits exist, only if the first pass succeeded.
+        </comment>
+        <createTable tableName="CALL_CACHING_AGGREGATION_ENTRY">
+            <column autoIncrement="true" name="CALL_CACHING_AGGREGATION_ENTRY_ID" type="INT">
+                <constraints primaryKey="true" primaryKeyName="PK_CALL_CACHING_AGGREGATION_ENTRY"/>
+            </column>
+            <column name="CALL_CACHING_ENTRY_ID" type="INT">
+                <constraints nullable="false"/>
+            </column>
+            <column name="BASE_AGGREGATION" type="VARCHAR(255)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="INPUT_FILES_AGGREGATION" type="VARCHAR(255)">
+                <constraints nullable="true"/>
+            </column>
+        </createTable>
+    </changeSet>
+
+    <changeSet author="tjeandet" id="call_caching_aggregation_entry_keys_and_indexes">
+        <addForeignKeyConstraint baseColumnNames="CALL_CACHING_ENTRY_ID"
+                                 baseTableName="CALL_CACHING_AGGREGATION_ENTRY"
+                                 constraintName="FK_CALL_CACHING_AGGREGATION_ENTRY_CALL_CACHING_ENTRY_ID"
+                                 referencedColumnNames="CALL_CACHING_ENTRY_ID"
+                                 referencedTableName="CALL_CACHING_ENTRY"/>
+        <createIndex indexName="IX_CALL_CACHING_AGGREGATION_ENTRY_BA_IFA"
+                     tableName="CALL_CACHING_AGGREGATION_ENTRY"
+                     unique="false">
+            <column name="BASE_AGGREGATION"/>
+            <column name="INPUT_FILES_AGGREGATION"/>
+        </createIndex>
+    </changeSet>
+
+    
+    <changeSet author="tjeandet" id="call_caching_aggregation_entry_populate_aggregation" dbms="mysql">
+        <preConditions onFailMessage="The group_concat_max_len value on the MySQL server is too low for this migration to run. Please increase it and restart Cromwell to migrate your call caching data. See https://github.com/broadinstitute/cromwell/blob/develop/CHANGELOG.md for help on what value it should be set to.">
+            <sqlCheck expectedResult="1">
+                SELECT @@GLOBAL.group_concat_max_len >= MAX(aggregated) as group_concat_max_len FROM
+                (
+                SELECT cche.CALL_CACHING_ENTRY_ID, SUM(LENGTH(CONCAT(cche.HASH_KEY, cche.HASH_VALUE))) AS aggregated
+                FROM CALL_CACHING_HASH_ENTRY cche
+                GROUP BY cche.CALL_CACHING_ENTRY_ID
+
+                -- Handle the case of the CALL_CACHING_HASH_ENTRY table possibly being empty by UNIONing to a fake row.
+                UNION
+
+                SELECT 0, 0 AS aggregated
+                ) aggregation
+            </sqlCheck>
+        </preConditions>
+        <comment>This changeset will only be run if the group_concat_max_len flag has a value that will allow it to run successfully</comment>
+        
+        <sqlFile relativeToChangelogFile="true" path="call_caching_aggregation/call_caching_base_aggregation.sql" />
+        <sqlFile relativeToChangelogFile="true" path="call_caching_aggregation/call_caching_input_files_aggregation.sql" />
+
+        <rollback>
+            <delete tableName="CALL_CACHING_AGGREGATION_ENTRY" />
+        </rollback>
+    </changeSet>
+</databaseChangeLog>
diff --git a/database/migration/src/main/resources/changesets/call_caching_aggregation/call_caching_base_aggregation.sql b/database/migration/src/main/resources/changesets/call_caching_aggregation/call_caching_base_aggregation.sql
new file mode 100644
index 000000000..94a1966c8
--- /dev/null
+++ b/database/migration/src/main/resources/changesets/call_caching_aggregation/call_caching_base_aggregation.sql
@@ -0,0 +1,18 @@
+INSERT INTO CALL_CACHING_AGGREGATION_ENTRY(
+  CALL_CACHING_ENTRY_ID,
+  BASE_AGGREGATION
+)
+  SELECT cche.CALL_CACHING_ENTRY_ID,
+  UPPER(MD5(GROUP_CONCAT(
+                CONCAT(cche.HASH_KEY, cche.HASH_VALUE) ORDER BY cche.HASH_KEY ASC SEPARATOR ''
+            ))) as aggregated
+FROM CALL_CACHING_HASH_ENTRY cche
+WHERE HASH_KEY NOT LIKE 'input: File%'
+AND
+      (HASH_KEY NOT LIKE 'runtime attribute: %' OR
+       HASH_KEY = 'runtime attribute: continueOnReturnCode' OR
+       HASH_KEY = 'runtime attribute: docker' OR
+       HASH_KEY = 'runtime attribute: failOnStderr'
+      )
+        
+GROUP BY cche.CALL_CACHING_ENTRY_ID;
diff --git a/database/migration/src/main/resources/changesets/call_caching_aggregation/call_caching_input_files_aggregation.sql b/database/migration/src/main/resources/changesets/call_caching_aggregation/call_caching_input_files_aggregation.sql
new file mode 100644
index 000000000..4ca4df9ae
--- /dev/null
+++ b/database/migration/src/main/resources/changesets/call_caching_aggregation/call_caching_input_files_aggregation.sql
@@ -0,0 +1,10 @@
+UPDATE CALL_CACHING_AGGREGATION_ENTRY ccae
+  INNER JOIN (SELECT cche.CALL_CACHING_ENTRY_ID,
+                UPPER(MD5(GROUP_CONCAT(
+                              CONCAT(cche.HASH_KEY, cche.HASH_VALUE) ORDER BY cche.HASH_KEY ASC SEPARATOR ''
+                          ))) as hash_aggregation
+              FROM CALL_CACHING_HASH_ENTRY cche
+              WHERE HASH_KEY LIKE 'input: File%'
+              GROUP BY cche.CALL_CACHING_ENTRY_ID) cche_aggregated
+    ON ccae.CALL_CACHING_ENTRY_ID = cche_aggregated.CALL_CACHING_ENTRY_ID
+SET ccae.INPUT_FILES_AGGREGATION = cche_aggregated.hash_aggregation
diff --git a/database/migration/src/main/resources/changesets/custom_label_entry.xml b/database/migration/src/main/resources/changesets/custom_label_entry.xml
new file mode 100644
index 000000000..49f09da85
--- /dev/null
+++ b/database/migration/src/main/resources/changesets/custom_label_entry.xml
@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
+
+    <changeSet id="custom_LABEL_entry" author="rmunshi">
+        <createTable tableName="CUSTOM_LABEL_ENTRY">
+            <column autoIncrement="true" name="CUSTOM_LABEL_ENTRY_ID" type="BIGINT">
+                <constraints primaryKey="true" primaryKeyName="PK_CUSTOM_LABEL_ENTRY"/>
+            </column>
+            <column name="CUSTOM_LABEL_KEY" type="VARCHAR(255)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="CUSTOM_LABEL_VALUE" type="VARCHAR(255)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="WORKFLOW_EXECUTION_UUID" type="VARCHAR(100)">
+                <constraints nullable="false"/>
+            </column>
+        </createTable>
+    </changeSet>
+
+    <changeSet author="rmunshi" id="WMSE_workflow_execution_uuid_foreign_key">
+        <addForeignKeyConstraint constraintName="FK_CUSTOM_LABEL_ENTRY_WORKFLOW_EXECUTION_UUID"
+                                 baseTableName="CUSTOM_LABEL_ENTRY"
+                                 baseColumnNames="WORKFLOW_EXECUTION_UUID"
+                                 referencedTableName="WORKFLOW_METADATA_SUMMARY_ENTRY"
+                                 referencedColumnNames="WORKFLOW_EXECUTION_UUID"
+                                 onDelete="CASCADE"
+        />
+    </changeSet>
+
+    <changeSet author="rmunshi" id="custom_label_entry_index">
+        <addUniqueConstraint constraintName="UC_CUSTOM_LABEL_ENTRY_CLK_CLV_WEU"
+                             tableName="CUSTOM_LABEL_ENTRY" columnNames="CUSTOM_LABEL_KEY, CUSTOM_LABEL_VALUE, WORKFLOW_EXECUTION_UUID" />
+
+    </changeSet>
+
+    <changeSet author="rmunshi" id="modify label key/value data types">
+        <modifyDataType tableName="CUSTOM_LABEL_ENTRY" columnName="CUSTOM_LABEL_KEY" newDataType="VARCHAR(63)"/>
+        <modifyDataType tableName="CUSTOM_LABEL_ENTRY" columnName="CUSTOM_LABEL_VALUE" newDataType="VARCHAR(63)"/>
+    </changeSet>
+
+
+</databaseChangeLog>
diff --git a/database/migration/src/main/resources/changesets/docker_hash_store.xml b/database/migration/src/main/resources/changesets/docker_hash_store.xml
new file mode 100644
index 000000000..7ec9fbbc4
--- /dev/null
+++ b/database/migration/src/main/resources/changesets/docker_hash_store.xml
@@ -0,0 +1,36 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
+
+    <changeSet author="tjeandet" id="DOCKER_HASH_STORE_ENTRY">
+        <comment>
+            Temporary storage area for docker hashes from workflows that are still in progress.
+        </comment>
+        <createTable tableName="DOCKER_HASH_STORE_ENTRY">
+            <column autoIncrement="true" name="DOCKER_HASH_STORE_ENTRY_ID" type="INT">
+                <constraints primaryKey="true" primaryKeyName="PK_DOCKER_HASH_STORE_ENTRY"/>
+            </column>
+            <column name="WORKFLOW_EXECUTION_UUID" type="VARCHAR(255)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="DOCKER_TAG" type="VARCHAR(255)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="DOCKER_HASH" type="VARCHAR(255)">
+                <constraints nullable="false"/>
+            </column>
+        </createTable>
+        <modifySql dbms="mysql">
+            <append value=" ENGINE=INNODB"/>
+        </modifySql>
+    </changeSet>
+
+    <changeSet author="tjeandet" id="docker_hash_store_uuid_unique_constraint">
+        <addUniqueConstraint
+                constraintName="UC_DOCKER_HASH_STORE_ENTRY_WEU_DT"
+                tableName="DOCKER_HASH_STORE_ENTRY"
+                columnNames="WORKFLOW_EXECUTION_UUID, DOCKER_TAG" />
+    </changeSet>
+
+</databaseChangeLog>
diff --git a/database/migration/src/main/resources/changesets/failure_metadata.xml b/database/migration/src/main/resources/changesets/failure_metadata.xml
new file mode 100644
index 000000000..55189d5cf
--- /dev/null
+++ b/database/migration/src/main/resources/changesets/failure_metadata.xml
@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
+
+    <changeSet author="cjllanwarne" id="failure_to_message" dbms="mysql">
+        <sqlFile relativeToChangelogFile="true" path="failure_metadata_scripts/failure_to_message.sql" />
+    </changeSet>
+
+    <changeSet author="cjllanwarne" id="remove_failure_timestamp" dbms="mysql">
+        <delete tableName="METADATA_ENTRY">
+            <where>METADATA_KEY LIKE '%failures[%]:timestamp'</where>
+        </delete>
+    </changeSet>
+
+    <changeSet author="cjllanwarne" id="causedByLists" dbms="mysql">
+        <sqlFile relativeToChangelogFile="true" path="failure_metadata_scripts/causedBy_lists.sql" />
+    </changeSet>
+
+    <changeSet author="cjllanwarne" id="deduplicate_failure_messages" dbms="mysql">
+        <customChange class="cromwell.database.migration.failuremetadata.DeduplicateFailureMessageIds"/>
+    </changeSet>
+
+    <changeSet author="cjllanwarne" id="guaranteed_caused_bys" dbms="mysql">
+        <sqlFile relativeToChangelogFile="true" path="failure_metadata_scripts/guarantee_causedBy.sql" />
+    </changeSet>
+
+</databaseChangeLog>
diff --git a/database/migration/src/main/resources/changesets/failure_metadata_2.xml b/database/migration/src/main/resources/changesets/failure_metadata_2.xml
new file mode 100644
index 000000000..152c956ab
--- /dev/null
+++ b/database/migration/src/main/resources/changesets/failure_metadata_2.xml
@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
+
+    <changeSet author="cjllanwarne" id="expand_single_failure_strings" dbms="mysql">
+        <customChange class="cromwell.database.migration.failuremetadata.ExpandSingleFailureStrings"/>
+    </changeSet>
+
+</databaseChangeLog>
diff --git a/database/migration/src/main/resources/changesets/failure_metadata_scripts/causedBy_lists.sql b/database/migration/src/main/resources/changesets/failure_metadata_scripts/causedBy_lists.sql
new file mode 100644
index 000000000..a3ec702ab
--- /dev/null
+++ b/database/migration/src/main/resources/changesets/failure_metadata_scripts/causedBy_lists.sql
@@ -0,0 +1,3 @@
+UPDATE METADATA_ENTRY
+SET METADATA_KEY = REPLACE(METADATA_KEY, "causedBy:", "causedBy[0]:")
+WHERE METADATA_KEY LIKE "%failures%causedBy:%";
diff --git a/database/migration/src/main/resources/changesets/failure_metadata_scripts/failure_to_message.sql b/database/migration/src/main/resources/changesets/failure_metadata_scripts/failure_to_message.sql
new file mode 100644
index 000000000..ccd193226
--- /dev/null
+++ b/database/migration/src/main/resources/changesets/failure_metadata_scripts/failure_to_message.sql
@@ -0,0 +1,3 @@
+UPDATE METADATA_ENTRY
+SET METADATA_KEY = CONCAT(TRIM(TRAILING ':failure' FROM METADATA_KEY), ":message")
+WHERE METADATA_KEY LIKE "%failures[%]%:failure";
diff --git a/database/migration/src/main/resources/changesets/failure_metadata_scripts/guarantee_causedBy.sql b/database/migration/src/main/resources/changesets/failure_metadata_scripts/guarantee_causedBy.sql
new file mode 100644
index 000000000..dc5648c25
--- /dev/null
+++ b/database/migration/src/main/resources/changesets/failure_metadata_scripts/guarantee_causedBy.sql
@@ -0,0 +1,13 @@
+INSERT INTO METADATA_ENTRY (WORKFLOW_EXECUTION_UUID, METADATA_KEY, CALL_FQN, JOB_SCATTER_INDEX, JOB_RETRY_ATTEMPT, METADATA_TIMESTAMP)
+SELECT t1.WORKFLOW_EXECUTION_UUID, CONCAT(TRIM(TRAILING ':message' FROM t1.METADATA_KEY), ":causedBy[]"), t1.CALL_FQN, t1.JOB_SCATTER_INDEX, t1.JOB_RETRY_ATTEMPT, t1.METADATA_TIMESTAMP
+FROM METADATA_ENTRY AS t1
+WHERE METADATA_KEY LIKE '%failures[%]%:message'
+AND NOT EXISTS (SELECT *
+             	FROM METADATA_ENTRY AS t2
+             	WHERE t2.WORKFLOW_EXECUTION_UUID = t1.WORKFLOW_EXECUTION_UUID
+             	  AND (t2.CALL_FQN = t1.CALL_FQN OR (t2.CALL_FQN IS NULL AND t1.CALL_FQN IS NULL))
+             	  AND (t2.JOB_SCATTER_INDEX = t1.JOB_SCATTER_INDEX OR (t2.JOB_SCATTER_INDEX IS NULL AND t1.JOB_SCATTER_INDEX IS NULL))
+             	  AND (t2.JOB_RETRY_ATTEMPT = t1.JOB_RETRY_ATTEMPT OR (t2.JOB_RETRY_ATTEMPT IS NULL AND t1.JOB_RETRY_ATTEMPT IS NULL))
+                  AND t2.METADATA_KEY LIKE CONCAT(TRIM(TRAILING ':message' FROM t1.METADATA_KEY), ":causedBy[%")
+                  AND t2.METADATA_JOURNAL_ID <> t1.METADATA_JOURNAL_ID
+               )
diff --git a/database/migration/src/main/resources/changesets/migration/workflow_store/replace_empty_custom_labels.sql b/database/migration/src/main/resources/changesets/migration/workflow_store/replace_empty_custom_labels.sql
new file mode 100644
index 000000000..1bd62c730
--- /dev/null
+++ b/database/migration/src/main/resources/changesets/migration/workflow_store/replace_empty_custom_labels.sql
@@ -0,0 +1,3 @@
+UPDATE WORKFLOW_STORE_ENTRY
+SET CUSTOM_LABELS = "{}"
+WHERE CUSTOM_LABELS = "";
diff --git a/database/migration/src/main/resources/changesets/preempted_is_a_backend_status.xml b/database/migration/src/main/resources/changesets/preempted_is_a_backend_status.xml
new file mode 100644
index 000000000..6670b13e9
--- /dev/null
+++ b/database/migration/src/main/resources/changesets/preempted_is_a_backend_status.xml
@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
+
+    <changeSet author="cjllanwarne" id="preempted_is_a_backend_status" dbms="mysql">
+        <customChange class="cromwell.database.migration.custom.PreemptedIsABackendStatus"/>
+    </changeSet>
+
+</databaseChangeLog>
diff --git a/database/migration/src/main/resources/changesets/remove_pre_pbe_tables.xml b/database/migration/src/main/resources/changesets/remove_pre_pbe_tables.xml
new file mode 100644
index 000000000..e2892b91b
--- /dev/null
+++ b/database/migration/src/main/resources/changesets/remove_pre_pbe_tables.xml
@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
+
+    <changeSet author="mcovarr" id="drop-pre-pbe-tables">
+        <comment>Remove the old pre-pluggable backend tables.</comment>
+        <dropTable tableName="EXECUTION_INFO" />
+        <dropTable tableName="EXECUTION_EVENT" />
+        <dropTable tableName="FAILURE_EVENT" />
+        <dropTable tableName="RUNTIME_ATTRIBUTES" />
+        <dropTable tableName="SYMBOL" />
+        <dropTable tableName="EXECUTION" />
+        <dropTable tableName="WORKFLOW_EXECUTION_AUX" />
+        <dropTable tableName="WORKFLOW_EXECUTION" />
+    </changeSet>
+
+</databaseChangeLog>
diff --git a/database/migration/src/main/resources/changesets/replace_empty_custom_labels.xml b/database/migration/src/main/resources/changesets/replace_empty_custom_labels.xml
new file mode 100644
index 000000000..508497a89
--- /dev/null
+++ b/database/migration/src/main/resources/changesets/replace_empty_custom_labels.xml
@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
+
+    <changeSet author="rmunshi" id="replace_empty_custom_labels" dbms="mysql">
+        <sqlFile relativeToChangelogFile="true" path="migration/workflow_store/replace_empty_custom_labels.sql" />
+    </changeSet>
+
+    <changeSet id="custom_labels_not_null" author="rmunshi">
+        <addNotNullConstraint tableName="WORKFLOW_STORE_ENTRY"
+                              columnName="CUSTOM_LABELS"
+                              columnDataType="LONGTEXT"
+                              defaultNullValue="{}"/>
+    </changeSet>
+
+</databaseChangeLog>
diff --git a/database/migration/src/main/resources/changesets/restart_and_recover_migration.xml b/database/migration/src/main/resources/changesets/restart_and_recover_migration.xml
index a594ea4fc..6cab40644 100644
--- a/database/migration/src/main/resources/changesets/restart_and_recover_migration.xml
+++ b/database/migration/src/main/resources/changesets/restart_and_recover_migration.xml
@@ -76,6 +76,23 @@
         </sql>
     </changeSet>
 
+    <changeSet id="JOB_STORE_RESULT_SIMPLETON_VALUE_NULLABLE" author="mcovarr">
+        <preConditions onFail="MARK_RAN">
+            <tableExists tableName="JOB_STORE_RESULT_SIMPLETON"/>
+        </preConditions>
+        <comment>
+            This changeset is a retroactive insertion into the changeset order to enable migration of
+            null output values in a 0.19 Cromwell installation to the job store simpleton table introduced
+            in pluggable backend Cromwell.  Because this is a retroactive insertion, the "tableExists"
+            precondition keeps this changeset from running on a database that has already been migrated to
+            the pluggable backend schema (this migration is unnecessary in a database that has already been
+            migrated to the pluggable backend schema since a later changeset also drops this constraint).
+        </comment>
+        <dropNotNullConstraint tableName="JOB_STORE_RESULT_SIMPLETON"
+                               columnName="SIMPLETON_VALUE"
+                               columnDataType="LONGTEXT"/>
+    </changeSet>
+
     <changeSet id="JOB_STORE_RESULT_SIMPLETON_MIGRATION" author="mcovarr" >
         <comment>
             Restart/recover migration from 0.19 to 0.21.
diff --git a/database/migration/src/main/resources/changesets/workflow_store_type_and_version.xml b/database/migration/src/main/resources/changesets/workflow_store_type_and_version.xml
new file mode 100644
index 000000000..bdb38ad09
--- /dev/null
+++ b/database/migration/src/main/resources/changesets/workflow_store_type_and_version.xml
@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
+
+    <changeSet author="mcovarr" id="workflow-store-type-and-version">
+        <addColumn tableName="WORKFLOW_STORE_ENTRY">
+            <column name="WORKFLOW_TYPE" type="VARCHAR(30)">
+                <constraints nullable="true"/>
+            </column>
+            <column name="WORKFLOW_TYPE_VERSION" type="VARCHAR(255)">
+                <constraints nullable="true"/>
+            </column>
+        </addColumn>
+    </changeSet>
+
+</databaseChangeLog>
diff --git a/database/migration/src/main/scala/cromwell/database/migration/WdlTransformation.scala b/database/migration/src/main/scala/cromwell/database/migration/WdlTransformation.scala
index 84f38a7db..aaf3289a0 100644
--- a/database/migration/src/main/scala/cromwell/database/migration/WdlTransformation.scala
+++ b/database/migration/src/main/scala/cromwell/database/migration/WdlTransformation.scala
@@ -6,19 +6,23 @@ import java.util.zip.GZIPInputStream
 
 import org.apache.commons.codec.binary.Base64
 import org.apache.commons.io.IOUtils
-import wdl4s.types.{WdlPrimitiveType, WdlType}
+import wdl4s.wdl.types.{WdlPrimitiveType, WdlType}
 
 import scala.util.Try
 
 private [migration] object WdlTransformation {
 
   def inflate(value: String): Try[String] = Try {
-    IOUtils.toString(new GZIPInputStream(new ByteArrayInputStream(Base64.decodeBase64(value))), Charset.defaultCharset)
+    Option(value) match {
+      case Some(v) => IOUtils.toString(new GZIPInputStream(new ByteArrayInputStream(Base64.decodeBase64(v))), Charset.defaultCharset)
+      case None => null
+    }
   } recover {
-    case e: IOException => value
+    case _: IOException => value
   }
 
   def coerceStringToWdl(wdlString: String, wdlType: WdlType) = wdlType match {
+    case _ if wdlString == null => null
     case p: WdlPrimitiveType => p.coerceRawValue(wdlString).get
     case o => o.fromWdlString(wdlString)
   }
diff --git a/database/migration/src/main/scala/cromwell/database/migration/custom/BatchedTaskChange.scala b/database/migration/src/main/scala/cromwell/database/migration/custom/BatchedTaskChange.scala
index 9339dc5d7..49db7e444 100644
--- a/database/migration/src/main/scala/cromwell/database/migration/custom/BatchedTaskChange.scala
+++ b/database/migration/src/main/scala/cromwell/database/migration/custom/BatchedTaskChange.scala
@@ -34,7 +34,8 @@ trait BatchedTaskChange extends MigrationTaskChange {
   def readBatchQuery: String
 
   /**
-    * Used to prepare the statement that will be passed repeatedly into migrateBatchRow.
+    * Used to prepare query statements that can be passed repeatedly into migrateBatchRow. Examples of
+    * individual elements follow.
     *
     * Example:
     * {{{
@@ -49,7 +50,7 @@ trait BatchedTaskChange extends MigrationTaskChange {
     *   SET [COLUMNS];
     * }}}
     */
-  def migrateBatchQuery: String
+  def migrateBatchQueries: List[String]
 
   /**
     * Migrate a row.
@@ -58,10 +59,10 @@ trait BatchedTaskChange extends MigrationTaskChange {
     * migrateStatement.addBatch(). Return the (estimated) number of rows to be written by this batch.
     *
     * @param readRow          The row to migrate
-    * @param migrateStatement The statement to add a new migrated row
+    * @param migrateStatements The statements used to migrate a row
     * @return The number of rows updated
     */
-  def migrateBatchRow(readRow: ResultSet, migrateStatement: PreparedStatement): Int
+  def migrateBatchRow(readRow: ResultSet, migrateStatements: List[PreparedStatement]): Int
 
   /**
     * Specify the size of a "page".
@@ -77,7 +78,7 @@ trait BatchedTaskChange extends MigrationTaskChange {
     */
   private val writeBatchSize = config.getInt("database.migration.write-batch-size")
 
-  override def migrate(connection: JdbcConnection) = {
+  override def migrate(connection: JdbcConnection): Unit = {
 
     logger.info(s"Running migration $migrationName with a read batch size of " +
       s"$readBatchSize and a write batch size of $writeBatchSize")
@@ -95,7 +96,7 @@ trait BatchedTaskChange extends MigrationTaskChange {
     val pageCount = Math.max(readCount / readBatchSize, 1)
 
     val readBatchStatement = connection.prepareStatement(readBatchQuery)
-    val migrateBatchStatement = connection.prepareStatement(migrateBatchQuery)
+    val migrateBatchStatements = migrateBatchQueries map connection.prepareStatement
 
     val paginator = new QueryPaginator(readBatchStatement, readBatchSize, readCount)
 
@@ -103,13 +104,12 @@ trait BatchedTaskChange extends MigrationTaskChange {
     paginator.zipWithIndex foreach {
       case (resultBatch, page) =>
         // Loop over rows in page
-        new ResultSetIterator(resultBatch).zipWithIndex foreach {
-          case (row, idx) =>
-            batchMigrationCounter += migrateBatchRow(row, migrateBatchStatement)
+        new ResultSetIterator(resultBatch) foreach { row =>
+            batchMigrationCounter += migrateBatchRow(row, migrateBatchStatements)
             // batchMigrationCounter can actually be bigger than writeBatchSize as wdlValues are processed atomically,
             // so this is a best effort
             if (batchMigrationCounter >= writeBatchSize) {
-              migrateBatchStatement.executeBatch()
+              migrateBatchStatements.foreach(_.executeBatch())
               connection.commit()
               batchMigrationCounter = 0
             }
@@ -118,11 +118,12 @@ trait BatchedTaskChange extends MigrationTaskChange {
         resultBatch.close()
 
         val progress = Math.min((page + 1) * 100 / pageCount, 100)
-        logger.info(s"[$migrationName] $progress%")
+        val progressMessage = s"[$migrationName] $progress%"
+        logger.info(progressMessage)
     }
 
     if (batchMigrationCounter != 0) {
-      migrateBatchStatement.executeBatch()
+      migrateBatchStatements.foreach(_.executeBatch())
       connection.commit()
     }
   }
diff --git a/database/migration/src/main/scala/cromwell/database/migration/custom/PreemptedIsABackendStatus.scala b/database/migration/src/main/scala/cromwell/database/migration/custom/PreemptedIsABackendStatus.scala
new file mode 100644
index 000000000..f534b455f
--- /dev/null
+++ b/database/migration/src/main/scala/cromwell/database/migration/custom/PreemptedIsABackendStatus.scala
@@ -0,0 +1,65 @@
+package cromwell.database.migration.custom
+
+import java.sql.{PreparedStatement, ResultSet}
+
+/**
+  * Makes sure that it's executionStatuses that are 'Failed' and backendStatuses can be 'Preempted'
+  */
+class PreemptedIsABackendStatus extends BatchedTaskChange {
+  val tableName = "METADATA_ENTRY"
+  val primaryKeyColumn = "METADATA_JOURNAL_ID"
+  val workflowIdColumn = "WORKFLOW_EXECUTION_UUID"
+  val metadataKeyColumn = "METADATA_KEY"
+  val callFqnColumn = "CALL_FQN"
+  val jobScatterIndexColumn = "JOB_SCATTER_INDEX"
+  val retryAttemptColumn = "JOB_RETRY_ATTEMPT"
+  val timestampColumn = "METADATA_TIMESTAMP"
+  val metadataValueColumn = "METADATA_VALUE"
+  val metadataValueTypeColumn = "METADATA_VALUE_TYPE"
+
+  val messageFilter = "METADATA_KEY = 'executionStatus' && METADATA_VALUE = 'Preempted'"
+
+  override def readCountQuery = s"SELECT MAX($primaryKeyColumn) FROM $tableName"
+
+  /**
+    * Find metadata keys in this range which are duplicates of earlier keys (for the same workflow).
+    */
+  override def readBatchQuery =
+    s"""|SELECT $primaryKeyColumn, $workflowIdColumn, $callFqnColumn, $jobScatterIndexColumn, $retryAttemptColumn, $timestampColumn
+        | FROM $tableName AS t1
+        | WHERE t1.$primaryKeyColumn >= ? AND t1.$primaryKeyColumn < ?
+        | AND $messageFilter
+        |""".stripMargin
+
+  override def migrateBatchQueries = List(
+    s"UPDATE $tableName SET $metadataKeyColumn = 'backendStatus' WHERE $primaryKeyColumn = ?",
+    s"INSERT INTO $tableName ($workflowIdColumn, $callFqnColumn, $jobScatterIndexColumn, $retryAttemptColumn, $timestampColumn, $metadataKeyColumn, $metadataValueColumn, $metadataValueTypeColumn) " +
+      s"VALUES (?, ?, ?, ?, ?, 'executionStatus', 'RetryableFailure', 'string')"
+  )
+
+  override def migrateBatchRow(readRow: ResultSet, migrateStatements: List[PreparedStatement]): Int = {
+    val rowId = readRow.getInt(primaryKeyColumn)
+    val workflowId = readRow.getString(workflowIdColumn)
+    val callFqn = readRow.getString(callFqnColumn)
+    val jobScatterIndex = readRow.getObject(jobScatterIndexColumn)
+    val retryAttempt = readRow.getObject(retryAttemptColumn)
+    val metadataTimestamp = readRow.getObject(timestampColumn)
+
+    // First, update the executionStatus to be a backendStatus
+    val updateStatement = migrateStatements.head
+    updateStatement.setInt(1, rowId)
+    updateStatement.addBatch()
+
+    // Second, insert the new 'RetryableFailure' executionStatus
+    val insertStatement = migrateStatements(1)
+    insertStatement.setString(1, workflowId)
+    insertStatement.setString(2, callFqn)
+    insertStatement.setObject(3, jobScatterIndex)
+    insertStatement.setObject(4, retryAttempt)
+    insertStatement.setObject(5, metadataTimestamp)
+    insertStatement.addBatch()
+
+    // We added two queries here, so that's what we return:
+    2
+  }
+}
diff --git a/database/migration/src/main/scala/cromwell/database/migration/failuremetadata/DeduplicateFailureMessageIds.scala b/database/migration/src/main/scala/cromwell/database/migration/failuremetadata/DeduplicateFailureMessageIds.scala
new file mode 100644
index 000000000..74ab0769b
--- /dev/null
+++ b/database/migration/src/main/scala/cromwell/database/migration/failuremetadata/DeduplicateFailureMessageIds.scala
@@ -0,0 +1,56 @@
+package cromwell.database.migration.failuremetadata
+
+import java.sql.{PreparedStatement, ResultSet}
+
+import cromwell.database.migration.custom.BatchedTaskChange
+
+import scala.util.Random
+
+/**
+  * Turns 'causedBy' objects into lists in the metadata.
+  */
+class DeduplicateFailureMessageIds extends BatchedTaskChange {
+  val tableName = "METADATA_ENTRY"
+  val primaryKeyColumn = "METADATA_JOURNAL_ID"
+  val workflowIdColumn = "WORKFLOW_EXECUTION_UUID"
+  val metadataKeyColumn = "METADATA_KEY"
+  val callFqnColumn = "CALL_FQN"
+  val jobScatterIndexColumn = "JOB_SCATTER_INDEX"
+  val retryAttemptColumn = "JOB_RETRY_ATTEMPT"
+  val contentEqualityCheck = List(workflowIdColumn, metadataKeyColumn, callFqnColumn, jobScatterIndexColumn, retryAttemptColumn)
+    .map(s => s"(t2.$s = t1.$s OR (t2.$s IS NULL AND t1.$s IS NULL))").mkString(" AND ")
+
+  val fixableFailureMessageFilter = "METADATA_KEY LIKE '%failures[%]%:message'"
+
+  override def readCountQuery = s"SELECT MAX($primaryKeyColumn) FROM $tableName;"
+
+  /**
+    * Find metadata keys in this range which are duplicates of earlier keys (for the same workflow).
+    */
+  override def readBatchQuery =
+    s"""|SELECT $primaryKeyColumn, $workflowIdColumn, $metadataKeyColumn
+        | FROM $tableName AS t1
+        | WHERE t1.$primaryKeyColumn >= ? AND t1.$primaryKeyColumn < ?
+        | AND $fixableFailureMessageFilter
+        | AND EXISTS (SELECT *
+        |              FROM $tableName AS t2
+        |              WHERE $contentEqualityCheck
+        |                AND t2.$primaryKeyColumn < t1.$primaryKeyColumn
+        |            )
+        |""".stripMargin
+
+  override def migrateBatchQueries = List(s"UPDATE $tableName SET $metadataKeyColumn = ? WHERE $primaryKeyColumn = ?;")
+
+  override def migrateBatchRow(readRow: ResultSet, migrateStatements: List[PreparedStatement]): Int = {
+    val migrateStatement = migrateStatements.head
+    val rowId = readRow.getInt(1)
+    val oldKey = readRow.getString(3)
+    val newRandomInt = Random.nextInt(Int.MaxValue)
+    val newKey = oldKey.replaceFirst("failures\\[\\d*\\]", s"failures[$newRandomInt]")
+
+    migrateStatement.setString(1, newKey)
+    migrateStatement.setInt(2, rowId)
+    migrateStatement.addBatch()
+    1
+  }
+}
diff --git a/database/migration/src/main/scala/cromwell/database/migration/failuremetadata/ExpandSingleFailureStrings.scala b/database/migration/src/main/scala/cromwell/database/migration/failuremetadata/ExpandSingleFailureStrings.scala
new file mode 100644
index 000000000..79dca7192
--- /dev/null
+++ b/database/migration/src/main/scala/cromwell/database/migration/failuremetadata/ExpandSingleFailureStrings.scala
@@ -0,0 +1,70 @@
+package cromwell.database.migration.failuremetadata
+
+import java.sql.{PreparedStatement, ResultSet}
+import cromwell.database.migration.custom.BatchedTaskChange
+
+/**
+  * Turns individual '| failures[0] | "blah" |' entries into '| failures[0]:message | "blah" |' and '| failures[0]:causedBy[] | NULL |'
+  */
+class ExpandSingleFailureStrings extends BatchedTaskChange {
+  val tableName = "METADATA_ENTRY"
+  val primaryKeyColumn = "METADATA_JOURNAL_ID"
+  val workflowIdColumn = "WORKFLOW_EXECUTION_UUID"
+  val metadataKeyColumn = "METADATA_KEY"
+  val callFqnColumn = "CALL_FQN"
+  val jobScatterIndexColumn = "JOB_SCATTER_INDEX"
+  val retryAttemptColumn = "JOB_RETRY_ATTEMPT"
+  val metadataTimestampColumn = "METADATA_TIMESTAMP"
+
+  val fixableFailureMessageFilter = "METADATA_KEY REGEXP '^failures\\\\[[0-9]*\\\\]$'"
+
+  override def readCountQuery: String = s"SELECT MAX($primaryKeyColumn) FROM $tableName;"
+
+  /**
+    * Find metadata keys in this range which can be fixed up.
+    */
+  override val readBatchQuery: String =
+    s"""|SELECT $primaryKeyColumn, $workflowIdColumn, $metadataKeyColumn, $callFqnColumn, $jobScatterIndexColumn, $retryAttemptColumn, $metadataTimestampColumn
+        | FROM $tableName
+        | WHERE $primaryKeyColumn >= ? AND $primaryKeyColumn < ?
+        | AND $fixableFailureMessageFilter
+        |""".stripMargin
+
+  override val migrateBatchQueries = List(
+    // Modify the existing element, changing failures[x] to failures[x]:message
+    s"UPDATE $tableName SET $metadataKeyColumn = ? WHERE $primaryKeyColumn = ?",
+    // Insert a new element, to contain the empty causedBy[]
+    s"""INSERT INTO $tableName ($workflowIdColumn, $metadataKeyColumn, $callFqnColumn, $jobScatterIndexColumn, $retryAttemptColumn, $metadataTimestampColumn)
+       | VALUES (?, ?, ?, ?, ?, ?)
+     """.stripMargin
+  )
+
+  override def migrateBatchRow(readRow: ResultSet, migrateStatements: List[PreparedStatement]): Int = {
+    val (modifyStatement :: insertStatement :: Nil) = migrateStatements
+
+    val rowId = readRow.getInt(primaryKeyColumn)
+    val workflowId = readRow.getString(workflowIdColumn)
+    val oldKey = readRow.getString(metadataKeyColumn)
+    val callFqn = readRow.getString(callFqnColumn)
+    val jobScatterIndex = readRow.getObject(jobScatterIndexColumn)
+    val retryAttempt = readRow.getObject(retryAttemptColumn)
+    val metadataTimestamp = readRow.getObject(metadataTimestampColumn)
+
+    val modifiedKey = oldKey + ":message"
+    val insertKey = oldKey + ":causedBy[]"
+
+    modifyStatement.setString(1, modifiedKey)
+    modifyStatement.setInt(2, rowId)
+    modifyStatement.addBatch()
+
+    insertStatement.setString(1, workflowId)
+    insertStatement.setString(2, insertKey)
+    insertStatement.setString(3, callFqn)
+    insertStatement.setObject(4, jobScatterIndex)
+    insertStatement.setObject(5, retryAttempt)
+    insertStatement.setObject(6, metadataTimestamp)
+    insertStatement.addBatch()
+
+    2
+  }
+}
diff --git a/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/CallOutputSymbolTableMigration.scala b/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/CallOutputSymbolTableMigration.scala
index 09cf42dbf..6ade3e068 100644
--- a/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/CallOutputSymbolTableMigration.scala
+++ b/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/CallOutputSymbolTableMigration.scala
@@ -2,7 +2,7 @@ package cromwell.database.migration.metadata.table.symbol
 
 import java.sql.PreparedStatement
 
-import wdl4s.values._
+import wdl4s.wdl.values._
 
 class CallOutputSymbolTableMigration extends SymbolTableMigration {
   override def processSymbol(statement: PreparedStatement,
diff --git a/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/InputSymbolTableMigration.scala b/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/InputSymbolTableMigration.scala
index 5740ac3d9..441da9387 100644
--- a/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/InputSymbolTableMigration.scala
+++ b/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/InputSymbolTableMigration.scala
@@ -2,7 +2,7 @@ package cromwell.database.migration.metadata.table.symbol
 
 import java.sql.PreparedStatement
 
-import wdl4s.values._
+import wdl4s.wdl.values._
 
 class InputSymbolTableMigration extends SymbolTableMigration {
 
diff --git a/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/MetadataStatement.scala b/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/MetadataStatement.scala
index 311b364fe..b9e5c53d0 100644
--- a/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/MetadataStatement.scala
+++ b/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/MetadataStatement.scala
@@ -5,7 +5,7 @@ import java.time.format.DateTimeFormatter
 import java.time.{OffsetDateTime, ZoneId, ZoneOffset}
 
 import org.slf4j.LoggerFactory
-import wdl4s.values.{WdlBoolean, WdlFloat, WdlInteger, WdlValue}
+import wdl4s.wdl.values._
 
 object MetadataStatement {
   val WorkflowIdIdx = 1
@@ -44,10 +44,10 @@ class MetadataStatementForWorkflow(preparedStatement: PreparedStatement, workflo
 
   private def metadataType(value: Any) = {
     value match {
-      case WdlInteger(i) => "int"
-      case WdlFloat(f) => "number"
-      case WdlBoolean(b) => "boolean"
-      case value: WdlValue => "string"
+      case WdlInteger(_) => "int"
+      case WdlFloat(_) => "number"
+      case WdlBoolean(_) => "boolean"
+      case _: WdlValue => "string"
       case _: Int | Long => "int"
       case _: Double | Float => "number"
       case _: Boolean => "boolean"
diff --git a/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/SymbolTableMigration.scala b/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/SymbolTableMigration.scala
index 19bbc1279..de1cf2d20 100644
--- a/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/SymbolTableMigration.scala
+++ b/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/SymbolTableMigration.scala
@@ -5,9 +5,9 @@ import java.sql.{PreparedStatement, ResultSet}
 import cromwell.core.simpleton.WdlValueSimpleton._
 import cromwell.database.migration.WdlTransformation
 import cromwell.database.migration.custom.BatchedTaskChange
-import wdl4s.WdlExpression
-import wdl4s.types.WdlType
-import wdl4s.values.WdlValue
+import wdl4s.wdl.WdlExpression
+import wdl4s.wdl.types.WdlType
+import wdl4s.wdl.values.WdlValue
 
 import scala.util.{Failure, Success, Try}
 
@@ -24,7 +24,7 @@ trait SymbolTableMigration extends BatchedTaskChange {
 
   override val readCountQuery = SymbolTableMigration.NbRowsQuery
 
-  override val readBatchQuery = """
+  override val readBatchQuery: String = """
       |SELECT
       |    WORKFLOW_EXECUTION_UUID,
       |    SYMBOL_NAME,
@@ -37,18 +37,19 @@ trait SymbolTableMigration extends BatchedTaskChange {
       |   WHERE TMP_SYMBOL_ID >= ? AND TMP_SYMBOL_ID < ?;
     """.stripMargin
 
-  override val migrateBatchQuery = MetadataStatement.InsertSql
+  override val migrateBatchQueries = List(MetadataStatement.InsertSql)
 
   /**
     * Migrate a row to the metadata table
     */
-  override def migrateBatchRow(row: ResultSet, statement: PreparedStatement): Int = {
+  override def migrateBatchRow(row: ResultSet, statements: List[PreparedStatement]): Int = {
+    val statement = statements.head
     // Try to coerce the value to a WdlValue
     val value = for {
       wdlType <- Try(WdlType.fromWdlString(row.getString("WDL_TYPE")))
       inflated <- row.getString("WDL_VALUE") match {
         case null => Success("") // Empty Strings are null in the DB
-        case nonNull => inflate(row.getString("WDL_VALUE"))
+        case nonNull @ _ => inflate(row.getString("WDL_VALUE"))
       }
     } yield WdlTransformation.coerceStringToWdl(inflated, wdlType)
 
diff --git a/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/WorkflowOutputSymbolTableMigration.scala b/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/WorkflowOutputSymbolTableMigration.scala
index 269d58a24..c8cab32c0 100644
--- a/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/WorkflowOutputSymbolTableMigration.scala
+++ b/database/migration/src/main/scala/cromwell/database/migration/metadata/table/symbol/WorkflowOutputSymbolTableMigration.scala
@@ -2,7 +2,7 @@ package cromwell.database.migration.metadata.table.symbol
 
 import java.sql.PreparedStatement
 
-import wdl4s.values._
+import wdl4s.wdl.values._
 
 class WorkflowOutputSymbolTableMigration extends SymbolTableMigration {
 
diff --git a/database/migration/src/main/scala/cromwell/database/migration/restart/table/JobStoreSimpletonMigration.scala b/database/migration/src/main/scala/cromwell/database/migration/restart/table/JobStoreSimpletonMigration.scala
index 9287add50..c836dbe16 100644
--- a/database/migration/src/main/scala/cromwell/database/migration/restart/table/JobStoreSimpletonMigration.scala
+++ b/database/migration/src/main/scala/cromwell/database/migration/restart/table/JobStoreSimpletonMigration.scala
@@ -3,7 +3,8 @@ package cromwell.database.migration.restart.table
 import cromwell.core.simpleton.WdlValueSimpleton._
 import cromwell.database.migration.WdlTransformation._
 import liquibase.database.jvm.JdbcConnection
-import wdl4s.types.WdlType
+import wdl4s.wdl.types.WdlType
+import wdl4s.wdl.values.WdlValue
 
 
 class JobStoreSimpletonMigration extends AbstractRestartMigration {
@@ -45,18 +46,28 @@ class JobStoreSimpletonMigration extends AbstractRestartMigration {
     lazy val insert = connection.prepareStatement(InsertJobStoreSimpleton)
     query.execute(QueryOutputsForDoneCallsInRunningWorkflows)
     val results = query.getResultSet
+
+    case class JobStoreSimpletonEntry(name: String, valueString: String, typeString: String)
+
+    def buildJobStoreSimpletonEntries(name: String, wdlValue: WdlValue, wdlType: WdlType) = Option(wdlValue) match {
+      case None => List(JobStoreSimpletonEntry(name, null, wdlType.toWdlString))
+      case Some(_) => wdlValue.simplify(name) map { s =>
+        JobStoreSimpletonEntry(s.simpletonKey, s.simpletonValue.valueString, s.simpletonValue.wdlType.toWdlString)
+      }
+    }
+
     while (results.next()) {
       val wdlType = WdlType.fromWdlString(results.getString(4))
       val rawValue = results.getString(3)
       val inflated = inflate(rawValue).get
       val wdlValue = coerceStringToWdl(inflated, wdlType)
       val name = results.getString(2)
-      val simpletons = wdlValue.simplify(name)
-      simpletons foreach { s =>
+      val jobStoreSimpletonEntry = buildJobStoreSimpletonEntries(name, wdlValue, wdlType)
+      jobStoreSimpletonEntry foreach { e =>
         insert.setInt(1, results.getInt(1))
-        insert.setString(2, s.simpletonKey)
-        insert.setString(3, s.simpletonValue.toWdlString)
-        insert.setString(4, s.simpletonValue.wdlType.toWdlString)
+        insert.setString(2, e.name)
+        insert.setString(3, e.valueString)
+        insert.setString(4, e.typeString)
         insert.execute()
       }
     }
diff --git a/database/migration/src/main/scala/cromwell/database/migration/workflowoptions/RenameWorkflowOptionsInMetadata.scala b/database/migration/src/main/scala/cromwell/database/migration/workflowoptions/RenameWorkflowOptionsInMetadata.scala
index e6842b459..40dc4533b 100644
--- a/database/migration/src/main/scala/cromwell/database/migration/workflowoptions/RenameWorkflowOptionsInMetadata.scala
+++ b/database/migration/src/main/scala/cromwell/database/migration/workflowoptions/RenameWorkflowOptionsInMetadata.scala
@@ -20,9 +20,10 @@ class RenameWorkflowOptionsInMetadata extends BatchedTaskChange {
         |  WHERE $primaryKeyColumn >= ? AND $primaryKeyColumn < ? $additionalReadBatchFilters;
         |""".stripMargin
 
-  override def migrateBatchQuery = s"UPDATE $tableName SET $workflowOptionsColumn = ? WHERE $primaryKeyColumn = ?;"
+  override def migrateBatchQueries = List(s"UPDATE $tableName SET $workflowOptionsColumn = ? WHERE $primaryKeyColumn = ?;")
 
-  override def migrateBatchRow(readRow: ResultSet, migrateStatement: PreparedStatement): Int = {
+  override def migrateBatchRow(readRow: ResultSet, migrateStatements: List[PreparedStatement]): Int = {
+    val migrateStatement = migrateStatements.head
     val rowId = readRow.getInt(1)
     
     val migratedJson = readRow.getString(2).parseJson match {
diff --git a/database/migration/src/main/scala/cromwell/database/migration/workflowoptions/WorkflowOptionsChange.scala b/database/migration/src/main/scala/cromwell/database/migration/workflowoptions/WorkflowOptionsChange.scala
index 3c1b6b68f..1b519fb65 100644
--- a/database/migration/src/main/scala/cromwell/database/migration/workflowoptions/WorkflowOptionsChange.scala
+++ b/database/migration/src/main/scala/cromwell/database/migration/workflowoptions/WorkflowOptionsChange.scala
@@ -47,9 +47,10 @@ trait WorkflowOptionsChange extends BatchedTaskChange {
         |  WHERE $primaryKeyColumn >= ? AND $primaryKeyColumn < ? $additionalReadBatchFilters;
         |""".stripMargin
 
-  override def migrateBatchQuery = s"UPDATE $tableName SET $workflowOptionsColumn = ? WHERE $primaryKeyColumn = ?;"
+  override def migrateBatchQueries = List(s"UPDATE $tableName SET $workflowOptionsColumn = ? WHERE $primaryKeyColumn = ?;")
 
-  override def migrateBatchRow(readRow: ResultSet, migrateStatement: PreparedStatement): Int = {
+  override def migrateBatchRow(readRow: ResultSet, migrateStatements: List[PreparedStatement]): Int = {
+    val migrateStatement = migrateStatements.head
     val rowId = readRow.getInt(1)
     val workflowOptionsJson = readRow.getString(2)
     WorkflowOptions.fromJsonString(workflowOptionsJson) match {
diff --git a/database/sql/src/main/scala/cromwell/database/slick/CallCachingSlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/CallCachingSlickDatabase.scala
index 518939697..266897da7 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/CallCachingSlickDatabase.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/CallCachingSlickDatabase.scala
@@ -1,9 +1,12 @@
 package cromwell.database.slick
 
 import cats.data.NonEmptyList
+import cats.instances.list._
+import cats.instances.tuple._
+import cats.syntax.foldable._
 import cromwell.database.sql._
 import cromwell.database.sql.joins.CallCachingJoin
-import cromwell.database.sql.tables.CallCachingEntry
+import cromwell.database.sql.tables._
 
 import scala.concurrent.{ExecutionContext, Future}
 
@@ -12,29 +15,70 @@ trait CallCachingSlickDatabase extends CallCachingSqlDatabase {
 
   import dataAccess.driver.api._
 
-  override def addCallCaching(callCachingJoin: CallCachingJoin)
+  override def addCallCaching(joins: Seq[CallCachingJoin], batchSize: Int)
                              (implicit ec: ExecutionContext): Future[Unit] = {
+
+    // Construct parallel lists of parent entries, hashes, simpletons, and detritus from `CallCachingJoin`s.
+    val (entries, hashes, simpletons, detritus, aggregations) = joins.toList.foldMap { j =>
+      (List(j.callCachingEntry), List(j.callCachingHashEntries), List(j.callCachingSimpletonEntries), List(j.callCachingDetritusEntries), List(j.callCachingAggregationEntry.toList)) }
+
+    // Use the supplied `assigner` function to assign parent entry row IDs into the parallel `Seq` of children entities.
+    def assignEntryIdsToChildren[C](ids: Seq[Int], groupingsOfChildren: Seq[Seq[C]], assigner: (Int, C) => C): Seq[C] = {
+      (ids zip groupingsOfChildren) flatMap { case (id, children) => children.map(assigner(id, _)) }
+    }
+
+    // Batch insert entities into the appropriate `Table`.
+    def batchInsert[E, T <: Table[E]](entries: Seq[E], tableQuery: TableQuery[T]): DBIO[_] = {
+      DBIO.sequence(entries.grouped(batchSize).map { tableQuery ++= _ })
+    }
+
+    // Functions to assign call cache entry IDs into child hash entry, simpleton, and detritus rows.
+    def hashAssigner(id: Int, hash: CallCachingHashEntry) = hash.copy(callCachingEntryId = Option(id))
+    def simpletonAssigner(id: Int, simpleton: CallCachingSimpletonEntry) = simpleton.copy(callCachingEntryId = Option(id))
+    def detritusAssigner(id: Int, detritus: CallCachingDetritusEntry) = detritus.copy(callCachingEntryId = Option(id))
+    def aggregationAssigner(id: Int, aggregation: CallCachingAggregationEntry) = aggregation.copy(callCachingEntryId = Option(id))
+
     val action = for {
-      callCachingEntryId <- dataAccess.callCachingEntryIdsAutoInc += callCachingJoin.callCachingEntry
-      _ <- dataAccess.callCachingHashEntryIdsAutoInc ++= callCachingJoin.callCachingHashEntries.
-        map(_.copy(callCachingEntryId = Option(callCachingEntryId)))
-      _ <- dataAccess.callCachingSimpletonEntryIdsAutoInc ++= callCachingJoin.callCachingSimpletonEntries.
-        map(_.copy(callCachingEntryId = Option(callCachingEntryId)))
-      _ <- dataAccess.callCachingDetritusEntryIdsAutoInc ++= callCachingJoin.callCachingDetritusEntries.
-        map(_.copy(callCachingEntryId = Option(callCachingEntryId)))
+      entryIds <- dataAccess.callCachingEntryIdsAutoInc ++= entries
+
+      hashEntries = assignEntryIdsToChildren(entryIds, hashes, hashAssigner)
+      _ <- batchInsert(hashEntries, dataAccess.callCachingHashEntries)
+
+      simpletonEntries = assignEntryIdsToChildren(entryIds, simpletons, simpletonAssigner)
+      _ <- batchInsert(simpletonEntries, dataAccess.callCachingSimpletonEntries)
+
+      detritusEntries = assignEntryIdsToChildren(entryIds, detritus, detritusAssigner)
+      _ <- batchInsert(detritusEntries, dataAccess.callCachingDetritusEntries)
+
+      aggregationEntries = assignEntryIdsToChildren(entryIds, aggregations, aggregationAssigner)
+      _ <- batchInsert(aggregationEntries, dataAccess.callCachingAggregationEntries)
     } yield ()
     runTransaction(action)
   }
 
-  override def queryCallCachingEntryIds(hashKeyHashValues: NonEmptyList[(String, String)])
-                               (implicit ec: ExecutionContext): Future[Seq[Int]] = {
-    val action = dataAccess.callCachingEntryIdsForHashKeyHashValues(hashKeyHashValues).result
+  override def hasMatchingCallCachingEntriesForBaseAggregation(baseAggregationHash: String)
+                                                              (implicit ec: ExecutionContext): Future[Boolean] = {
+    val action = dataAccess.existsCallCachingEntriesForBaseAggregationHash(baseAggregationHash).result
+
+    runTransaction(action)
+  }
+
+  override def findCacheHitForAggregation(baseAggregationHash: String, inputFilesAggregationHash: Option[String], hitNumber: Int)
+                                         (implicit ec: ExecutionContext): Future[Option[Int]] = {
+    val action = dataAccess.callCachingEntriesForAggregatedHashes(baseAggregationHash, inputFilesAggregationHash, hitNumber).result.headOption
+
+    runTransaction(action)
+  }
+
+  override def hasMatchingCallCachingEntriesForHashKeyValues(hashKeyHashValues: NonEmptyList[(String, String)])
+                                                            (implicit ec: ExecutionContext): Future[Boolean] = {
+    val action = dataAccess.existsMatchingCachingEntryIdsForHashKeyHashValues(hashKeyHashValues).result
 
     runTransaction(action)
   }
 
-  override def queryCallCaching(callCachingEntryId: Int)
-                               (implicit ec: ExecutionContext): Future[Option[CallCachingJoin]] = {
+  override def queryResultsForCacheId(callCachingEntryId: Int)
+                                     (implicit ec: ExecutionContext): Future[Option[CallCachingJoin]] = {
     val action = for {
       callCachingEntryOption <- dataAccess.
         callCachingEntriesForId(callCachingEntryId).result.headOption
@@ -43,18 +87,28 @@ trait CallCachingSlickDatabase extends CallCachingSqlDatabase {
       callCachingDetritusEntries <- dataAccess.
         callCachingDetritusEntriesForCallCachingEntryId(callCachingEntryId).result
     } yield callCachingEntryOption.map(
-      CallCachingJoin(_, Seq.empty, callCachingSimpletonEntries, callCachingDetritusEntries))
+      CallCachingJoin(_, Seq.empty, None, callCachingSimpletonEntries, callCachingDetritusEntries))
+
+    runTransaction(action)
+  }
+
+  override def cacheEntryExistsForCall(workflowExecutionUuid: String, callFqn: String, index: Int)
+                                      (implicit ec: ExecutionContext): Future[Boolean] = {
+    val action = for {
+      callCachingEntryOption <- dataAccess.
+        callCachingEntriesForWorkflowFqnIndex((workflowExecutionUuid, callFqn, index)).result.headOption
+    } yield callCachingEntryOption.nonEmpty
 
     runTransaction(action)
   }
 
   override def invalidateCall(callCachingEntryId: Int)
-                               (implicit ec: ExecutionContext): Future[Option[CallCachingEntry]] = {
+                             (implicit ec: ExecutionContext): Future[Option[CallCachingEntry]] = {
     val action = for {
       _ <- dataAccess.allowResultReuseForCallCachingEntryId(callCachingEntryId).update(false)
       callCachingEntryOption <- dataAccess.callCachingEntriesForId(callCachingEntryId).result.headOption
     } yield callCachingEntryOption
-    
+
     runTransaction(action)
   }
 }
diff --git a/database/sql/src/main/scala/cromwell/database/slick/DockerHashStoreSlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/DockerHashStoreSlickDatabase.scala
new file mode 100644
index 000000000..f59930fea
--- /dev/null
+++ b/database/sql/src/main/scala/cromwell/database/slick/DockerHashStoreSlickDatabase.scala
@@ -0,0 +1,42 @@
+package cromwell.database.slick
+
+import cats.instances.future._
+import cats.syntax.functor._
+import cromwell.database.sql.DockerHashStoreSqlDatabase
+import cromwell.database.sql.tables.DockerHashStoreEntry
+
+import scala.concurrent.{ExecutionContext, Future}
+import scala.language.postfixOps
+
+trait DockerHashStoreSlickDatabase extends DockerHashStoreSqlDatabase {
+  this: SlickDatabase =>
+
+  import dataAccess.driver.api._
+
+  /**
+    * Adds a docker hash entry to the store.
+    */
+  override def addDockerHashStoreEntry(dockerHashStoreEntry: DockerHashStoreEntry)
+                                      (implicit ec: ExecutionContext): Future[Unit] = {
+    val action = dataAccess.dockerHashStoreEntries += dockerHashStoreEntry
+    runTransaction(action) void
+  }
+
+  /**
+    * Retrieves docker hash entries for a workflow.
+    *
+    */
+  override def queryDockerHashStoreEntries(workflowExecutionUuid: String)
+                                          (implicit ec: ExecutionContext): Future[Seq[DockerHashStoreEntry]] = {
+    val action = dataAccess.dockerHashStoreEntriesForWorkflowExecutionUuid(workflowExecutionUuid).result
+    runTransaction(action)
+  }
+
+  /**
+    * Deletes docker hash entries related to a workflow, returning the number of rows affected.
+    */
+  override def removeDockerHashStoreEntries(workflowExecutionUuid: String)(implicit ec: ExecutionContext): Future[Int] = {
+    val action = dataAccess.dockerHashStoreEntriesForWorkflowExecutionUuid(workflowExecutionUuid).delete
+    runTransaction(action)
+  }
+}
diff --git a/database/sql/src/main/scala/cromwell/database/slick/JobStoreSlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/JobStoreSlickDatabase.scala
index 829ddd86a..7f910c35e 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/JobStoreSlickDatabase.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/JobStoreSlickDatabase.scala
@@ -1,30 +1,35 @@
 package cromwell.database.slick
 
-import cats.instances.future._
-import cats.syntax.functor._
 import cromwell.database.sql.JobStoreSqlDatabase
 import cromwell.database.sql.joins.JobStoreJoin
+import cromwell.database.sql.tables.JobStoreSimpletonEntry
 
 import scala.concurrent.{ExecutionContext, Future}
-import scala.language.postfixOps
 
 trait JobStoreSlickDatabase extends JobStoreSqlDatabase {
   this: SlickDatabase =>
 
   import dataAccess.driver.api._
 
-  private def addJobStore(jobStoreJoin: JobStoreJoin)(implicit ec: ExecutionContext): DBIO[Unit] = {
-    for {
-      jobStoreEntryId <- dataAccess.jobStoreEntryIdsAutoInc += jobStoreJoin.jobStoreEntry
-      _ <- dataAccess.jobStoreSimpletonEntryIdsAutoInc ++=
-        jobStoreJoin.jobStoreSimpletonEntries.map(_.copy(jobStoreEntryId = Option(jobStoreEntryId)))
+  override def addJobStores(jobStoreJoins: Seq[JobStoreJoin], batchSize: Int)
+                           (implicit ec: ExecutionContext): Future[Unit] = {
+
+    def assignJobStoreIdsToSimpletons(jobStoreIds: Seq[Int]): Seq[JobStoreSimpletonEntry] = {
+      val simpletonsByJobStoreEntry = jobStoreJoins map { _.jobStoreSimpletonEntries }
+      val jobStoreIdsAndSimpletons = jobStoreIds.toList zip simpletonsByJobStoreEntry
+      jobStoreIdsAndSimpletons flatMap { case (id, simpletons) => simpletons.map(_.copy(jobStoreEntryId = Option(id))) }
+    }
+
+    val jobStoreEntries = jobStoreJoins map { _.jobStoreEntry }
+
+    val action = for {
+      jobStoreIds <- dataAccess.jobStoreEntryIdsAutoInc ++= jobStoreEntries.toList
+      simpletons = assignJobStoreIdsToSimpletons(jobStoreIds)
+      simpletonInserts = simpletons.grouped(batchSize) map { dataAccess.jobStoreSimpletonEntries ++= _ }
+      _ <- DBIO.sequence(simpletonInserts)
     } yield ()
-  }
 
-  override def addJobStores(jobStoreJoins: Seq[JobStoreJoin])
-                           (implicit ec: ExecutionContext): Future[Unit] = {
-    val action = DBIO.sequence(jobStoreJoins map addJobStore)
-    runTransaction(action) void
+    runTransaction(action)
   }
 
   override def queryJobStores(workflowExecutionUuid: String, callFqn: String, jobScatterIndex: Int,
diff --git a/database/sql/src/main/scala/cromwell/database/slick/MetadataSlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/MetadataSlickDatabase.scala
index fd22db710..2832eb477 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/MetadataSlickDatabase.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/MetadataSlickDatabase.scala
@@ -4,7 +4,9 @@ import java.sql.Timestamp
 
 import cats.data.NonEmptyList
 import cromwell.database.sql.MetadataSqlDatabase
-import cromwell.database.sql.tables.{MetadataEntry, WorkflowMetadataSummaryEntry}
+import cromwell.database.sql.tables.{CustomLabelEntry, MetadataEntry, WorkflowMetadataSummaryEntry}
+import cromwell.database.sql.SqlConverters._
+import cromwell.database.sql.joins.{CallOrWorkflowQuery, CallQuery, MetadataJobQueryValue, WorkflowQuery}
 
 import scala.concurrent.{ExecutionContext, Future}
 
@@ -41,7 +43,7 @@ trait MetadataSlickDatabase extends MetadataSqlDatabase {
   override def queryMetadataEntries(workflowExecutionUuid: String,
                                     callFullyQualifiedName: String,
                                     jobIndex: Option[Int],
-                                    jobAttempt: Int)
+                                    jobAttempt: Option[Int])
                                    (implicit ec: ExecutionContext): Future[Seq[MetadataEntry]] = {
     val action = dataAccess.
       metadataEntriesForJobKey((workflowExecutionUuid, callFullyQualifiedName, jobIndex, jobAttempt)).result
@@ -52,7 +54,7 @@ trait MetadataSlickDatabase extends MetadataSqlDatabase {
                                     metadataKey: String,
                                     callFullyQualifiedName: String,
                                     jobIndex: Option[Int],
-                                    jobAttempt: Int)
+                                    jobAttempt: Option[Int])
                                    (implicit ec: ExecutionContext): Future[Seq[MetadataEntry]] = {
     val action = dataAccess.metadataEntriesForJobKeyAndMetadataKey((
       workflowUuid, metadataKey, callFullyQualifiedName, jobIndex, jobAttempt)).result
@@ -61,19 +63,28 @@ trait MetadataSlickDatabase extends MetadataSqlDatabase {
 
   override def queryMetadataEntriesLikeMetadataKeys(workflowExecutionUuid: String,
                                                     metadataKeys: NonEmptyList[String],
-                                                    requireEmptyJobKey: Boolean)
+                                                    metadataJobQueryValue: MetadataJobQueryValue)
                                                    (implicit ec: ExecutionContext): Future[Seq[MetadataEntry]] = {
-    val action =
-      dataAccess.metadataEntriesLikeMetadataKeys(workflowExecutionUuid, metadataKeys, requireEmptyJobKey).result
+    val action = metadataJobQueryValue match {
+      case CallQuery(callFqn, jobIndex, jobAttempt) =>
+        dataAccess.metadataEntriesLikeMetadataKeysWithJob(workflowExecutionUuid, metadataKeys, callFqn, jobIndex, jobAttempt).result
+      case WorkflowQuery => dataAccess.metadataEntriesLikeMetadataKeys(workflowExecutionUuid, metadataKeys, requireEmptyJobKey = true).result
+      case CallOrWorkflowQuery => dataAccess.metadataEntriesLikeMetadataKeys(workflowExecutionUuid, metadataKeys, requireEmptyJobKey = false).result
+    }
+      
     runTransaction(action)
   }
 
   override def queryMetadataEntryNotLikeMetadataKeys(workflowExecutionUuid: String,
                                                      metadataKeys: NonEmptyList[String],
-                                                     requireEmptyJobKey: Boolean)
+                                                     metadataJobQueryValue: MetadataJobQueryValue)
                                                     (implicit ec: ExecutionContext): Future[Seq[MetadataEntry]] = {
-    val action =
-      dataAccess.metadataEntriesNotLikeMetadataKeys(workflowExecutionUuid, metadataKeys, requireEmptyJobKey).result
+    val action = metadataJobQueryValue match {
+      case CallQuery(callFqn, jobIndex, jobAttempt) =>
+        dataAccess.metadataEntriesNotLikeMetadataKeysWithJob(workflowExecutionUuid, metadataKeys, callFqn, jobIndex, jobAttempt).result
+      case WorkflowQuery => dataAccess.metadataEntriesNotLikeMetadataKeys(workflowExecutionUuid, metadataKeys, requireEmptyJobKey = true).result
+      case CallOrWorkflowQuery => dataAccess.metadataEntriesNotLikeMetadataKeys(workflowExecutionUuid, metadataKeys, requireEmptyJobKey = false).result
+    }
     runTransaction(action)
   }
 
@@ -93,6 +104,25 @@ trait MetadataSlickDatabase extends MetadataSqlDatabase {
     } yield ()
   }
 
+  private def upsertCustomLabelEntry(metadataEntry: MetadataEntry)
+                                    (implicit ec: ExecutionContext): DBIO[Unit] = {
+    //Extracting the label key from the MetadataEntry key
+    val labelKey = metadataEntry.metadataKey.split("\\:", 2)(1)
+    val labelValue = metadataEntry.metadataValue.toRawString
+    val customLabelEntry = CustomLabelEntry(labelKey, labelValue, metadataEntry.workflowExecutionUuid)
+    for {
+      exists <- dataAccess.existsWorkflowIdLabelKeyAndValue((
+        customLabelEntry.workflowExecutionUuid,
+        customLabelEntry.customLabelKey,
+        customLabelEntry.customLabelValue)).result
+      _ <- if (!exists) {
+        dataAccess.customLabelEntryIdsAutoInc += customLabelEntry
+      } else {
+        DBIO.successful(())
+      }
+    } yield ()
+  }
+
   private def upsertWorkflowMetadataSummaryEntry(workflowMetadataSummaryEntry: WorkflowMetadataSummaryEntry)
                                                 (implicit ec: ExecutionContext): DBIO[Unit] = {
     if (useSlickUpserts) {
@@ -112,19 +142,22 @@ trait MetadataSlickDatabase extends MetadataSqlDatabase {
     }
   }
 
-  override def refreshMetadataSummaryEntries(metadataKey1: String, metadataKey2: String, metadataKey3: String,
-                                             metadataKey4: String, buildUpdatedSummary:
+  override def refreshMetadataSummaryEntries(startMetadataKey: String, endMetadataKey: String, nameMetadataKey: String,
+                                             statusMetadataKey: String, labelMetadataKey: String, buildUpdatedSummary:
                                              (Option[WorkflowMetadataSummaryEntry], Seq[MetadataEntry]) =>
                                                WorkflowMetadataSummaryEntry)
                                             (implicit ec: ExecutionContext): Future[Long] = {
+    val likeMetadataLabelKey = labelMetadataKey + "%"
     val action = for {
       previousMetadataEntryIdOption <- getSummaryStatusEntryMaximumId(
         "WORKFLOW_METADATA_SUMMARY_ENTRY", "METADATA_ENTRY")
       previousMetadataEntryId = previousMetadataEntryIdOption.getOrElse(0L)
       metadataEntries <- dataAccess.metadataEntriesForIdGreaterThanOrEqual((
-        previousMetadataEntryId + 1L, metadataKey1, metadataKey2, metadataKey3, metadataKey4)).result
-      metadataByWorkflowUuid = metadataEntries.groupBy(_.workflowExecutionUuid)
-      _ <- DBIO.sequence(metadataByWorkflowUuid map updateWorkflowMetadataSummaryEntry(buildUpdatedSummary))
+        previousMetadataEntryId + 1L, startMetadataKey, endMetadataKey, nameMetadataKey, statusMetadataKey, likeMetadataLabelKey)).result
+      metadataWithoutLabels = metadataEntries.filterNot(_.metadataKey.contains(labelMetadataKey)).groupBy(_.workflowExecutionUuid)
+      customLabelEntries = metadataEntries.filter(_.metadataKey.contains(labelMetadataKey))
+      _ <- DBIO.sequence(metadataWithoutLabels map updateWorkflowMetadataSummaryEntry(buildUpdatedSummary))
+      _ <- DBIO.sequence(customLabelEntries map upsertCustomLabelEntry)
       maximumMetadataEntryId = previousOrMaximum(previousMetadataEntryId, metadataEntries.map(_.metadataEntryId.get))
       _ <- upsertSummaryStatusEntryMaximumId(
         "WORKFLOW_METADATA_SUMMARY_ENTRY", "METADATA_ENTRY", maximumMetadataEntryId)
@@ -142,20 +175,22 @@ trait MetadataSlickDatabase extends MetadataSqlDatabase {
   }
 
   override def queryWorkflowSummaries(workflowStatuses: Set[String], workflowNames: Set[String],
-                                      workflowExecutionUuids: Set[String], startTimestampOption: Option[Timestamp],
+                                      workflowExecutionUuids: Set[String], labelKeyLabelValues: Set[(String,String)],
+                                      startTimestampOption: Option[Timestamp],
                                       endTimestampOption: Option[Timestamp], page: Option[Int], pageSize: Option[Int])
                                      (implicit ec: ExecutionContext): Future[Seq[WorkflowMetadataSummaryEntry]] = {
     val action = dataAccess.queryWorkflowMetadataSummaryEntries(workflowStatuses, workflowNames, workflowExecutionUuids,
-      startTimestampOption, endTimestampOption, page, pageSize).result
+      labelKeyLabelValues, startTimestampOption, endTimestampOption, page, pageSize).result
     runTransaction(action)
   }
 
   override def countWorkflowSummaries(workflowStatuses: Set[String], workflowNames: Set[String],
-                                      workflowExecutionUuids: Set[String], startTimestampOption: Option[Timestamp],
+                                      workflowExecutionUuids: Set[String], labelKeyLabelValues: Set[(String, String)],
+                                      startTimestampOption: Option[Timestamp],
                                       endTimestampOption: Option[Timestamp])
                                      (implicit ec: ExecutionContext): Future[Int] = {
     val action = dataAccess.countWorkflowMetadataSummaryEntries(workflowStatuses, workflowNames, workflowExecutionUuids,
-      startTimestampOption, endTimestampOption).result
+      labelKeyLabelValues, startTimestampOption, endTimestampOption).result
     runTransaction(action)
   }
 }
diff --git a/database/sql/src/main/scala/cromwell/database/slick/SlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/SlickDatabase.scala
index 860ad65b3..f56b035c9 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/SlickDatabase.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/SlickDatabase.scala
@@ -1,18 +1,17 @@
 package cromwell.database.slick
 
 import java.sql.Connection
-import java.util.concurrent.{ExecutorService, Executors}
 
 import com.typesafe.config.Config
 import cromwell.database.slick.tables.DataAccessComponent
 import cromwell.database.sql.SqlDatabase
 import net.ceedubs.ficus.Ficus._
 import org.slf4j.LoggerFactory
-import slick.backend.DatabaseConfig
-import slick.driver.JdbcProfile
+import slick.basic.DatabaseConfig
+import slick.jdbc.{JdbcCapabilities, JdbcProfile}
 
 import scala.concurrent.duration._
-import scala.concurrent.{Await, ExecutionContext, Future}
+import scala.concurrent.{Await, Future}
 
 object SlickDatabase {
   /**
@@ -59,12 +58,13 @@ class SlickDatabase(override val originalDatabaseConfig: Config) extends SqlData
   with JobStoreSlickDatabase
   with CallCachingSlickDatabase
   with SummaryStatusSlickDatabase
-  with SubWorkflowStoreSlickDatabase {
+  with SubWorkflowStoreSlickDatabase
+  with DockerHashStoreSlickDatabase {
 
   override val urlKey = SlickDatabase.urlKey(originalDatabaseConfig)
   private val slickConfig = DatabaseConfig.forConfig[JdbcProfile]("", databaseConfig)
 
-  val dataAccess = new DataAccessComponent(slickConfig.driver)
+  val dataAccess = new DataAccessComponent(slickConfig.profile)
 
   // Allows creation of a Database, plus implicits for running transactions
   import dataAccess.driver.api._
@@ -74,36 +74,10 @@ class SlickDatabase(override val originalDatabaseConfig: Config) extends SqlData
 
   SlickDatabase.log.info(s"Running with database $urlKey = ${databaseConfig.getString(urlKey)}")
 
-  /**
-    * Create a special execution context, a fixed thread pool, to run each of our composite database actions. Running
-    * each composite action as a runnable within the pool will ensure that-- at most-- the same number of actions are
-    * running as there are available connections. Thus there should never be a connection deadlock, as outlined in
-    * - https://github.com/slick/slick/issues/1274
-    * - https://groups.google.com/d/msg/scalaquery/5MCUnwaJ7U0/NLLMotX9BQAJ
-    *
-    * Custom future thread pool based on:
-    * - http://stackoverflow.com/questions/15285284/how-to-configure-a-fine-tuned-thread-pool-for-futures#comment23278672_15285441
-    *
-    * Database config parameter defaults based on: (expand the `forConfig` scaladoc for a full list of values)
-    * - http://slick.typesafe.com/doc/3.1.0/api/index.html#slick.jdbc.JdbcBackend$DatabaseFactoryDef@forConfig(path:String,config:com.typesafe.config.Config,driver:java.sql.Driver,classLoader:ClassLoader):JdbcBackend.this.Database
-    *
-    * Reuses the error reporter from the database's executionContext.
-    */
-  private val actionThreadPool: ExecutorService = {
-    val dbNumThreads = databaseConfig.as[Option[Int]]("db.numThreads").getOrElse(20)
-    val dbMaximumPoolSize = databaseConfig.as[Option[Int]]("db.maxConnections").getOrElse(dbNumThreads * 5)
-    val actionThreadPoolSize = databaseConfig.as[Option[Int]]("actionThreadPoolSize").getOrElse(dbNumThreads) min dbMaximumPoolSize
-    Executors.newFixedThreadPool(actionThreadPoolSize)
-  }
-
-  private val actionExecutionContext: ExecutionContext = ExecutionContext.fromExecutor(
-    actionThreadPool, database.executor.executionContext.reportFailure
-  )
-
   protected[this] lazy val insertBatchSize = databaseConfig.as[Option[Int]]("insert-batch-size").getOrElse(2000)
 
   protected[this] lazy val useSlickUpserts =
-    dataAccess.driver.capabilities.contains(JdbcProfile.capabilities.insertOrUpdate)
+    dataAccess.driver.capabilities.contains(JdbcCapabilities.insertOrUpdate)
 
   protected[this] def assertUpdateCount(description: String, updates: Int, expected: Int): DBIO[Unit] = {
     if (updates == expected) {
@@ -123,12 +97,10 @@ class SlickDatabase(override val originalDatabaseConfig: Config) extends SqlData
   }
 
   override def close(): Unit = {
-    actionThreadPool.shutdown()
     database.close()
   }
 
   protected[this] def runTransaction[R](action: DBIO[R]): Future[R] = {
-    //database.run(action.transactionally) <-- https://github.com/slick/slick/issues/1274
-    Future(Await.result(database.run(action.transactionally), Duration.Inf))(actionExecutionContext)
+    database.run(action.transactionally)
   }
 }
diff --git a/database/sql/src/main/scala/cromwell/database/slick/SummaryStatusSlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/SummaryStatusSlickDatabase.scala
index 6903a29e6..cc75511e2 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/SummaryStatusSlickDatabase.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/SummaryStatusSlickDatabase.scala
@@ -9,8 +9,7 @@ trait SummaryStatusSlickDatabase {
 
   import dataAccess.driver.api._
 
-  private[slick] def getSummaryStatusEntryMaximumId(summaryTableName: String, summarizedTableName: String)
-                                                   (implicit ec: ExecutionContext): DBIO[Option[Long]] = {
+  private[slick] def getSummaryStatusEntryMaximumId(summaryTableName: String, summarizedTableName: String): DBIO[Option[Long]] = {
     dataAccess.
       maximumIdForSummaryTableNameSummarizedTableName((summaryTableName, summarizedTableName)).
       result.headOption
diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingAggregationEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingAggregationEntryComponent.scala
new file mode 100644
index 000000000..2c7850399
--- /dev/null
+++ b/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingAggregationEntryComponent.scala
@@ -0,0 +1,57 @@
+package cromwell.database.slick.tables
+
+import cromwell.database.sql.tables.CallCachingAggregationEntry
+
+
+trait CallCachingAggregationEntryComponent {
+
+  this: DriverComponent with CallCachingEntryComponent =>
+
+  import driver.api._
+
+  class CallCachingAggregationEntries(tag: Tag) extends Table[CallCachingAggregationEntry](tag, "CALL_CACHING_AGGREGATION_ENTRY") {
+    def callCachingAggregationEntryId = column[Int]("CALL_CACHING_AGGREGATION_ENTRY_ID", O.PrimaryKey, O.AutoInc)
+
+    def baseAggregation = column[String]("BASE_AGGREGATION")
+
+    def inputFilesAggregation = column[Option[String]]("INPUT_FILES_AGGREGATION")
+
+    def callCachingEntryId = column[Int]("CALL_CACHING_ENTRY_ID")
+
+    override def * = (baseAggregation, inputFilesAggregation, callCachingEntryId.?, callCachingAggregationEntryId.?) <>
+      (CallCachingAggregationEntry.tupled, CallCachingAggregationEntry.unapply)
+
+    def fkCallCachingAggregationEntryCallCachingEntryId = foreignKey("FK_CALL_CACHING_AGGREGATION_ENTRY_CALL_CACHING_ENTRY_ID",
+      callCachingEntryId, callCachingEntries)(_.callCachingEntryId)
+
+    def ixCallCachingAggregationEntryBaIfa =
+      index("IX_CALL_CACHING_AGGREGATION_ENTRY_BA_IFA", (baseAggregation, inputFilesAggregation), unique = false)
+  }
+
+  val callCachingAggregationEntries = TableQuery[CallCachingAggregationEntries]
+
+  val callCachingAggregationEntryIdsAutoInc = callCachingAggregationEntries returning
+    callCachingAggregationEntries.map(_.callCachingAggregationEntryId)
+
+  val existsCallCachingEntriesForBaseAggregationHash = Compiled(
+    (baseAggregation: Rep[String]) => (for {
+      callCachingEntry <- callCachingEntries
+      if callCachingEntry.allowResultReuse
+      callCachingAggregationEntry <- callCachingAggregationEntries
+      if callCachingEntry.callCachingEntryId === callCachingAggregationEntry.callCachingEntryId
+      if callCachingAggregationEntry.baseAggregation === baseAggregation
+    } yield ()).exists
+  )
+
+  def callCachingEntriesForAggregatedHashes(baseAggregation: Rep[String], inputFilesAggregation: Rep[Option[String]], number: Int) = {
+    (for {
+      callCachingEntry <- callCachingEntries
+      if callCachingEntry.allowResultReuse
+      callCachingAggregationEntry <- callCachingAggregationEntries
+      if callCachingEntry.callCachingEntryId === callCachingAggregationEntry.callCachingEntryId
+      if callCachingAggregationEntry.baseAggregation === baseAggregation
+      if (callCachingAggregationEntry.inputFilesAggregation.isEmpty && inputFilesAggregation.isEmpty) ||
+        (callCachingAggregationEntry.inputFilesAggregation === inputFilesAggregation)
+    } yield callCachingAggregationEntry.callCachingEntryId).drop(number - 1).take(1)
+  }
+}
diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingDetritusEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingDetritusEntryComponent.scala
index 040d61f64..79bccfbe2 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingDetritusEntryComponent.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingDetritusEntryComponent.scala
@@ -31,7 +31,7 @@ trait CallCachingDetritusEntryComponent {
       index("UC_CALL_CACHING_DETRITUS_ENTRY_CCEI_DK", (callCachingEntryId, detritusKey), unique = true)
   }
 
-  protected val callCachingDetritusEntries = TableQuery[CallCachingDetritusEntries]
+  val callCachingDetritusEntries = TableQuery[CallCachingDetritusEntries]
 
   val callCachingDetritusEntryIdsAutoInc =
     callCachingDetritusEntries returning callCachingDetritusEntries.map(_.callCachingDetritusEntryId)
diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingEntryComponent.scala
index d692e57c1..b1449fc84 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingEntryComponent.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingEntryComponent.scala
@@ -48,4 +48,13 @@ trait CallCachingEntryComponent {
       if callCachingEntry.callCachingEntryId === callCachingEntryId
     } yield callCachingEntry.allowResultReuse
   )
+
+  val callCachingEntriesForWorkflowFqnIndex = Compiled(
+    (workflowId: Rep[String], callFqn: Rep[String],  jobIndex: Rep[Int]) => for {
+        callCachingEntry <- callCachingEntries
+        if callCachingEntry.workflowExecutionUuid === workflowId
+        if callCachingEntry.callFullyQualifiedName === callFqn
+        if callCachingEntry.jobIndex === jobIndex
+      } yield callCachingEntry
+  )
 }
diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingHashEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingHashEntryComponent.scala
index 71140ce5f..53b00ed2d 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingHashEntryComponent.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingHashEntryComponent.scala
@@ -28,7 +28,7 @@ trait CallCachingHashEntryComponent {
       index("UC_CALL_CACHING_HASH_ENTRY_CCEI_HK", (callCachingEntryId, hashKey), unique = true)
   }
 
-  protected val callCachingHashEntries = TableQuery[CallCachingHashEntries]
+  val callCachingHashEntries = TableQuery[CallCachingHashEntries]
 
   val callCachingHashEntryIdsAutoInc = callCachingHashEntries returning
     callCachingHashEntries.map(_.callCachingHashEntryId)
@@ -65,16 +65,13 @@ trait CallCachingHashEntryComponent {
   }
 
   /**
-    * Returns the callCachingEntryIds where all the hash keys and hash values exist, and are allowed to be reused.
-    *
-    * @param hashKeyHashValues The hash keys and hash values as tuples.
-    * @return The callCachingEntryIds with all of the hash keys and hash values.
+    * Returns whether or not there exists at least one callCachingEntryId for which all the hash keys and hash values match, and are allowed to be reused.
     */
-  def callCachingEntryIdsForHashKeyHashValues(hashKeyHashValues: NonEmptyList[(String, String)]) = {
-    for {
+  def existsMatchingCachingEntryIdsForHashKeyHashValues(hashKeyHashValues: NonEmptyList[(String, String)]) = {
+    (for {
       callCachingEntry <- callCachingEntries
       if existsAllCallCachingEntryIdHashKeyHashValues(
         callCachingEntry.callCachingEntryId, hashKeyHashValues) && callCachingEntry.allowResultReuse
-    } yield callCachingEntry.callCachingEntryId
+    } yield callCachingEntry.callCachingEntryId).exists
   }
 }
diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingSimpletonEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingSimpletonEntryComponent.scala
index e2077e027..bd7e3a807 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingSimpletonEntryComponent.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingSimpletonEntryComponent.scala
@@ -32,7 +32,7 @@ trait CallCachingSimpletonEntryComponent {
       index("UC_CALL_CACHING_SIMPLETON_ENTRY_CCEI_SK", (callCachingEntryId, simpletonKey), unique = true)
   }
 
-  protected val callCachingSimpletonEntries = TableQuery[CallCachingSimpletonEntries]
+  val callCachingSimpletonEntries = TableQuery[CallCachingSimpletonEntries]
 
   val callCachingSimpletonEntryIdsAutoInc = callCachingSimpletonEntries returning
     callCachingSimpletonEntries.map(_.callCachingSimpletonEntryId)
diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/CustomLabelEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/CustomLabelEntryComponent.scala
new file mode 100644
index 000000000..9afff18ff
--- /dev/null
+++ b/database/sql/src/main/scala/cromwell/database/slick/tables/CustomLabelEntryComponent.scala
@@ -0,0 +1,55 @@
+package cromwell.database.slick.tables
+
+import cromwell.database.sql.tables.CustomLabelEntry
+import slick.model.ForeignKeyAction.Cascade
+
+trait CustomLabelEntryComponent {
+
+  this: DriverComponent with WorkflowMetadataSummaryEntryComponent =>
+
+  import driver.api._
+
+  class CustomLabelEntries(tag: Tag)
+    extends Table[CustomLabelEntry](tag, "CUSTOM_LABEL_ENTRY") {
+    def customLabelEntryId = column[Long]("CUSTOM_LABEL_ENTRY_ID", O.PrimaryKey, O.AutoInc)
+
+    def customLabelKey = column[String]("CUSTOM_LABEL_KEY", O.Length(63))
+
+    def customLabelValue = column[String]("CUSTOM_LABEL_VALUE", O.Length(63))
+
+    def workflowExecutionUuid = column[String]("WORKFLOW_EXECUTION_UUID", O.Length(100))
+
+    override def * = (customLabelKey, customLabelValue, workflowExecutionUuid,
+    customLabelEntryId.?) <> (CustomLabelEntry.tupled, CustomLabelEntry.unapply)
+
+    def fkCustomLabelEntryWorkflowExecutionUuid = foreignKey("FK_CUSTOM_LABEL_ENTRY_WORKFLOW_EXECUTION_UUID",
+      workflowExecutionUuid, workflowMetadataSummaryEntries)(_.workflowExecutionUuid, onDelete = Cascade)
+
+    def ucCustomLabelEntryClkClvWeu = index("UC_CUSTOM_LABEL_ENTRY_CLK_CLV_WEU",
+      (customLabelKey, customLabelValue, workflowExecutionUuid), unique = true)
+}
+
+  val customLabelEntries = TableQuery[CustomLabelEntries]
+
+  val customLabelEntryIdsAutoInc = customLabelEntries returning
+    customLabelEntries.map(_.customLabelEntryId)
+
+  val existsWorkflowIdLabelKeyAndValue = Compiled(
+    (workflowUuid: Rep[String], labelKey: Rep[String], labelValue: Rep[String]) => (for {
+      customLabelEntry <- customLabelEntries
+      if customLabelEntry.workflowExecutionUuid === workflowUuid &&
+        customLabelEntry.customLabelKey === labelKey &&
+        customLabelEntry.customLabelValue === labelValue
+    } yield ()).exists
+  )
+
+  def existsWorkflowIdLabelKeyAndValue(workflowId: Rep[String],
+                                       labelKey: Rep[String],
+                                       labelValue: Rep[String]): Rep[Boolean] = {
+    customLabelEntries.filter(customLabelEntry =>
+      customLabelEntry.workflowExecutionUuid === workflowId &&
+        customLabelEntry.customLabelKey === labelKey &&
+        customLabelEntry.customLabelValue === labelValue
+    ).exists
+  }
+}
diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/DataAccessComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/DataAccessComponent.scala
index b0c70abf3..9e0e77711 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/tables/DataAccessComponent.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/tables/DataAccessComponent.scala
@@ -1,12 +1,13 @@
 package cromwell.database.slick.tables
 
-import slick.driver.JdbcProfile
+import slick.jdbc.JdbcProfile
 
 class DataAccessComponent(val driver: JdbcProfile)
   extends DriverComponent
     with CallCachingDetritusEntryComponent
     with CallCachingEntryComponent
     with CallCachingHashEntryComponent
+    with CallCachingAggregationEntryComponent
     with CallCachingSimpletonEntryComponent
     with JobKeyValueEntryComponent
     with JobStoreEntryComponent
@@ -15,7 +16,9 @@ class DataAccessComponent(val driver: JdbcProfile)
     with SummaryStatusEntryComponent
     with WorkflowMetadataSummaryEntryComponent
     with WorkflowStoreEntryComponent
-    with SubWorkflowStoreEntryComponent {
+    with SubWorkflowStoreEntryComponent
+    with CustomLabelEntryComponent
+    with DockerHashStoreEntryComponent {
 
   import driver.api._
 
@@ -24,6 +27,8 @@ class DataAccessComponent(val driver: JdbcProfile)
       callCachingEntries.schema ++
       callCachingHashEntries.schema ++
       callCachingSimpletonEntries.schema ++
+      callCachingAggregationEntries.schema ++
+      dockerHashStoreEntries.schema ++
       jobKeyValueEntries.schema ++
       jobStoreEntries.schema ++
       jobStoreSimpletonEntries.schema ++
@@ -31,5 +36,6 @@ class DataAccessComponent(val driver: JdbcProfile)
       summaryStatusEntries.schema ++
       workflowMetadataSummaryEntries.schema ++
       workflowStoreEntries.schema ++
-      subWorkflowStoreEntries.schema
+      subWorkflowStoreEntries.schema ++
+      customLabelEntries.schema
 }
diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/DockerHashStoreEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/DockerHashStoreEntryComponent.scala
new file mode 100644
index 000000000..9be7b35cf
--- /dev/null
+++ b/database/sql/src/main/scala/cromwell/database/slick/tables/DockerHashStoreEntryComponent.scala
@@ -0,0 +1,38 @@
+package cromwell.database.slick.tables
+
+import cromwell.database.sql.tables.DockerHashStoreEntry
+
+trait DockerHashStoreEntryComponent {
+
+  this: DriverComponent =>
+
+  import driver.api._
+
+  class DockerHashStoreEntries(tag: Tag) extends Table[DockerHashStoreEntry](tag, "DOCKER_HASH_STORE_ENTRY") {
+    def dockerHashStoreEntryId = column[Int]("DOCKER_HASH_STORE_ENTRY_ID", O.PrimaryKey, O.AutoInc)
+
+    def workflowExecutionUuid = column[String]("WORKFLOW_EXECUTION_UUID")
+
+    def dockerTag = column[String]("DOCKER_TAG")
+
+    def dockerHash = column[String]("DOCKER_HASH")
+
+    override def * = (workflowExecutionUuid, dockerTag, dockerHash, dockerHashStoreEntryId.?) <> (DockerHashStoreEntry.tupled, DockerHashStoreEntry.unapply)
+
+    def ucDockerHashStoreEntryWeuDt = index("UC_DOCKER_HASH_STORE_ENTRY_WEU_DT", (workflowExecutionUuid, dockerTag), unique = true)
+  }
+
+  val dockerHashStoreEntries = TableQuery[DockerHashStoreEntries]
+
+  val dockerHashStoreEntryIdsAutoInc = dockerHashStoreEntries returning dockerHashStoreEntries.map(_.dockerHashStoreEntryId)
+
+  /**
+    * Useful for finding the docker hash store for a given workflow execution UUID
+    */
+  val dockerHashStoreEntriesForWorkflowExecutionUuid = Compiled(
+    (workflowExecutionUuid: Rep[String]) => for {
+      dockerHashStoreEntry <- dockerHashStoreEntries
+      if dockerHashStoreEntry.workflowExecutionUuid === workflowExecutionUuid
+    } yield dockerHashStoreEntry
+  )
+}
diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/DriverComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/DriverComponent.scala
index f3fba44d4..d5f786018 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/tables/DriverComponent.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/tables/DriverComponent.scala
@@ -1,6 +1,6 @@
 package cromwell.database.slick.tables
 
-import slick.driver.JdbcProfile
+import slick.jdbc.JdbcProfile
 
 trait DriverComponent {
   val driver: JdbcProfile
diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/JobStoreSimpletonEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/JobStoreSimpletonEntryComponent.scala
index 4364170ac..16df2fb3c 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/tables/JobStoreSimpletonEntryComponent.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/tables/JobStoreSimpletonEntryComponent.scala
@@ -32,7 +32,7 @@ trait JobStoreSimpletonEntryComponent {
       index("UC_JOB_STORE_SIMPLETON_ENTRY_JSEI_SK", (jobStoreEntryId, simpletonKey), unique = true)
   }
 
-  protected val jobStoreSimpletonEntries = TableQuery[JobStoreSimpletonEntries]
+  val jobStoreSimpletonEntries = TableQuery[JobStoreSimpletonEntries]
 
   val jobStoreSimpletonEntryIdsAutoInc = jobStoreSimpletonEntries returning
     jobStoreSimpletonEntries.map(_.jobStoreSimpletonEntryId)
diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/MetadataEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/MetadataEntryComponent.scala
index 745a1e593..8209f04f2 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/tables/MetadataEntryComponent.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/tables/MetadataEntryComponent.scala
@@ -62,10 +62,10 @@ trait MetadataEntryComponent {
   val metadataEntryIdsAutoInc = metadataEntries returning metadataEntries.map(_.metadataEntryId)
 
   val metadataEntriesForWorkflowExecutionUuid = Compiled(
-    (workflowExecutionUuid: Rep[String]) => for {
+    (workflowExecutionUuid: Rep[String]) => (for {
       metadataEntry <- metadataEntries
       if metadataEntry.workflowExecutionUuid === workflowExecutionUuid
-    } yield metadataEntry
+    } yield metadataEntry).sortBy(_.metadataTimestamp)
   )
 
   val metadataEntryExistsForWorkflowExecutionUuid = Compiled(
@@ -76,71 +76,119 @@ trait MetadataEntryComponent {
   )
 
   val metadataEntriesForWorkflowExecutionUuidAndMetadataKey = Compiled(
-    (workflowExecutionUuid: Rep[String], metadataKey: Rep[String]) => for {
+    (workflowExecutionUuid: Rep[String], metadataKey: Rep[String]) => (for {
       metadataEntry <- metadataEntries
       if metadataEntry.workflowExecutionUuid === workflowExecutionUuid
       if metadataEntry.metadataKey === metadataKey
       if metadataEntry.callFullyQualifiedName.isEmpty
       if metadataEntry.jobIndex.isEmpty
       if metadataEntry.jobAttempt.isEmpty
-    } yield metadataEntry
+    } yield metadataEntry).sortBy(_.metadataTimestamp)
   )
 
   val metadataEntriesForJobKey = Compiled(
     (workflowExecutionUuid: Rep[String], callFullyQualifiedName: Rep[String], jobIndex: Rep[Option[Int]],
-     jobAttempt: Rep[Int]) => for {
+     jobAttempt: Rep[Option[Int]]) => (for {
       metadataEntry <- metadataEntries
       if metadataEntry.workflowExecutionUuid === workflowExecutionUuid
       if metadataEntry.callFullyQualifiedName === callFullyQualifiedName
-      if (metadataEntry.jobIndex === jobIndex) ||
-        (metadataEntry.jobIndex.isEmpty && jobIndex.isEmpty)
-      if metadataEntry.jobAttempt === jobAttempt
-    } yield metadataEntry
+      if hasSameIndex(metadataEntry, jobIndex)
+      if hasSameAttempt(metadataEntry, jobAttempt)
+    } yield metadataEntry).sortBy(_.metadataTimestamp)
   )
 
   val metadataEntriesForJobKeyAndMetadataKey = Compiled(
     (workflowExecutionUuid: Rep[String], metadataKey: Rep[String], callFullyQualifiedName: Rep[String],
-     jobIndex: Rep[Option[Int]], jobAttempt: Rep[Int]) => for {
+     jobIndex: Rep[Option[Int]], jobAttempt: Rep[Option[Int]]) => (for {
       metadataEntry <- metadataEntries
       if metadataEntry.workflowExecutionUuid === workflowExecutionUuid
       if metadataEntry.metadataKey === metadataKey
       if metadataEntry.callFullyQualifiedName === callFullyQualifiedName
-      if (metadataEntry.jobIndex === jobIndex) ||
-        (metadataEntry.jobIndex.isEmpty && jobIndex.isEmpty)
-      if metadataEntry.jobAttempt === jobAttempt
-    } yield metadataEntry
+      if hasSameIndex(metadataEntry, jobIndex)
+      if hasSameAttempt(metadataEntry, jobAttempt)
+    } yield metadataEntry).sortBy(_.metadataTimestamp)
   )
 
   val metadataEntriesForIdGreaterThanOrEqual = Compiled(
-    (metadataEntryId: Rep[Long], metadataKey1: Rep[String], metadataKey2: Rep[String], metadataKey3: Rep[String],
-     metadataKey4: Rep[String]) => for {
+    (metadataEntryId: Rep[Long], startMetadataKey: Rep[String], endMetadataKey: Rep[String], nameMetadataKey: Rep[String],
+     statusMetadataKey: Rep[String], likeLabelMetadataKey: Rep[String]) => (for {
       metadataEntry <- metadataEntries
       if metadataEntry.metadataEntryId >= metadataEntryId
-      if (metadataEntry.metadataKey === metadataKey1 || metadataEntry.metadataKey === metadataKey2 ||
-        metadataEntry.metadataKey === metadataKey3 || metadataEntry.metadataKey === metadataKey4) &&
+      if (metadataEntry.metadataKey === startMetadataKey || metadataEntry.metadataKey === endMetadataKey ||
+        metadataEntry.metadataKey === nameMetadataKey || metadataEntry.metadataKey === statusMetadataKey ||
+        metadataEntry.metadataKey.like(likeLabelMetadataKey)) &&
         (metadataEntry.callFullyQualifiedName.isEmpty && metadataEntry.jobIndex.isEmpty &&
           metadataEntry.jobAttempt.isEmpty)
-    } yield metadataEntry
+    } yield metadataEntry).sortBy(_.metadataTimestamp)
   )
 
+  /**
+    * Returns metadata entries that are "like" metadataKeys for the specified workflow.
+    * If requireEmptyJobKey is true, only workflow level keys are returned, otherwise both workflow and call level
+    * keys are returned.
+    */
   def metadataEntriesLikeMetadataKeys(workflowExecutionUuid: String, metadataKeys: NonEmptyList[String],
                                       requireEmptyJobKey: Boolean) = {
-    for {
+    (for {
       metadataEntry <- metadataEntries
       if metadataEntry.workflowExecutionUuid === workflowExecutionUuid
       if metadataEntryHasMetadataKeysLike(metadataEntry, metadataKeys)
       if metadataEntryHasEmptyJobKey(metadataEntry, requireEmptyJobKey)
-    } yield metadataEntry
+    } yield metadataEntry).sortBy(_.metadataTimestamp)
+  }
+
+  /**
+    * Returns metadata entries that are "like" metadataKeys for the specified call.
+    * If jobAttempt has no value, all metadata keys for all attempts are returned.
+    */
+  def metadataEntriesLikeMetadataKeysWithJob(workflowExecutionUuid: String, metadataKeys: NonEmptyList[String],
+                                             callFqn: String, jobIndex: Option[Int], jobAttempt: Option[Int]) = {
+    (for {
+      metadataEntry <- metadataEntries
+      if metadataEntry.workflowExecutionUuid === workflowExecutionUuid
+      if metadataEntryHasMetadataKeysLike(metadataEntry, metadataKeys)
+      if metadataEntry.callFullyQualifiedName === callFqn
+      if hasSameIndex(metadataEntry, jobIndex)
+      // Assume that every metadata entry for a call should have a non null attempt value
+      // Because of that, if the jobAttempt paramater is Some(_), make sure it matches, otherwise take all entries
+      // regardless of the attempt
+      if (metadataEntry.jobAttempt === jobAttempt) || jobAttempt.isEmpty
+    } yield metadataEntry).sortBy(_.metadataTimestamp)
   }
 
+  /**
+    * Returns metadata entries that are NOT "like" metadataKeys for the specified workflow.
+    * If requireEmptyJobKey is true, only workflow level keys are returned, otherwise both workflow and call level
+    * keys are returned.
+    */
   def metadataEntriesNotLikeMetadataKeys(workflowExecutionUuid: String, metadataKeys: NonEmptyList[String],
                                          requireEmptyJobKey: Boolean) = {
-    for {
+    (for {
       metadataEntry <- metadataEntries
       if metadataEntry.workflowExecutionUuid === workflowExecutionUuid
       if !metadataEntryHasMetadataKeysLike(metadataEntry, metadataKeys)
       if metadataEntryHasEmptyJobKey(metadataEntry, requireEmptyJobKey)
-    } yield metadataEntry
+    } yield metadataEntry).sortBy(_.metadataTimestamp)
+  }
+
+  /**
+    * Returns metadata entries that are NOT "like" metadataKeys for the specified call.
+    * If jobIndex (resp. jobAttempt) has no value, all metadata keys for all indices (resp. attempt)
+    * are returned.
+    */
+  def metadataEntriesNotLikeMetadataKeysWithJob(workflowExecutionUuid: String, metadataKeys: NonEmptyList[String],
+                                                callFqn: String, jobIndex: Option[Int], jobAttempt: Option[Int]) = {
+    (for {
+      metadataEntry <- metadataEntries
+      if metadataEntry.workflowExecutionUuid === workflowExecutionUuid
+      if !metadataEntryHasMetadataKeysLike(metadataEntry, metadataKeys)
+      if metadataEntry.callFullyQualifiedName === callFqn
+      if hasSameIndex(metadataEntry, jobIndex)
+        // Assume that every metadata entry for a call should have a non null attempt value
+        // Because of that, if the jobAttempt parameter is Some(_), make sure it matches, otherwise take all entries
+        // regardless of the attempt
+      if (metadataEntry.jobAttempt === jobAttempt) || jobAttempt.isEmpty
+    } yield metadataEntry).sortBy(_.metadataTimestamp)
   }
 
   private[this] def metadataEntryHasMetadataKeysLike(metadataEntry: MetadataEntries,
@@ -148,6 +196,14 @@ trait MetadataEntryComponent {
     metadataKeys.toList.map(metadataEntry.metadataKey like _).reduce(_ || _)
   }
 
+  private[this] def hasSameIndex(metadataEntry: MetadataEntries, jobIndex: Rep[Option[Int]]) = {
+    (metadataEntry.jobIndex.isEmpty && jobIndex.isEmpty) || (metadataEntry.jobIndex === jobIndex)
+  }
+
+  private[this] def hasSameAttempt(metadataEntry: MetadataEntries, jobAttempt: Rep[Option[Int]]) = {
+    (metadataEntry.jobAttempt.isEmpty && jobAttempt.isEmpty) || (metadataEntry.jobAttempt === jobAttempt)
+  }
+
   private[this] def metadataEntryHasEmptyJobKey(metadataEntry: MetadataEntries,
                                                 requireEmptyJobKey: Rep[Boolean]): Rep[Boolean] = {
     !requireEmptyJobKey ||
diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowMetadataSummaryEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowMetadataSummaryEntryComponent.scala
index 2142c7188..b37140d98 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowMetadataSummaryEntryComponent.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowMetadataSummaryEntryComponent.scala
@@ -6,7 +6,7 @@ import cromwell.database.sql.tables.WorkflowMetadataSummaryEntry
 
 trait WorkflowMetadataSummaryEntryComponent {
 
-  this: DriverComponent =>
+  this: DriverComponent with CustomLabelEntryComponent =>
 
   import driver.api._
 
@@ -55,7 +55,9 @@ trait WorkflowMetadataSummaryEntryComponent {
   )
 
   def filterWorkflowMetadataSummaryEntries(workflowStatuses: Set[String], workflowNames: Set[String],
-                                           workflowExecutionUuids: Set[String], startTimestampOption: Option[Timestamp],
+                                           workflowExecutionUuids: Set[String],
+                                           labelKeyLabelValues: Set[(String,String)],
+                                           startTimestampOption: Option[Timestamp],
                                            endTimestampOption: Option[Timestamp]):
   (WorkflowMetadataSummaryEntries) => Rep[Boolean] = {
     val include: Rep[Boolean] = true
@@ -82,10 +84,14 @@ trait WorkflowMetadataSummaryEntryComponent {
       val workflowStatusFilter = workflowStatuses.
         map(workflowStatus => workflowMetadataSummaryEntry.workflowStatus.fold(ifEmpty = exclude)(_ === workflowStatus)).
         reduceLeftOption(_ || _)
-
+      // Workflows can be queried by multiple labels, the reduceLeftOption ANDs together all workflows are associated
+      // to all the labels from the query.
+      val labelsFilter = labelKeyLabelValues.map { case (labelKey, labelValue) =>
+          existsWorkflowIdLabelKeyAndValue(workflowMetadataSummaryEntry.workflowExecutionUuid, labelKey, labelValue) }.
+        reduceLeftOption(_ && _)
       // Put all the optional filters above together in one place.
       val optionalFilters: List[Option[Rep[Boolean]]] = List(
-        workflowNameFilter, workflowExecutionUuidFilter, workflowStatusFilter, startTimestampFilter, endTimestampFilter)
+        workflowNameFilter, workflowExecutionUuidFilter, workflowStatusFilter, labelsFilter, startTimestampFilter, endTimestampFilter)
       // Unwrap the optional filters.  If any of these filters are not defined, replace with `include` to include all
       // rows which might otherwise have been filtered.
       val filters = optionalFilters.map(_.getOrElse(include))
@@ -95,10 +101,11 @@ trait WorkflowMetadataSummaryEntryComponent {
   }
 
   def countWorkflowMetadataSummaryEntries(workflowStatuses: Set[String], workflowNames: Set[String],
-                                          workflowExecutionUuids: Set[String], startTimestampOption: Option[Timestamp],
+                                          workflowExecutionUuids: Set[String], labelKeyLabelValues: Set[(String,String)],
+                                          startTimestampOption: Option[Timestamp],
                                           endTimestampOption: Option[Timestamp]) = {
     val filter = filterWorkflowMetadataSummaryEntries(
-      workflowStatuses, workflowNames, workflowExecutionUuids, startTimestampOption, endTimestampOption)
+      workflowStatuses, workflowNames, workflowExecutionUuids, labelKeyLabelValues, startTimestampOption, endTimestampOption)
     workflowMetadataSummaryEntries.filter(filter).length
   }
 
@@ -106,11 +113,12 @@ trait WorkflowMetadataSummaryEntryComponent {
     * Query workflow execution using the filter criteria encapsulated by the `WorkflowExecutionQueryParameters`.
     */
   def queryWorkflowMetadataSummaryEntries(workflowStatuses: Set[String], workflowNames: Set[String],
-                                          workflowExecutionUuids: Set[String], startTimestampOption: Option[Timestamp],
+                                          workflowExecutionUuids: Set[String], labelKeyLabelValues: Set[(String,String)],
+                                          startTimestampOption: Option[Timestamp],
                                           endTimestampOption: Option[Timestamp], page: Option[Int],
                                           pageSize: Option[Int]) = {
     val filter = filterWorkflowMetadataSummaryEntries(
-      workflowStatuses, workflowNames, workflowExecutionUuids, startTimestampOption, endTimestampOption)
+      workflowStatuses, workflowNames, workflowExecutionUuids, labelKeyLabelValues, startTimestampOption, endTimestampOption)
     val query = workflowMetadataSummaryEntries.filter(filter)
     (page, pageSize) match {
       case (Some(p), Some(ps)) => query.drop((p - 1) * ps).take(ps)
diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowStoreEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowStoreEntryComponent.scala
index 733fc7848..0c2b08106 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowStoreEntryComponent.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowStoreEntryComponent.scala
@@ -15,13 +15,17 @@ trait WorkflowStoreEntryComponent {
 
     def workflowExecutionUuid = column[String]("WORKFLOW_EXECUTION_UUID")
 
+    def workflowType = column[Option[String]]("WORKFLOW_TYPE", O.Length(30))
+
+    def workflowTypeVersion = column[Option[String]]("WORKFLOW_TYPE_VERSION")
+
     def workflowDefinition = column[Option[Clob]]("WORKFLOW_DEFINITION")
 
     def workflowInputs = column[Option[Clob]]("WORKFLOW_INPUTS")
 
     def workflowOptions = column[Option[Clob]]("WORKFLOW_OPTIONS")
 
-    def customLabels = column[Option[Clob]]("CUSTOM_LABELS")
+    def customLabels = column[Clob]("CUSTOM_LABELS")
 
     def workflowState = column[String]("WORKFLOW_STATE", O.Length(15))
 
@@ -29,7 +33,7 @@ trait WorkflowStoreEntryComponent {
 
     def importsZip = column[Option[Blob]]("IMPORTS_ZIP")
 
-    override def * = (workflowExecutionUuid, workflowDefinition, workflowInputs, workflowOptions, workflowState,
+    override def * = (workflowExecutionUuid, workflowDefinition, workflowType, workflowTypeVersion, workflowInputs, workflowOptions, workflowState,
       submissionTime, importsZip, customLabels, workflowStoreEntryId.?) <> (WorkflowStoreEntry.tupled, WorkflowStoreEntry.unapply)
 
     def ucWorkflowStoreEntryWeu = index("UC_WORKFLOW_STORE_ENTRY_WEU", workflowExecutionUuid, unique = true)
diff --git a/database/sql/src/main/scala/cromwell/database/sql/CallCachingSqlDatabase.scala b/database/sql/src/main/scala/cromwell/database/sql/CallCachingSqlDatabase.scala
index ee38700ac..4354ba634 100644
--- a/database/sql/src/main/scala/cromwell/database/sql/CallCachingSqlDatabase.scala
+++ b/database/sql/src/main/scala/cromwell/database/sql/CallCachingSqlDatabase.scala
@@ -7,13 +7,22 @@ import cromwell.database.sql.tables.CallCachingEntry
 import scala.concurrent.{ExecutionContext, Future}
 
 trait CallCachingSqlDatabase {
-  def addCallCaching(callCachingJoin: CallCachingJoin)(implicit ec: ExecutionContext): Future[Unit]
+  def addCallCaching(joins: Seq[CallCachingJoin], batchSize: Int)(implicit ec: ExecutionContext): Future[Unit]
 
-  def queryCallCachingEntryIds(hashKeyHashValues: NonEmptyList[(String, String)])
-                              (implicit ec: ExecutionContext): Future[Seq[Int]]
+  def hasMatchingCallCachingEntriesForBaseAggregation(baseAggregationHash: String)
+                                                     (implicit ec: ExecutionContext): Future[Boolean]
 
-  def queryCallCaching(callCachingEntryId: Int)
-                      (implicit ec: ExecutionContext): Future[Option[CallCachingJoin]]
+  def hasMatchingCallCachingEntriesForHashKeyValues(hashKeyHashValues: NonEmptyList[(String, String)])
+                                                   (implicit ec: ExecutionContext): Future[Boolean]
+
+  def findCacheHitForAggregation(baseAggregationHash: String, inputFilesAggregationHash: Option[String], hitNumber: Int)
+                                (implicit ec: ExecutionContext): Future[Option[Int]]
+
+  def queryResultsForCacheId(callCachingEntryId: Int)
+                            (implicit ec: ExecutionContext): Future[Option[CallCachingJoin]]
+  
+  def cacheEntryExistsForCall(workflowExecutionUuid: String, callFqn: String, index: Int)
+                             (implicit ec: ExecutionContext): Future[Boolean]
 
   def invalidateCall(callCachingEntryId: Int)
                     (implicit ec: ExecutionContext): Future[Option[CallCachingEntry]]
diff --git a/database/sql/src/main/scala/cromwell/database/sql/DockerHashStoreSqlDatabase.scala b/database/sql/src/main/scala/cromwell/database/sql/DockerHashStoreSqlDatabase.scala
new file mode 100644
index 000000000..8a719185c
--- /dev/null
+++ b/database/sql/src/main/scala/cromwell/database/sql/DockerHashStoreSqlDatabase.scala
@@ -0,0 +1,28 @@
+package cromwell.database.sql
+
+import cromwell.database.sql.tables.DockerHashStoreEntry
+
+import scala.concurrent.{ExecutionContext, Future}
+
+trait DockerHashStoreSqlDatabase {
+  this: SqlDatabase =>
+
+  /**
+    * Adds a docker hash entry to the store.
+    *
+    */
+  def addDockerHashStoreEntry(dockerHashStoreEntry: DockerHashStoreEntry)
+                             (implicit ec: ExecutionContext): Future[Unit]
+
+  /**
+    * Retrieves docker hash entries for a workflow.
+    *
+    */
+  def queryDockerHashStoreEntries(workflowExecutionUuid: String)
+                                 (implicit ec: ExecutionContext): Future[Seq[DockerHashStoreEntry]]
+
+  /**
+    * Deletes docker hash entries related to a workflow, returning the number of rows affected.
+    */
+  def removeDockerHashStoreEntries(workflowExecutionUuid: String)(implicit ec: ExecutionContext): Future[Int]
+}
diff --git a/database/sql/src/main/scala/cromwell/database/sql/JobStoreSqlDatabase.scala b/database/sql/src/main/scala/cromwell/database/sql/JobStoreSqlDatabase.scala
index fd01a8354..a5497ae49 100644
--- a/database/sql/src/main/scala/cromwell/database/sql/JobStoreSqlDatabase.scala
+++ b/database/sql/src/main/scala/cromwell/database/sql/JobStoreSqlDatabase.scala
@@ -7,7 +7,7 @@ import scala.concurrent.{ExecutionContext, Future}
 trait JobStoreSqlDatabase {
   this: SqlDatabase =>
 
-  def addJobStores(jobStoreJoins: Seq[JobStoreJoin])(implicit ec: ExecutionContext): Future[Unit]
+  def addJobStores(jobStoreJoins: Seq[JobStoreJoin], batchSize: Int)(implicit ec: ExecutionContext): Future[Unit]
 
   def queryJobStores(workflowExecutionUuid: String, callFqn: String, jobScatterIndex: Int, jobScatterAttempt: Int)
                     (implicit ec: ExecutionContext): Future[Option[JobStoreJoin]]
diff --git a/database/sql/src/main/scala/cromwell/database/sql/MetadataSqlDatabase.scala b/database/sql/src/main/scala/cromwell/database/sql/MetadataSqlDatabase.scala
index 3f8b83a14..e0eabe668 100644
--- a/database/sql/src/main/scala/cromwell/database/sql/MetadataSqlDatabase.scala
+++ b/database/sql/src/main/scala/cromwell/database/sql/MetadataSqlDatabase.scala
@@ -3,6 +3,7 @@ package cromwell.database.sql
 import java.sql.Timestamp
 
 import cats.data.NonEmptyList
+import cromwell.database.sql.joins.MetadataJobQueryValue
 import cromwell.database.sql.tables.{MetadataEntry, WorkflowMetadataSummaryEntry}
 
 import scala.concurrent.{ExecutionContext, Future}
@@ -37,24 +38,24 @@ trait MetadataSqlDatabase {
   def queryMetadataEntries(workflowExecutionUuid: String,
                            callFullyQualifiedName: String,
                            jobIndex: Option[Int],
-                           jobAttempt: Int)
+                           jobAttempt: Option[Int])
                           (implicit ec: ExecutionContext): Future[Seq[MetadataEntry]]
 
   def queryMetadataEntries(workflowUuid: String,
                            metadataKey: String,
                            callFullyQualifiedName: String,
                            jobIndex: Option[Int],
-                           jobAttempt: Int)
+                           jobAttempt: Option[Int])
                           (implicit ec: ExecutionContext): Future[Seq[MetadataEntry]]
 
   def queryMetadataEntriesLikeMetadataKeys(workflowExecutionUuid: String,
                                            metadataKeys: NonEmptyList[String],
-                                           requireEmptyJobKey: Boolean)
+                                           metadataJobQueryValue: MetadataJobQueryValue)
                                           (implicit ec: ExecutionContext): Future[Seq[MetadataEntry]]
 
   def queryMetadataEntryNotLikeMetadataKeys(workflowExecutionUuid: String,
                                             metadataKeys: NonEmptyList[String],
-                                            requireEmptyJobKey: Boolean)
+                                            metadataJobQueryValue: MetadataJobQueryValue)
                                            (implicit ec: ExecutionContext): Future[Seq[MetadataEntry]]
 
   /**
@@ -63,7 +64,8 @@ trait MetadataSqlDatabase {
     * @param buildUpdatedSummary Takes in the optional existing summary and the metadata, returns the new summary.
     * @return A `Future` with the maximum metadataEntryId summarized by the invocation of this method.
     */
-  def refreshMetadataSummaryEntries(metadataKey1: String, metadataKey2: String, metadataKey3: String, metadataKey4: String,
+  def refreshMetadataSummaryEntries(startMetadataKey: String, endMetadataKey: String, nameMetadataKey: String,
+                                    statusMetadataKey: String, labelMetadataKey: String,
                                     buildUpdatedSummary:
                                     (Option[WorkflowMetadataSummaryEntry], Seq[MetadataEntry])
                                       => WorkflowMetadataSummaryEntry)
@@ -72,12 +74,14 @@ trait MetadataSqlDatabase {
   def getWorkflowStatus(workflowExecutionUuid: String)(implicit ec: ExecutionContext): Future[Option[String]]
 
   def queryWorkflowSummaries(workflowStatuses: Set[String], workflowNames: Set[String],
-                             workflowExecutionUuids: Set[String], startTimestampOption: Option[Timestamp],
-                             endTimestampOption: Option[Timestamp], page: Option[Int], pageSize: Option[Int])
-                            (implicit ec: ExecutionContext): Future[Traversable[WorkflowMetadataSummaryEntry]]
+                             workflowExecutionUuids: Set[String], labelKeyLabelValues: Set[(String,String)],
+                             startTimestampOption: Option[Timestamp], endTimestampOption: Option[Timestamp],
+                             page: Option[Int], pageSize: Option[Int])
+                             (implicit ec: ExecutionContext): Future[Traversable[WorkflowMetadataSummaryEntry]]
 
   def countWorkflowSummaries(workflowStatuses: Set[String], workflowNames: Set[String],
-                             workflowExecutionUuids: Set[String], startTimestampOption: Option[Timestamp],
+                             workflowExecutionUuids: Set[String], labelKeyLabelValues: Set[(String, String)],
+                             startTimestampOption: Option[Timestamp],
                              endTimestampOption: Option[Timestamp])
-                            (implicit ec: ExecutionContext): Future[Int]
+                             (implicit ec: ExecutionContext): Future[Int]
 }
diff --git a/database/sql/src/main/scala/cromwell/database/sql/SqlConverters.scala b/database/sql/src/main/scala/cromwell/database/sql/SqlConverters.scala
index 3c6abd3ed..1799040cd 100644
--- a/database/sql/src/main/scala/cromwell/database/sql/SqlConverters.scala
+++ b/database/sql/src/main/scala/cromwell/database/sql/SqlConverters.scala
@@ -19,8 +19,7 @@ object SqlConverters {
   }
 
   implicit class ClobOptionToRawString(val clobOption: Option[Clob]) extends AnyVal {
-    // yes, it starts at 1
-    def toRawStringOption: Option[String] = clobOption.map(clob => clob.getSubString(1, clob.length.toInt))
+    def toRawStringOption: Option[String] = clobOption.map(_.toRawString)
 
     def toRawString: String = toRawStringOption.getOrElse("")
 
@@ -29,26 +28,58 @@ object SqlConverters {
     }
   }
 
+  implicit class ClobToRawString(val clob: Clob) extends AnyVal {
+    // yes, it starts at 1
+    def toRawString: String = clob.getSubString(1, clob.length.toInt)
+  }
+
   implicit class StringOptionToClobOption(val strOption: Option[String]) extends AnyVal {
-    def toClob: Option[Clob] = strOption.flatMap(_.toClob)
+    def toClobOption: Option[Clob] = strOption.flatMap(_.toClobOption)
   }
 
   implicit class StringToClobOption(val str: String) extends AnyVal {
-    def toClob: Option[Clob] = if (str.isEmpty) None else Option(new SerialClob(str.toCharArray))
+    /*
+    Many, many Clob implementations have problems with empty char arrays.
+    http://hg.openjdk.java.net/jdk8/jdk8/jdk/file/687fd7c7986d/src/share/classes/javax/sql/rowset/serial/SerialClob.java#l268
+    http://hg.openjdk.java.net/jdk9/client/jdk/file/1158c3e5bd9c/src/java.sql.rowset/share/classes/javax/sql/rowset/serial/SerialClob.java#l270
+    https://github.com/apache/derby/blob/10.13/java/engine/org/apache/derby/iapi/types/HarmonySerialClob.java#L109
+    https://github.com/arteam/hsqldb/blob/2.3.4/src/org/hsqldb/jdbc/JDBCClob.java#L196
+     */
+
+    import eu.timepit.refined.api.Refined
+    import eu.timepit.refined.collection.NonEmpty
+
+    def toClobOption: Option[Clob] = if (str.isEmpty) None else Option(new SerialClob(str.toCharArray))
+
+    def toClob(default: String Refined NonEmpty): Clob = {
+      val nonEmpty = if (str.isEmpty) default.value else str
+      new SerialClob(nonEmpty.toCharArray)
+    }
   }
 
-  implicit class BlobToBytes(val blobOption: Option[Blob]) extends AnyVal {
+  implicit class BlobToBytes(val blob: Blob) extends AnyVal {
     // yes, it starts at 1
-    def toBytesOption: Option[Array[Byte]] = blobOption.map(blob => blob.getBytes(1, blob.length.toInt))
+    def toBytes: Array[Byte] = blob.getBytes(1, blob.length.toInt)
+  }
+
+  implicit class BlobOptionToBytes(val blobOption: Option[Blob]) extends AnyVal {
+    def toBytesOption: Option[Array[Byte]] = blobOption.map(_.toBytes)
 
     def toBytes: Array[Byte] = toBytesOption.getOrElse(Array.empty)
   }
 
   implicit class BytesOptionToBlob(val bytesOption: Option[Array[Byte]]) extends AnyVal {
-    def toBlob: Option[Blob] = bytesOption.flatMap(_.toBlob)
+    /*
+    Many, many Blob implementations (but fewer than Clob) have problems with empty byte arrays.
+    http://hg.openjdk.java.net/jdk8/jdk8/jdk/file/687fd7c7986d/src/share/classes/javax/sql/rowset/serial/SerialBlob.java#l178
+    http://hg.openjdk.java.net/jdk9/client/jdk/file/1158c3e5bd9c/src/java.sql.rowset/share/classes/javax/sql/rowset/serial/SerialBlob.java#l178
+    https://github.com/apache/derby/blob/10.13/java/engine/org/apache/derby/iapi/types/HarmonySerialBlob.java#L111
+    OK! -> https://github.com/arteam/hsqldb/blob/2.3.4/src/org/hsqldb/jdbc/JDBCBlob.java#L184
+     */
+    def toBlobOption: Option[Blob] = bytesOption.flatMap(_.toBlobOption)
   }
 
-  implicit class BytesToBlob(val bytes: Array[Byte]) extends AnyVal {
-    def toBlob: Option[Blob] = if (bytes.isEmpty) None else Option(new SerialBlob(bytes))
+  implicit class BytesToBlobOption(val bytes: Array[Byte]) extends AnyVal {
+    def toBlobOption: Option[Blob] = if (bytes.isEmpty) None else Option(new SerialBlob(bytes))
   }
 }
diff --git a/database/sql/src/main/scala/cromwell/database/sql/SqlDatabase.scala b/database/sql/src/main/scala/cromwell/database/sql/SqlDatabase.scala
index e1431de76..c76c291b4 100644
--- a/database/sql/src/main/scala/cromwell/database/sql/SqlDatabase.scala
+++ b/database/sql/src/main/scala/cromwell/database/sql/SqlDatabase.scala
@@ -11,7 +11,8 @@ trait SqlDatabase extends AutoCloseable
   with JobStoreSqlDatabase
   with MetadataSqlDatabase
   with WorkflowStoreSqlDatabase
-  with SubWorkflowStoreSqlDatabase {
+  with SubWorkflowStoreSqlDatabase
+  with DockerHashStoreSqlDatabase {
 
   protected val urlKey: String
   protected val originalDatabaseConfig: Config
diff --git a/database/sql/src/main/scala/cromwell/database/sql/joins/CallCachingDiffJoin.scala b/database/sql/src/main/scala/cromwell/database/sql/joins/CallCachingDiffJoin.scala
new file mode 100644
index 000000000..2b56e9f88
--- /dev/null
+++ b/database/sql/src/main/scala/cromwell/database/sql/joins/CallCachingDiffJoin.scala
@@ -0,0 +1,5 @@
+package cromwell.database.sql.joins
+
+import cromwell.database.sql.tables.CallCachingEntry
+
+case class CallCachingDiffJoin(cacheEntryA: CallCachingEntry, cacheEntryB: CallCachingEntry, diff: Seq[(Option[(String, String)], Option[(String, String)])])
diff --git a/database/sql/src/main/scala/cromwell/database/sql/joins/CallCachingJoin.scala b/database/sql/src/main/scala/cromwell/database/sql/joins/CallCachingJoin.scala
index 8976c71cd..07313980b 100644
--- a/database/sql/src/main/scala/cromwell/database/sql/joins/CallCachingJoin.scala
+++ b/database/sql/src/main/scala/cromwell/database/sql/joins/CallCachingJoin.scala
@@ -1,11 +1,12 @@
 package cromwell.database.sql.joins
 
-import cromwell.database.sql.tables.{CallCachingDetritusEntry, CallCachingEntry, CallCachingHashEntry, CallCachingSimpletonEntry}
+import cromwell.database.sql.tables._
 
 case class CallCachingJoin
 (
   callCachingEntry: CallCachingEntry,
   callCachingHashEntries: Seq[CallCachingHashEntry],
+  callCachingAggregationEntry: Option[CallCachingAggregationEntry],
   callCachingSimpletonEntries: Seq[CallCachingSimpletonEntry],
   callCachingDetritusEntries: Seq[CallCachingDetritusEntry]
 )
diff --git a/database/sql/src/main/scala/cromwell/database/sql/joins/MetadataJobQueryValue.scala b/database/sql/src/main/scala/cromwell/database/sql/joins/MetadataJobQueryValue.scala
new file mode 100644
index 000000000..5bfa69c4a
--- /dev/null
+++ b/database/sql/src/main/scala/cromwell/database/sql/joins/MetadataJobQueryValue.scala
@@ -0,0 +1,18 @@
+package cromwell.database.sql.joins
+
+sealed trait MetadataJobQueryValue
+
+/**
+  * Applies a filter to retrieve only metadata keys matching this job
+  */
+case class CallQuery(callFqn: String, jobIndex: Option[Int], jobAttempt: Option[Int]) extends MetadataJobQueryValue
+
+/**
+  * Used to filter metadata keys at the workflow level only
+  */
+case object WorkflowQuery extends MetadataJobQueryValue
+
+/**
+  * Applies no filter on metadata job key. Both workflow level and job level keys are retrieved, for any job
+  */
+case object CallOrWorkflowQuery extends MetadataJobQueryValue
diff --git a/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingAggregationEntry.scala b/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingAggregationEntry.scala
new file mode 100644
index 000000000..3b5dab286
--- /dev/null
+++ b/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingAggregationEntry.scala
@@ -0,0 +1,9 @@
+package cromwell.database.sql.tables
+
+case class CallCachingAggregationEntry
+(
+  baseAggregation: String,
+  inputFilesAggregation: Option[String],
+  callCachingEntryId: Option[Int] = None,
+  callCachingAggregationEntryId: Option[Int] = None
+)
diff --git a/database/sql/src/main/scala/cromwell/database/sql/tables/CustomLabelsEntry.scala b/database/sql/src/main/scala/cromwell/database/sql/tables/CustomLabelsEntry.scala
new file mode 100644
index 000000000..030a5e0bb
--- /dev/null
+++ b/database/sql/src/main/scala/cromwell/database/sql/tables/CustomLabelsEntry.scala
@@ -0,0 +1,9 @@
+package cromwell.database.sql.tables
+
+case class CustomLabelEntry
+(
+  customLabelKey: String,
+  customLabelValue: String,
+  workflowExecutionUuid: String,
+  customLabelEntryId: Option[Long] = None
+)
diff --git a/database/sql/src/main/scala/cromwell/database/sql/tables/DockerHashStoreEntry.scala b/database/sql/src/main/scala/cromwell/database/sql/tables/DockerHashStoreEntry.scala
new file mode 100644
index 000000000..a32613144
--- /dev/null
+++ b/database/sql/src/main/scala/cromwell/database/sql/tables/DockerHashStoreEntry.scala
@@ -0,0 +1,9 @@
+package cromwell.database.sql.tables
+
+case class DockerHashStoreEntry
+(
+  workflowExecutionUuid: String,
+  dockerTag: String,
+  dockerHash: String,
+  dockerHashStoreEntryId: Option[Int] = None
+)
diff --git a/database/sql/src/main/scala/cromwell/database/sql/tables/WorkflowStoreEntry.scala b/database/sql/src/main/scala/cromwell/database/sql/tables/WorkflowStoreEntry.scala
index 467c6589c..41ca7794e 100644
--- a/database/sql/src/main/scala/cromwell/database/sql/tables/WorkflowStoreEntry.scala
+++ b/database/sql/src/main/scala/cromwell/database/sql/tables/WorkflowStoreEntry.scala
@@ -6,11 +6,13 @@ case class WorkflowStoreEntry
 (
   workflowExecutionUuid: String,
   workflowDefinition: Option[Clob],
+  workflowType: Option[String],
+  workflowTypeVersion: Option[String],
   workflowInputs: Option[Clob],
   workflowOptions: Option[Clob],
   workflowState: String,
   submissionTime: Timestamp,
   importsZip: Option[Blob],
-  customLabels: Option[Clob],
+  customLabels: Clob,
   workflowStoreEntryId: Option[Int] = None
 )
diff --git a/dockerHashing/src/main/scala/cromwell/docker/DockerClientHelper.scala b/dockerHashing/src/main/scala/cromwell/docker/DockerClientHelper.scala
new file mode 100644
index 000000000..9e97ca931
--- /dev/null
+++ b/dockerHashing/src/main/scala/cromwell/docker/DockerClientHelper.scala
@@ -0,0 +1,27 @@
+package cromwell.docker
+
+import akka.actor.{Actor, ActorLogging, ActorRef}
+import cromwell.core.actor.RobustClientHelper
+import cromwell.docker.DockerHashActor.DockerHashResponse
+
+import scala.concurrent.duration.FiniteDuration
+
+trait DockerClientHelper extends RobustClientHelper { this: Actor with ActorLogging =>
+  
+  protected def dockerHashingActor: ActorRef
+  
+  private [docker] def dockerResponseReceive: Receive = {
+    case dockerResponse: DockerHashResponse if hasTimeout(dockerResponse.request) =>
+      cancelTimeout(dockerResponse.request)
+      receive.apply(dockerResponse)
+    case (context: Any, dockerResponse: DockerHashResponse) if hasTimeout(context -> dockerResponse.request) =>
+      cancelTimeout(context -> dockerResponse.request)
+      receive.apply(context -> dockerResponse)
+  }
+
+  def sendDockerCommand(dockerHashRequest: DockerHashRequest, timeout: FiniteDuration = RobustClientHelper.DefaultRequestLostTimeout) = {
+    robustSend(dockerHashRequest, dockerHashingActor, timeout)
+  }
+
+  def dockerReceive = robustReceive orElse dockerResponseReceive
+}
diff --git a/core/src/main/scala/cromwell/core/callcaching/docker/DockerFlow.scala b/dockerHashing/src/main/scala/cromwell/docker/DockerFlow.scala
similarity index 91%
rename from core/src/main/scala/cromwell/core/callcaching/docker/DockerFlow.scala
rename to dockerHashing/src/main/scala/cromwell/docker/DockerFlow.scala
index cc5dc8bf0..e553cb8ee 100644
--- a/core/src/main/scala/cromwell/core/callcaching/docker/DockerFlow.scala
+++ b/dockerHashing/src/main/scala/cromwell/docker/DockerFlow.scala
@@ -1,4 +1,4 @@
-package cromwell.core.callcaching.docker
+package cromwell.docker
 
 import akka.NotUsed
 import akka.stream.{FlowShape, Graph}
diff --git a/core/src/main/scala/cromwell/core/callcaching/docker/DockerHashActor.scala b/dockerHashing/src/main/scala/cromwell/docker/DockerHashActor.scala
similarity index 55%
rename from core/src/main/scala/cromwell/core/callcaching/docker/DockerHashActor.scala
rename to dockerHashing/src/main/scala/cromwell/docker/DockerHashActor.scala
index 5e816de15..480a00c3d 100644
--- a/core/src/main/scala/cromwell/core/callcaching/docker/DockerHashActor.scala
+++ b/dockerHashing/src/main/scala/cromwell/docker/DockerHashActor.scala
@@ -1,15 +1,15 @@
-package cromwell.core.callcaching.docker
+package cromwell.docker
 
 import akka.actor.{Actor, ActorLogging, ActorRef, Props}
-import akka.pattern.pipe
-import akka.stream.QueueOfferResult.{Dropped, Enqueued, QueueClosed}
 import akka.stream._
 import akka.stream.scaladsl.{GraphDSL, Merge, Partition, Sink, Source}
 import com.google.common.cache.CacheBuilder
-import cromwell.core.callcaching.docker.DockerHashActor._
+import cromwell.core.Dispatcher
+import cromwell.core.actor.StreamActorHelper
+import cromwell.core.actor.StreamIntegration.StreamContext
+import cromwell.docker.DockerHashActor._
 import org.slf4j.LoggerFactory
 
-import scala.concurrent.Future
 import scala.concurrent.duration.FiniteDuration
 
 final class DockerHashActor(
@@ -17,7 +17,7 @@ final class DockerHashActor(
                            queueBufferSize: Int,
                            cacheEntryTTL: FiniteDuration,
                            cacheSize: Long
-                          )(implicit val materializer: ActorMaterializer) extends Actor with ActorLogging {
+                          )(implicit val materializer: ActorMaterializer) extends Actor with ActorLogging with StreamActorHelper[DockerHashContext] {
 
   implicit val system = context.system
   implicit val ec = context.dispatcher
@@ -47,29 +47,15 @@ final class DockerHashActor(
     .expireAfterWrite(cacheEntryTTL._1, cacheEntryTTL._2)
     .maximumSize(cacheSize)
     .build[DockerImageIdentifierWithoutHash, DockerHashResult]()
-  
-  /*
-   * Never fail the stream, if an exception occurs just drop the element
-   * The sender is responsible for sending a new message if it didn't get a response
-   * within a reasonable amount of time
-   */
-  private val decider: Supervision.Decider = _ => Supervision.Resume
 
   /*
    * Intermediate sink responsible for updating the cache as soon as a successful hash is retrieved
    */
   private val updateCacheSink = Sink.foreach[(DockerHashResponse, DockerHashContext)] {
-    case (response: DockerHashResponseSuccess, dockerHashContext) => 
+    case (response: DockerHashSuccessResponse, dockerHashContext) => 
       cache.put(dockerHashContext.dockerImageID, response.dockerHash)
     case _ => // Only put successful hashes in the cache
   }
-  
-  /*
-   * Final sink of the stream. Sends the response to the caller.
-   */
-  private val replySink = Sink.foreach[(DockerHashResponse, DockerHashContext)] {
-    case (response, dockerHashContext) => dockerHashContext.replyTo ! response
-  }
 
   /*
     Send each message to the corresponding docker registry flow if it exists,
@@ -104,65 +90,33 @@ final class DockerHashActor(
       partitionFlows.out(i) ~> flows(i) ~> mergeFlows.in(i)
     }
 
-    val noMatch = partitionFlows.out(dockerRegistryFlows.size).map(dockerContext => (DockerHashUnknownRegistry(dockerContext.dockerImageID), dockerContext)).outlet
+    val noMatch = partitionFlows.out(dockerRegistryFlows.size).map(dockerContext => (DockerHashUnknownRegistry(dockerContext.request), dockerContext)).outlet
     
     noMatch ~> mergeFlows.in(dockerRegistryFlows.size)
     
     FlowShape(partitionFlows.in, mergeFlows.out)
   }
 
-  private val queue = Source.queue[DockerHashContext](queueBufferSize, OverflowStrategy.dropNew)
-    .via(dockerFlow)
-    .alsoTo(updateCacheSink)
-    .to(replySink)
-    .withAttributes(ActorAttributes.supervisionStrategy(decider))
-    .run()
-  
-  override def receive: Receive = {
-    // When receiving a request, just enqueue it and pipe the result back to self
-    // Note: could be worth having another actor to send it to, to prevent overflowing this mailbox
+  private def checkCache(dockerHashRequest: DockerHashRequest) = {
+    Option(cache.getIfPresent(dockerHashRequest.dockerImageID)) map { hashResult => 
+      DockerHashSuccessResponse(hashResult, dockerHashRequest) 
+    }
+  }
+
+  override protected def actorReceive: Receive = {
     case request: DockerHashRequest =>
       val replyTo = sender()
-      
-      checkCache(request.dockerImageID) match {
+
+      checkCache(request) match {
         case Some(cacheHit) => replyTo ! cacheHit
         case None => sendToStream(DockerHashContext(request, replyTo))
       }
-    case EnqueueResponse(Enqueued, dockerHashContext) => // All good !
-    case EnqueueResponse(Dropped, dockerHashContext) => dockerHashContext.replyTo ! DockerHashBackPressure(dockerHashContext.request)
-      // In any of the below case, the stream is in a state where it will not be able to receive new elements.
-      // This means something blew off so we can just restart the actor to re-instantiate a new stream
-    case EnqueueResponse(QueueClosed, dockerHashContext) =>
-      val exception = new RuntimeException(s"Failed to enqueue docker hash request ${dockerHashContext.request}. Queue was closed")
-      logAndRestart(exception)
-    case EnqueueResponse(QueueOfferResult.Failure(failure), dockerHashContext) => 
-      logAndRestart(failure)
-    case FailedToEnqueue(throwable, dockerHashContext) =>
-      logAndRestart(throwable)
   }
-  
-  private def logAndRestart(throwable: Throwable) = {
-    log.warning("Failed to process docker hash request", throwable)
-    // Throw the exception that will be caught by supervisor and restart the actor
-    throw DockerHashActorException(throwable)
-  }
-  
-  private def checkCache(dockerImageIdentifierWithoutHash: DockerImageIdentifierWithoutHash) = {
-    Option(cache.getIfPresent(dockerImageIdentifierWithoutHash)) map { hashResult => 
-      DockerHashResponseSuccess(hashResult) 
-    }
-  }
-  
-  private def sendToStream(dockerHashContext: DockerHashContext) = {
-    val enqueue = queue offer dockerHashContext map { result =>
-      EnqueueResponse(result, dockerHashContext)
-    } recoverWith {
-      case t => Future.successful(FailedToEnqueue(t, dockerHashContext))
-    }
 
-    pipe(enqueue) to self
-    ()
-  }
+  override protected def streamSource = Source.queue[DockerHashContext](queueBufferSize, OverflowStrategy.dropNew)
+    .via(dockerFlow)
+    .alsoTo(updateCacheSink)
+    .withAttributes(ActorAttributes.dispatcher(Dispatcher.IoDispatcher))
 }
 
 object DockerHashActor {
@@ -170,31 +124,30 @@ object DockerHashActor {
   val logger = LoggerFactory.getLogger("DockerRegistry")
 
   /* Response Messages */
-  sealed trait DockerHashResponse
+  sealed trait DockerHashResponse {
+    def request: DockerHashRequest
+  }
   
-  case class DockerHashResponseSuccess(dockerHash: DockerHashResult) extends DockerHashResponse
+  case class DockerHashSuccessResponse(dockerHash: DockerHashResult, request: DockerHashRequest) extends DockerHashResponse
   
   sealed trait DockerHashFailureResponse extends DockerHashResponse {
     def reason: String
   }
-  case class DockerHashFailedResponse(failure: Throwable, dockerIdentifier: DockerImageIdentifierWithoutHash) extends DockerHashFailureResponse {
-    override val reason = s"Failed to get docker hash for ${dockerIdentifier.fullName} ${failure.getMessage}"
+  case class DockerHashFailedResponse(failure: Throwable, request: DockerHashRequest) extends DockerHashFailureResponse {
+    override val reason = s"Failed to get docker hash for ${request.dockerImageID.fullName} ${failure.getMessage}"
   }
-  case class DockerHashUnknownRegistry(dockerImageId: DockerImageIdentifierWithoutHash) extends DockerHashFailureResponse {
-    override val reason = s"Registry ${dockerImageId.host} is not supported"
+  case class DockerHashUnknownRegistry(request: DockerHashRequest) extends DockerHashFailureResponse {
+    override val reason = s"Registry ${request.dockerImageID.host} is not supported"
   }
-  case class DockerHashNotFound(dockerImageId: DockerImageIdentifierWithoutHash) extends DockerHashFailureResponse {
-    override val reason = s"Docker image ${dockerImageId.fullName} not found"
+  case class DockerHashNotFound(request: DockerHashRequest) extends DockerHashFailureResponse {
+    override val reason = s"Docker image ${request.dockerImageID.fullName} not found"
   }
-  case class DockerHashUnauthorized(dockerImageId: DockerImageIdentifierWithoutHash) extends DockerHashFailureResponse {
-    override val reason = s"Unauthorized to get docker hash ${dockerImageId.fullName}"
+  case class DockerHashUnauthorized(request: DockerHashRequest) extends DockerHashFailureResponse {
+    override val reason = s"Unauthorized to get docker hash ${request.dockerImageID.fullName}"
   }
 
-  case class DockerHashBackPressure(originalRequest: DockerHashRequest) extends DockerHashResponse
-  case class DockerHashActorException(failure: Throwable) extends RuntimeException(failure)
-  
   /* Internal ADTs */
-  case class DockerHashContext(request: DockerHashRequest, replyTo: ActorRef) {
+  case class DockerHashContext(request: DockerHashRequest, replyTo: ActorRef) extends StreamContext {
     val dockerImageID = request.dockerImageID
     val credentials = request.credentials
   }
@@ -205,5 +158,7 @@ object DockerHashActor {
   def props(dockerRegistryFlows: Seq[DockerFlow],
             queueBufferSize: Int = 100,
             cacheEntryTTL: FiniteDuration,
-            cacheSize: Long)(materializer: ActorMaterializer) = Props(new DockerHashActor(dockerRegistryFlows, queueBufferSize, cacheEntryTTL, cacheSize)(materializer))
+            cacheSize: Long)(materializer: ActorMaterializer) = {
+    Props(new DockerHashActor(dockerRegistryFlows, queueBufferSize, cacheEntryTTL, cacheSize)(materializer)).withDispatcher(Dispatcher.IoDispatcher)
+  }
 }
diff --git a/core/src/main/scala/cromwell/core/callcaching/docker/DockerHashRequest.scala b/dockerHashing/src/main/scala/cromwell/docker/DockerHashRequest.scala
similarity index 73%
rename from core/src/main/scala/cromwell/core/callcaching/docker/DockerHashRequest.scala
rename to dockerHashing/src/main/scala/cromwell/docker/DockerHashRequest.scala
index 6c7206d25..c6956318f 100644
--- a/core/src/main/scala/cromwell/core/callcaching/docker/DockerHashRequest.scala
+++ b/dockerHashing/src/main/scala/cromwell/docker/DockerHashRequest.scala
@@ -1,3 +1,3 @@
-package cromwell.core.callcaching.docker
+package cromwell.docker
 
 case class DockerHashRequest(dockerImageID: DockerImageIdentifierWithoutHash, credentials: List[Any] = List.empty)
diff --git a/dockerHashing/src/main/scala/cromwell/docker/DockerHashResult.scala b/dockerHashing/src/main/scala/cromwell/docker/DockerHashResult.scala
new file mode 100644
index 000000000..e3120516a
--- /dev/null
+++ b/dockerHashing/src/main/scala/cromwell/docker/DockerHashResult.scala
@@ -0,0 +1,19 @@
+package cromwell.docker
+
+import scala.util.{Failure, Success, Try}
+
+object DockerHashResult {
+  // See https://docs.docker.com/registry/spec/api/#/content-digests
+  val DigestRegex = """([a-zA-Z0-9_+.-]+):([a-zA-Z0-9]+)""".r
+  
+  def fromString(str: String): Try[DockerHashResult] = {
+    str match {
+      case DigestRegex(alg, hash) => Success(DockerHashResult(alg, hash))
+      case _ => Failure(new IllegalArgumentException(s"Hash value $str does not have the expected 'algorithm:hash' syntax"))
+    }
+  }
+}
+
+case class DockerHashResult(hashAlgorithm: String, hashValue: String) {
+  val algorithmAndHash = s"$hashAlgorithm:$hashValue"
+}
diff --git a/dockerHashing/src/main/scala/cromwell/docker/DockerImageIdentifier.scala b/dockerHashing/src/main/scala/cromwell/docker/DockerImageIdentifier.scala
new file mode 100644
index 000000000..eb3ecf6c3
--- /dev/null
+++ b/dockerHashing/src/main/scala/cromwell/docker/DockerImageIdentifier.scala
@@ -0,0 +1,91 @@
+package cromwell.docker
+
+import scala.util.{Failure, Success, Try}
+
+sealed trait DockerImageIdentifier {
+  def host: Option[String]
+  def repository: Option[String]
+  def image: String
+  def reference: String
+
+  // The name of the image with a repository prefix iff a repository was explicitly specified.
+  lazy val name = repository map { r => s"$r/$image" } getOrElse image
+  // The name of the image with a repository prefix if a repository was specified, or with a default repository prefix of
+  // "library" if no repository was specified.
+  lazy val nameWithDefaultRepository = repository.getOrElse("library") + s"/$image"
+  lazy val hostAsString = host map { h => s"$h/" } getOrElse ""
+  // The full name of this image, including a repository prefix only if a repository was explicitly specified.
+  lazy val fullName = s"$hostAsString$name:$reference"
+}
+
+case class DockerImageIdentifierWithoutHash(host: Option[String], repository: Option[String], image: String, reference: String) extends DockerImageIdentifier {
+  def withHash(hash: DockerHashResult) = DockerImageIdentifierWithHash(host, repository, image, reference, hash)
+}
+
+case class DockerImageIdentifierWithHash(host: Option[String], repository: Option[String], image: String, reference: String, hash: DockerHashResult) extends DockerImageIdentifier {
+  override lazy val fullName: String = s"$hostAsString$name@${hash.algorithmAndHash}"
+}
+
+object DockerImageIdentifier {
+  private val DefaultDockerTag = "latest"
+  
+  private val DockerStringRegex =
+    s"""
+       (?x)                                     # Turn on comments and whitespace insensitivity
+       
+       (                                        # Begin capturing group for name
+          [a-z0-9]+(?:[._-][a-z0-9]+)*          # API v2 name component regex - see https://docs.docker.com/registry/spec/api/#/overview
+          (?:/[a-z0-9]+(?:[._-][a-z0-9]+)*)*    # Optional additional name components separated by /
+       )                                        # End capturing group for name
+       
+       (?:   
+          :                                     # Tag separator. ':' is followed by a tag
+       
+          (                                     # Begin capturing group for reference 
+            [A-Za-z0-9]+(?:[.:_-][A-Za-z0-9]+)* # Reference
+          )                                     # End capturing group for reference  
+       )?
+       (?:   
+          @                                     # Tag separator '@' is followed by a digest
+             
+          (                                     # Begin capturing group for reference 
+            [A-Za-z0-9]+(?:[.:_-][A-Za-z0-9]+)* # Reference
+          )                                     # End capturing group for reference  
+       )?
+       """.trim.r
+  
+  def fromString(dockerString: String): Try[DockerImageIdentifier] = {
+    dockerString match {
+      case DockerStringRegex(name, tag, hash) => buildId(name, Option(tag), Option(hash))
+      case _ => Failure(new IllegalArgumentException(s"Docker image $dockerString has an invalid syntax."))
+    }
+  }
+  
+  private def isRegistryHostName(str: String) = str.contains('.')
+  
+  private def buildId(name: String, tag: Option[String], hash: Option[String]) = {
+    val (dockerHost, dockerRepo, dockerImage): (Option[String], Option[String], String) = name.split('/').toList match {
+      // If just one component (e.g ubuntu)
+      case image :: Nil => (None, None, image)
+      // If repo/image (e.g broadinstitute/cromwell) without host
+      case repo :: image :: Nil if !isRegistryHostName(repo) => (None, Option(repo), image)
+      // If host/image (e.g index.docker.io/ubuntu), assume default repo
+      case host :: image :: Nil if isRegistryHostName(host) => (Option(host), None, image)
+      // Not a host followed more than one components
+      case nothost :: rest if !isRegistryHostName(nothost) =>
+        val repo = s"$nothost/${rest.init.mkString("/")}"
+        (None, Option(repo), rest.last)
+      // A host followed more than one components (e.g gcr.io/google-containers/alpine-with-bash)
+      case host :: rest if isRegistryHostName(host) =>
+        val repo = rest.init.mkString("/")
+        (Option(host), Option(repo), rest.last)
+    }
+    
+    (tag, hash) match {
+      case (None, None) => Success(DockerImageIdentifierWithoutHash(dockerHost, dockerRepo, dockerImage, DefaultDockerTag))
+      case (Some(t), None) => Success(DockerImageIdentifierWithoutHash(dockerHost, dockerRepo, dockerImage, t))
+      case (None, Some(h)) => DockerHashResult.fromString(h) map { hash => DockerImageIdentifierWithHash(dockerHost, dockerRepo, dockerImage, h, hash) }
+      case (Some(t), Some(h)) => DockerHashResult.fromString(h) map { hash => DockerImageIdentifierWithHash(dockerHost, dockerRepo, dockerImage, s"$t@$h", hash) }
+    }
+  }
+}
diff --git a/dockerHashing/src/main/scala/cromwell/docker/local/DockerCliClient.scala b/dockerHashing/src/main/scala/cromwell/docker/local/DockerCliClient.scala
new file mode 100644
index 000000000..9f3372422
--- /dev/null
+++ b/dockerHashing/src/main/scala/cromwell/docker/local/DockerCliClient.scala
@@ -0,0 +1,105 @@
+package cromwell.docker.local
+
+import scala.util.Try
+
+/**
+  * Wrapper around the docker cli.
+  * https://docs.docker.com/engine/reference/commandline/docker/
+  *
+  * An alternative to using REST calls directly to the docker engine that has several versions, for example:
+  * https://docs.docker.com/engine/api/v1.27/
+  */
+trait DockerCliClient {
+  /**
+    * Looks up a docker hash.
+    *
+    * @param dockerCliKey The docker hash to lookup.
+    * @return The hash if found, None if not found, and Failure if an error occurs.
+    */
+  def lookupHash(dockerCliKey: DockerCliKey): Try[Option[String]] = {
+    /*
+    The stdout contains the tab separated repository/tag/digest for __all__ local images.
+    Would be great to just get a single hash using the key... unfortunately
+    https://github.com/docker/docker/issues/29901
+     */
+    forRun("docker", "images", "--digests", "--format", """{{printf "%s\t%s\t%s" .Repository .Tag .Digest}}""") {
+      _.flatMap(parseHashLine).find(_.key == dockerCliKey).map(_.digest)
+    }
+  }
+
+  /**
+    * Pulls a docker image.
+    * @param dockerCliKey The docker hash to lookup.
+    * @return Failure if an error occurs.
+    */
+  def pull(dockerCliKey: DockerCliKey): Try[Unit] = {
+    forRun("docker", "pull", dockerCliKey.fullName) { _ => () }
+  }
+
+  /**
+    * Tries to run the command, then feeds the stdout to `f`. If the exit code is non-zero, returns a `Failure` with
+    * a `RuntimeException` containing the stderr.
+    *
+    * @param cmd Command line to run.
+    * @param f The function to run on the stdout contents.
+    * @tparam A Return type.
+    * @return An attempt to run A.
+    */
+  private def forRun[A](cmd: String*)(f: Seq[String] => A): Try[A] = {
+    Try {
+      val dockerCliResult = run(cmd)
+      if (dockerCliResult.exitCode == 0) {
+        f(dockerCliResult.stdout)
+      } else {
+        throw new RuntimeException(
+          s"""|Error running: ${cmd.mkString(" ")}
+              |Exit code: ${dockerCliResult.exitCode}
+              |${dockerCliResult.stderr.mkString("\n")}
+              |""".stripMargin)
+      }
+    }
+  }
+
+  /**
+    * Run a command and return the result. Overridable for testing.
+    *
+    * @param cmd The command to run.
+    * @return The results of the run wrapped in a DockerCliResult.
+    */
+  private[local] def run(cmd: Seq[String]): DockerCliResult = {
+    import sys.process._
+    var stdout = List.empty[String]
+    var stderr = List.empty[String]
+    val exitCode = cmd.!(ProcessLogger(line => stdout :+= line, line => stderr :+= line))
+    DockerCliResult(exitCode, stdout, stderr)
+  }
+
+  /**
+    * Parses a line for `lookupHash`, returning None for lines that contain the string "<none>" for any of the columns.
+    * @param hashLine The line output by the stdout of the `lookupHash` command.
+    * @return An optional `DockerCliHash`, if the all the columns are found.
+    */
+  private[local] def parseHashLine(hashLine: String): Option[DockerCliHash] = {
+    val none = "<none>"
+    val tokens = hashLine.split("\t").lift
+    for {
+      repository <- tokens(0) if repository != none
+      tag <- tokens(1) if tag != none
+      digest <- tokens(2) if digest != none
+    } yield DockerCliHash(DockerCliKey(repository, tag), digest)
+  }
+}
+
+object DockerCliClient extends DockerCliClient
+
+/**
+  * Used by `lookupHash` to return a mapping of keys to digests.
+  * See note in `lookupHash` regarding why we even have to loop over all images.
+  */
+case class DockerCliHash(key: DockerCliKey, digest: String)
+
+/**
+  * Utility that encapsulates a command's exit code, stdout, and stderr.
+  * Also used by tests to simulate cli responses.
+  */
+case class DockerCliResult(exitCode: Int, stdout: Seq[String], stderr: Seq[String])
diff --git a/dockerHashing/src/main/scala/cromwell/docker/local/DockerCliFlow.scala b/dockerHashing/src/main/scala/cromwell/docker/local/DockerCliFlow.scala
new file mode 100644
index 000000000..94088beb4
--- /dev/null
+++ b/dockerHashing/src/main/scala/cromwell/docker/local/DockerCliFlow.scala
@@ -0,0 +1,144 @@
+package cromwell.docker.local
+
+import java.util.concurrent.TimeoutException
+
+import akka.actor.Scheduler
+import akka.stream.scaladsl.{Flow, GraphDSL, Merge, Partition}
+import akka.stream.FlowShape
+import cromwell.docker.DockerHashActor._
+import cromwell.docker.{DockerFlow, DockerHashActor, DockerHashResult, DockerImageIdentifierWithoutHash}
+
+import scala.concurrent.duration._
+import scala.concurrent.{ExecutionContext, Future}
+import scala.util.{Failure, Success}
+
+/**
+  * A docker flow using the CLI to return docker hashes.
+  */
+class DockerCliFlow(implicit ec: ExecutionContext, scheduler: Scheduler) extends DockerFlow {
+
+  // If the docker cli hangs it would be difficult to debug. So timeout the first request after a short duration.
+  // https://github.com/docker/docker/issues/18279
+  // https://github.com/docker/docker/issues/12606
+  lazy val firstLookupTimeout = 5.seconds
+
+  override def accepts(dockerImageIdentifierWithoutHash: DockerImageIdentifierWithoutHash): Boolean = true
+
+  override def buildFlow() = GraphDSL.create() { implicit builder =>
+    import GraphDSL.Implicits._
+
+    // Try a lookup.
+    val firstLookup = builder.add(
+      Flow
+        .fromFunction(DockerCliFlow.lookupHashOrTimeout(firstLookupTimeout))
+        .mapAsync(1)(identity)
+    )
+
+    // Check if the hash was not found
+    val partition = builder.add(Partition[(DockerHashResponse, DockerHashContext)](2, {
+      case (_: DockerHashNotFound, _) => 0 // If we didn't find the docker hash, go to out(0)
+      case _ => 1 // Otherwise, go to out(1)
+    }))
+
+    // If the hash wasn't found, `docker pull` then try a lookup a second time. If the docker pull fails for any
+    // reason, the error is ignored, allowing the second lookup to return not found.
+    val dockerPull = builder.add(Flow.fromFunction(Function.tupled(DockerCliFlow.pull _)))
+    val secondLookup = builder.add(Flow.fromFunction(DockerCliFlow.lookupHash))
+
+    // Use either the first or second docker lookup.
+    val merge = builder.add(Merge[(DockerHashResponse, DockerHashContext)](2))
+
+    // @formatter:off
+    firstLookup.out ~> partition
+                       partition.out(0) ~> dockerPull ~> secondLookup ~> merge.in(0)
+                       partition.out(1)               ~>                 merge.in(1)
+    // @formatter:on
+
+    FlowShape(firstLookup.in, merge.out)
+  }
+}
+
+object DockerCliFlow {
+  /**
+    * Lookup the hash for the image referenced in the context.
+    *
+    * @param context The image to lookup.
+    * @return The docker hash response plus the context of our flow.
+    */
+  private def lookupHash(context: DockerHashContext): (DockerHashResponse, DockerHashContext) = {
+    val dockerCliKey = cliKeyFromImageId(context)
+    DockerHashActor.logger.debug("Looking up hash of {}", dockerCliKey.fullName)
+    val result = DockerCliClient.lookupHash(dockerCliKey) match {
+      case Success(None) => DockerHashNotFound(context.request)
+      case Success(Some(hash)) => DockerHashResult.fromString(hash) match {
+        case Success(r) => DockerHashSuccessResponse(r, context.request)
+        case Failure(t) => DockerHashFailedResponse(t, context.request)
+      }
+      case Failure(throwable) => DockerHashFailedResponse(throwable, context.request)
+    }
+    // give the compiler a hint on the debug() override we're trying to use.
+    DockerHashActor.logger.debug("Hash result of {} was {}", dockerCliKey.fullName, result.asInstanceOf[Any])
+    (result, context)
+  }
+
+  /**
+    * Lookup the hash for the image referenced in the context within the timeout, or DockerHashFailedResponse
+    * containing a TimeoutException.
+    *
+    * @param timeout   How long to wait for the exception.
+    * @param context   The image to lookup.
+    * @param scheduler Schedules the timout exception.
+    * @return The docker hash response plus the context of our flow.
+    */
+  private def lookupHashOrTimeout(timeout: FiniteDuration)
+                                 (context: DockerHashContext)
+                                 (implicit ec: ExecutionContext,
+                                  scheduler: Scheduler): Future[(DockerHashResponse, DockerHashContext)] = {
+    val normal = Future(lookupHash(context))
+    val delayed = akka.pattern.after(timeout, scheduler) {
+      val dockerCliKey = cliKeyFromImageId(context)
+      val exception = new TimeoutException(
+        s"""|Timeout while looking up hash of ${dockerCliKey.fullName}.
+            |Ensure that docker is running correctly.
+            |""".stripMargin)
+      val response = DockerHashFailedResponse(exception, context.request)
+      Future.successful((response, context))
+    }
+    Future.firstCompletedOf(Seq(normal, delayed))
+  }
+
+  /**
+    * Pull the docker image referenced in context.
+    *
+    * @param notUsed Output of `lookupHash`, passed in only to help creating the graph easier.
+    * @param context The image to pull.
+    * @return The context of our flow.
+    */
+  private def pull(notUsed: DockerHashResponse, context: DockerHashContext): DockerHashContext = {
+    val dockerCliKey = cliKeyFromImageId(context)
+    DockerHashActor.logger.info(s"Attempting to pull {}", dockerCliKey.fullName)
+    val result = DockerCliClient.pull(dockerCliKey)
+    result match {
+      case Success(_) => DockerHashActor.logger.info("Pulled {}", dockerCliKey.fullName)
+      case Failure(throwable) => DockerHashActor.logger.error("Docker pull failed", throwable)
+    }
+    context
+  }
+
+  /** Utility for converting the flow image id to the format output by the docker cli. */
+  private def cliKeyFromImageId(context: DockerHashContext): DockerCliKey = {
+    val imageId = context.dockerImageID
+    (imageId.host, imageId.repository) match {
+      case (None, None) =>
+        // For docker hub images (host == None), and don't include "library".
+        val repository = imageId.image
+        val tag = imageId.reference
+        DockerCliKey(repository, tag)
+      case _ =>
+        // For all other images, include the host and repository.
+        val repository = s"${imageId.hostAsString}${imageId.nameWithDefaultRepository}"
+        val tag = imageId.reference
+        DockerCliKey(repository, tag)
+    }
+  }
+}
diff --git a/dockerHashing/src/main/scala/cromwell/docker/local/DockerCliKey.scala b/dockerHashing/src/main/scala/cromwell/docker/local/DockerCliKey.scala
new file mode 100644
index 000000000..a074a968b
--- /dev/null
+++ b/dockerHashing/src/main/scala/cromwell/docker/local/DockerCliKey.scala
@@ -0,0 +1,14 @@
+package cromwell.docker.local
+
+/**
+  * Used by the docker cli for lookups or pulls.
+  *
+  * The format of `repository` is based on the output of `docker images`. It is also valid as input for the cli
+  * commands.
+  *
+  * @param repository The repository as listed by `docker images`. Includes the image host and name, but not the tag.
+  * @param tag The docker image tag.
+  */
+case class DockerCliKey(repository: String, tag: String) {
+  val fullName = s"$repository:$tag"
+}
diff --git a/core/src/main/scala/cromwell/core/callcaching/docker/registryv2/DockerRegistryV2AbstractFlow.scala b/dockerHashing/src/main/scala/cromwell/docker/registryv2/DockerRegistryV2AbstractFlow.scala
similarity index 56%
rename from core/src/main/scala/cromwell/core/callcaching/docker/registryv2/DockerRegistryV2AbstractFlow.scala
rename to dockerHashing/src/main/scala/cromwell/docker/registryv2/DockerRegistryV2AbstractFlow.scala
index 0514e0c59..fbaae329a 100644
--- a/core/src/main/scala/cromwell/core/callcaching/docker/registryv2/DockerRegistryV2AbstractFlow.scala
+++ b/dockerHashing/src/main/scala/cromwell/docker/registryv2/DockerRegistryV2AbstractFlow.scala
@@ -1,6 +1,7 @@
-package cromwell.core.callcaching.docker.registryv2
+package cromwell.docker.registryv2
 
 import akka.NotUsed
+import akka.actor.Scheduler
 import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._
 import akka.http.scaladsl.model.HttpHeader.ParsingResult
 import akka.http.scaladsl.model._
@@ -8,17 +9,21 @@ import akka.http.scaladsl.model.headers.{Authorization, OAuth2BearerToken}
 import akka.http.scaladsl.unmarshalling.Unmarshal
 import akka.stream._
 import akka.stream.scaladsl.{Flow, GraphDSL, Merge}
-import cromwell.core.callcaching.docker.DockerHashActor._
-import cromwell.core.callcaching.docker.registryv2.DockerRegistryV2AbstractFlow._
-import cromwell.core.callcaching.docker.registryv2.flows.{FlowUtils, HttpFlowWithRetry}
-import cromwell.core.callcaching.docker.{DockerFlow, DockerHashResult, DockerImageIdentifierWithoutHash}
+import cromwell.docker.DockerHashActor._
+import cromwell.docker.registryv2.DockerRegistryV2AbstractFlow._
+import cromwell.docker.registryv2.flows.HttpFlowWithRetry.ContextWithRequest
+import cromwell.docker.registryv2.flows.{FlowUtils, HttpFlowWithRetry}
+import cromwell.docker.{DockerFlow, DockerHashResult, DockerImageIdentifierWithoutHash}
 import spray.json._
 
+import scala.concurrent.duration._
 import scala.concurrent.{ExecutionContext, Future}
+import scala.language.postfixOps
 import scala.util.{Failure, Success, Try}
 
 object DockerRegistryV2AbstractFlow {
-  type HttpDockerFlow = Flow[(HttpRequest, (DockerHashContext, HttpRequest)), (Try[HttpResponse], (DockerHashContext, HttpRequest)), NotUsed]
+  type HttpDockerFlow = Flow[(HttpRequest, ContextWithRequest[DockerHashContext]), (Try[HttpResponse], ContextWithRequest[DockerHashContext]), NotUsed]
+  val StrictTimeout = 30 seconds
   
   val DigestHeaderName = "Docker-Content-Digest".toLowerCase
   val AcceptHeader = HttpHeader.parse("Accept", "application/vnd.docker.distribution.manifest.v2+json") match {
@@ -28,6 +33,8 @@ object DockerRegistryV2AbstractFlow {
       header
     case ParsingResult.Error(error) => throw new RuntimeException(error.formatPretty)
   }
+
+  case class UnsuccessfulHttpResponseException(httpResponse: HttpResponse) extends Exception
 }
 
 /**
@@ -35,16 +42,50 @@ object DockerRegistryV2AbstractFlow {
   * from a registry server conforming to the docker registry API v2 Specification:
   * https://docs.docker.com/registry/spec/api/
   */
-abstract class DockerRegistryV2AbstractFlow(httpClientFlow: HttpDockerFlow)(implicit ec: ExecutionContext, materializer: ActorMaterializer) extends DockerFlow {
+abstract class DockerRegistryV2AbstractFlow(httpClientFlow: HttpDockerFlow)(implicit ec: ExecutionContext, materializer: ActorMaterializer, scheduler: Scheduler) extends DockerFlow {
   // Wraps the Http flow in a retryable flow to enable auto retries
-  final private val httpFlowWithRetry = new HttpFlowWithRetry[DockerHashContext](httpClientFlow, isRetryable = isRetryable).flow
+  final private val httpFlowWithRetry = GraphDSL.create() { implicit builder =>
+    import GraphDSL.Implicits._
+    val retryHttpFlow = builder.add(new HttpFlowWithRetry[DockerHashContext](httpClientFlow).flow)
+    
+    // Force the response entity to be strict. This makes sure that whatever happens to the response later we
+    // won't leave it hanging and potentially lock the pool. 
+    // See http://doc.akka.io/docs/akka-http/10.0.5/scala/http/client-side/request-level.html#using-the-future-based-api-in-actors
+    // Note that in this particular case it's ok to force the loading of the entity in memory
+    // because we use HEAD Http method when we only care about the headers. Therefore there's no unnecessary memory usage.
+    /* Returns a (Try[HttpResponse], DockerHashContext) */
+    val strictHttpResponse = retryHttpFlow.out0.mapAsync(1){
+      case (response, context) => response.toStrict(StrictTimeout) map { Success(_) -> context } recoverWith {
+        case failure => Future.successful(Failure(failure) -> context)
+      }
+    }
+    
+    // Splits successful `toStrict` responses from failures
+    val partitionStrictResponse = builder.add(FlowUtils.fanOutTry[HttpResponse, DockerHashContext])
+
+    // Merge failures from retryHttpFlow.out1 (failed http responses)
+    // and partitionStrictResponse.out1 (failed to `toStrict` the response)
+    val mergeFailures = builder.add(Merge[(Throwable, DockerHashContext)](2))
+    
+    strictHttpResponse.outlet ~> partitionStrictResponse.in
 
-  final private val tokenFlow = {
-    val responseHandlerFlow = Flow[(HttpResponse, DockerHashContext)].mapAsync(1)(Function.tupled(tokenResponseHandler))
+    retryHttpFlow.out1 ~> mergeFailures
+    partitionStrictResponse.out1 ~> mergeFailures
+    
+    new FanOutShape2(retryHttpFlow.in, partitionStrictResponse.out0, mergeFailures.out)
+  }
+
+  private [registryv2] val tokenFlow = {
+    val responseHandlerFlow = Flow[(HttpResponse, DockerHashContext)]
+      .mapAsync(1)(Function.tupled(tokenResponseHandler))
+      // Map the Try[String] token to a Try[Option[String]].
+      // This allows potential users of this class to override this flow and return an empty token
+      .map { case (tryToken, context) => (tryToken map Option.apply, context) }
+    
     requestTransformFlow(buildTokenRequest, responseHandlerFlow)
   }
 
-  final private val manifestFlow = {
+  private [registryv2] val manifestFlow = {
     val responseHandlerFlow = Flow[(HttpResponse, DockerHashContext)].map(Function.tupled(manifestResponseHandler))
     requestTransformFlow(Function.tupled(buildManifestRequest _), responseHandlerFlow)
   }
@@ -54,12 +95,9 @@ abstract class DockerRegistryV2AbstractFlow(httpClientFlow: HttpDockerFlow)(impl
     */
   def buildFlow() = GraphDSL.create() { implicit builder =>
     import GraphDSL.Implicits._
-    
-    // Decouple the token flow from the manifest flow with ".async" 
-    // this way they can run in parallel from each other 
-    // while still maintaining the final ordering
-    val token = builder.add(tokenFlow.async)
-    val manifest = builder.add(manifestFlow.async)
+
+    val token = builder.add(tokenFlow)
+    val manifest = builder.add(manifestFlow)
     val mergeResponses = builder.add(Merge[(DockerHashResponse, DockerHashContext)](3))
     
     token.out0 ~> manifest.in
@@ -96,18 +134,6 @@ abstract class DockerRegistryV2AbstractFlow(httpClientFlow: HttpDockerFlow)(impl
   /* Methods that may be overridden by a subclass */
 
   /**
-    * Returns true if an http response failed in such a way that it can be retried,
-    * false otherwise.
-    */
-  protected def isRetryable(httpResponse: HttpResponse) = {
-    httpResponse.status match {
-      case StatusCodes.TooManyRequests => true
-      case StatusCodes.InternalServerError => true
-      case _ => false
-    }
-  }
-
-  /**
     * service parameter that can be used in the token request
     * (e.g https://auth.docker.io/token?service=registry.docker.io&scope=repository:library/ubuntu:pull)
     */
@@ -118,29 +144,13 @@ abstract class DockerRegistryV2AbstractFlow(httpClientFlow: HttpDockerFlow)(impl
     */
   protected def buildTokenRequestUri(dockerImageID: DockerImageIdentifierWithoutHash): String = {
     val service = serviceName map { name => s"service=$name&" } getOrElse ""
-    s"https://$authorizationServerHostName/token?${service}scope=repository:${dockerImageID.name}:pull"
+    s"https://$authorizationServerHostName/token?${service}scope=repository:${dockerImageID.nameWithDefaultRepository}:pull"
   }
 
   /**
     * Http method used for the manifest request
     */
-  protected def manifestRequestHttpMethod: HttpMethod = HttpMethods.GET
-
-  /**
-    * Maps a failed http response to a DockerHashFailureResponse
-    */
-  protected def httpFailureMapper(response: HttpResponse, dockerHashContext: DockerHashContext) = {
-    response.status match {
-      case StatusCodes.NotFound => (DockerHashNotFound(dockerHashContext.dockerImageID), dockerHashContext)
-      case StatusCodes.Unauthorized => (DockerHashUnauthorized(dockerHashContext.dockerImageID), dockerHashContext)
-      case other => (
-        DockerHashFailedResponse(
-          new Exception(s"Docker hash lookup failed with code ${other.intValue()}. ${other.defaultMessage()}"), dockerHashContext.dockerImageID
-        ),
-        dockerHashContext
-        )
-    }
-  }
+  protected def manifestRequestHttpMethod: HttpMethod = HttpMethods.HEAD
 
   /**
     * Generic method to build a flow that creates a request, sends it,
@@ -162,24 +172,32 @@ abstract class DockerRegistryV2AbstractFlow(httpClientFlow: HttpDockerFlow)(impl
 
     // Perform http requests
     val http = builder.add(httpFlowWithRetry)
-    val successfulResponse = http.out0.via(responseHandlerFlow)
-    val failedResponse = http.out1.map(Function.tupled(httpFailureMapper))
+    
+    // We got a HttpResponse, it might still be an unsuccessful response though.
+    // Regardless send it through the responseHandlerFlow
+    val finalResponse = http.out0.via(responseHandlerFlow)
+    
+    // The request could not be executed, map it to a failure message
+    val failedRequest = http.out1.map({
+      case (failure, commandContext) => throwableToFailureMessage(failure, commandContext)
+    })
 
-    // Partition parsing results
-    val parser = builder.add(FlowUtils.fanOutTry[B, DockerHashContext])
-    val successfulParsing: Outlet[(B, DockerHashContext)] = parser.out0
-    val failedParsing = parser.out1.map(Function.tupled(throwableToFailureMessage))
+    // Partition processed responses
+    val processResponse = builder.add(FlowUtils.fanOutTry[B, DockerHashContext])
+    
+    val successfulProcessing = processResponse.out0
+    val failedProcessing = processResponse.out1.map(Function.tupled(throwableToFailureMessage))
 
     // Merge failures from failed response and failed parsing
     val mergeFailures = builder.add(Merge[(DockerHashResponse, DockerHashContext)](2))
 
     request ~> http.in
-    successfulResponse ~> parser.in
+    finalResponse ~> processResponse.in
 
-    failedResponse ~> mergeFailures.in(0)
-    failedParsing ~> mergeFailures.in(1)
+    failedRequest ~> mergeFailures.in(0)
+    failedProcessing ~> mergeFailures.in(1)
 
-    new FanOutShape2(request.in, successfulParsing, mergeFailures.out)
+    new FanOutShape2(request.in, successfulProcessing, mergeFailures.out)
   }
 
   /**
@@ -213,8 +231,8 @@ abstract class DockerRegistryV2AbstractFlow(httpClientFlow: HttpDockerFlow)(impl
     case httpResponse =>
       Future.successful(
         (
-        Failure(new Exception(s"Request to obtain docker registry access token returned with a failure code: ${httpResponse.status.value}")),
-        dockerHashContext
+          Failure(UnsuccessfulHttpResponseException(httpResponse)),
+          dockerHashContext
         )
       )
   }
@@ -222,10 +240,10 @@ abstract class DockerRegistryV2AbstractFlow(httpClientFlow: HttpDockerFlow)(impl
   /**
     * Extract the access token from the json body of the http response
     */
-  private def extractToken(jsObject: JsObject) = {
+  private def extractToken(jsObject: JsObject): Try[String] = {
     jsObject.fields.get("token") match {
       case Some(token: JsString) => Success(token.value)
-      case Some(other) => Failure(new Exception("Token response contains a non-string token field"))
+      case Some(_) => Failure(new Exception("Token response contains a non-string token field"))
       case None => Failure(new Exception("Token response did not contain a token field"))
     }
   }
@@ -234,20 +252,26 @@ abstract class DockerRegistryV2AbstractFlow(httpClientFlow: HttpDockerFlow)(impl
     * Builds the manifest URI to be queried based on a DockerImageID
     */
   private def buildManifestUri(dockerImageID: DockerImageIdentifierWithoutHash): String = {
-    s"https://$registryHostName/v2/${dockerImageID.name}/manifests/${dockerImageID.reference}"
+    s"https://$registryHostName/v2/${dockerImageID.nameWithDefaultRepository}/manifests/${dockerImageID.reference}"
   }
 
   /**
     * Builds the manifest http request
     */
-  private def buildManifestRequest(token: String, dockerHashContext: DockerHashContext) = {
-    val authorizationHeader = Authorization(OAuth2BearerToken(token))
-
-    val manifestRequest = HttpRequest(
-      method = manifestRequestHttpMethod,
-      uri = buildManifestUri(dockerHashContext.dockerImageID),
-      headers = scala.collection.immutable.Seq(AcceptHeader, authorizationHeader)
-    )
+  private def buildManifestRequest(token: Option[String], dockerHashContext: DockerHashContext) = {
+    val manifestRequest = token match {
+      case Some(authToken) =>
+        HttpRequest(
+          method = manifestRequestHttpMethod,
+          uri = buildManifestUri(dockerHashContext.dockerImageID),
+          headers = scala.collection.immutable.Seq(AcceptHeader, Authorization(OAuth2BearerToken(authToken)))
+        )
+      case None =>
+        HttpRequest(
+          method = manifestRequestHttpMethod,
+          uri = buildManifestUri(dockerHashContext.dockerImageID)
+        )
+    }
 
     (manifestRequest, dockerHashContext)
   }
@@ -255,24 +279,54 @@ abstract class DockerRegistryV2AbstractFlow(httpClientFlow: HttpDockerFlow)(impl
   /**
     * Handles the http response from the manifest request
     */
-  private def manifestResponseHandler(response: HttpResponse, dockerHashContext: DockerHashContext) = {
-    (extractContentDigest(response.headers), dockerHashContext) 
+  private def manifestResponseHandler(response: HttpResponse, dockerHashContext: DockerHashContext) = response match {
+    case httpResponse if httpResponse.status.isSuccess() =>
+      (
+        extractContentDigest(response.headers, dockerHashContext),
+        dockerHashContext
+        )
+    case httpResponse => 
+      (
+        Failure(UnsuccessfulHttpResponseException(httpResponse)),
+        dockerHashContext
+      )
   }
   
   /**
     * Extracts the digest from the response headers
     */
-  private def extractContentDigest(headers: Seq[HttpHeader]) = {
+  private def extractContentDigest(headers: Seq[HttpHeader], dockerHashContext: DockerHashContext) = {
     headers find { _.is(DigestHeaderName) } match {
-      case Some(digestHeader) => Success(DockerHashResponseSuccess(DockerHashResult(digestHeader.value())))
+      case Some(digestHeader) =>
+        DockerHashResult.fromString(digestHeader.value()) map { DockerHashSuccessResponse(_, dockerHashContext.request)}
       case None => Failure(new Exception("Cannot find digest header"))
     }
   }
 
   /**
-    * Wraps a throwable into a DockerHashFailedResponse
+    * Maps a throwable into a DockerHashFailedResponse
     */
   private def throwableToFailureMessage(throwable: Throwable, dockerHashContext: DockerHashContext): (DockerHashResponse, DockerHashContext) = {
-    (DockerHashFailedResponse(throwable, dockerHashContext.dockerImageID), dockerHashContext)
+    throwable match {
+      case UnsuccessfulHttpResponseException(httpResponse) => httpFailureMapper(httpResponse, dockerHashContext)
+      case other => (DockerHashFailedResponse(other, dockerHashContext.request), dockerHashContext)
+    }
+  }
+
+  /**
+    * Maps a failed http response to a DockerHashFailureResponse
+    */
+  protected def httpFailureMapper(response: HttpResponse, dockerHashContext: DockerHashContext) = {
+    response.status match {
+      case StatusCodes.NotFound => (DockerHashNotFound(dockerHashContext.request), dockerHashContext)
+      case StatusCodes.Unauthorized => (DockerHashUnauthorized(dockerHashContext.request), dockerHashContext)
+      case other => (
+        DockerHashFailedResponse(
+          new Exception(s"Docker hash lookup failed with code ${other.intValue()}. ${other.defaultMessage()}"),
+          dockerHashContext.request
+        ),
+        dockerHashContext
+        )
+    }
   }
 }
diff --git a/core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/FlowUtils.scala b/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/FlowUtils.scala
similarity index 89%
rename from core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/FlowUtils.scala
rename to dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/FlowUtils.scala
index 715d7a724..7fa64a6cc 100644
--- a/core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/FlowUtils.scala
+++ b/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/FlowUtils.scala
@@ -1,4 +1,4 @@
-package cromwell.core.callcaching.docker.registryv2.flows
+package cromwell.docker.registryv2.flows
 
 import akka.stream.FanOutShape2
 import akka.stream.scaladsl.{GraphDSL, Partition}
@@ -17,8 +17,8 @@ object FlowUtils {
     import GraphDSL.Implicits._
 
     val partition = builder.add(Partition[(Try[T], U)](2, {
-      case (Success(v), _) => 0
-      case (Failure(f), _) => 1
+      case (Success(_), _) => 0
+      case (Failure(_), _) => 1
     }))
 
     val successOut: PortOps[(T, U)] = partition.out(0) collect {
diff --git a/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/HttpFlowWithRetry.scala b/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/HttpFlowWithRetry.scala
new file mode 100644
index 000000000..5aab9a546
--- /dev/null
+++ b/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/HttpFlowWithRetry.scala
@@ -0,0 +1,166 @@
+package cromwell.docker.registryv2.flows
+
+import akka.NotUsed
+import akka.actor.Scheduler
+import akka.http.scaladsl.model.{HttpRequest, HttpResponse, StatusCodes}
+import akka.stream.{ActorMaterializer, FanOutShape2}
+import akka.stream.javadsl.MergePreferred
+import akka.stream.scaladsl.{Flow, GraphDSL, Partition}
+import cromwell.docker.registryv2.flows.FlowUtils._
+import cromwell.docker.registryv2.flows.HttpFlowWithRetry._
+import cromwell.core.retry.{Backoff, SimpleExponentialBackoff}
+import org.slf4j.LoggerFactory
+
+import scala.concurrent.duration._
+import scala.concurrent.{ExecutionContext, Future}
+import scala.util.Try
+
+object HttpFlowWithRetry {
+  val Logger = LoggerFactory.getLogger("HttpLogger")
+  
+  def isRetryable(response: HttpResponse) = {
+    response.status match {
+      case StatusCodes.InternalServerError => true
+      case StatusCodes.ServiceUnavailable => true
+      case StatusCodes.BadGateway => true
+      case StatusCodes.GatewayTimeout => true
+      case StatusCodes.RequestTimeout => true
+      case other @ _ => isTransient(response)
+    }
+  }
+
+  def isTransient(response: HttpResponse) = {
+    response.status match {
+      case StatusCodes.TooManyRequests => true
+      case _ => false
+    }
+  }
+
+  /**
+    * In order to allow for retries, the http context object needs to encapsulate the original request,
+    * so that it can be re-submitted if necessary.
+    * This type provides this encapsulation by creating a pair of (T, HttpRequest) where T is the custom type that
+    * the user wishes to use as a context for their specific use case.
+    */
+  case class ContextWithRequest[T](userContext: T, request: HttpRequest, currentAttempt: Int, backoff: Backoff) {
+    // Back off is mutable ! This will change every time !
+    def retryIn = backoff.backoffMillis.millis
+    
+    def withNextAttempt = copy(currentAttempt = currentAttempt + 1)
+  }
+
+  /**
+    * Type for a http flow using ContextWithRequest as context type.
+    */
+  type RetryableHttpFlow[T] = Flow[(HttpRequest, ContextWithRequest[T]), (Try[HttpResponse], ContextWithRequest[T]), NotUsed]
+}
+
+/**
+  * Adds additional retry logic around an existing akka http flow.
+  * @param httpClientFlow The akka http flow to use underneath
+  * @param retryBufferSize size of the buffer for requests to be retried. When the buffer is full, requests WILL BE DROPPED
+  *                        Act accordingly by resubmitting requests after a certain amount of time with no response for example.
+  * @tparam T Type of the context data to be passed along with the request
+  * @return A http flow that will retry on request failures and on failed responses as seen fit by the isRetryable method
+  */
+case class HttpFlowWithRetry[T](
+                            httpClientFlow: RetryableHttpFlow[T],
+                            retryBufferSize: Int = 100,
+                            requestBackoff: () => Backoff = () => SimpleExponentialBackoff(1 second, 2 minutes, 3D),
+                            maxAttempts: Int = 3
+                          )(implicit val scheduler: Scheduler, ec: ExecutionContext, mat: ActorMaterializer) {
+  
+  lazy val flow = GraphDSL.create() { implicit builder =>
+    import GraphDSL.Implicits._
+
+    // Http shape
+    val http = builder.add(httpClientFlow)
+    
+    // Wrap the user context into a pair of (flowContext, httpRequest) to allow for retries
+    val source = builder.add(Flow[(HttpRequest, T)] map {
+      case (httpRequest, flowContext) => (httpRequest, ContextWithRequest(flowContext, httpRequest, 1, requestBackoff()))
+    })
+    
+    // Partition Try[HttpResponse] into Success and Failure
+    val partitionResponseTry = builder.add(fanOutTry[HttpResponse, ContextWithRequest[T]])
+    
+    // Those are pairs like: (HttpResponse, Context) 
+    val requestSuccessful = partitionResponseTry.out0
+    
+    // Those are pairs like: (Throwable, Context) 
+    // -> akka-http already retries request if they fail, so we won't retry this and send it to the failure port
+    val requestFailed = partitionResponseTry.out1 map {
+      case (failure, ContextWithRequest(flowContext, _, _, _)) => failure -> flowContext
+    }
+
+    // Splits successful http response into 2 categories:
+    // - out(0) emits terminal http resonse (successful or unsuccessful non-retryable)
+    // - out(1) emits unsusccessful but retryable responses
+    val partitionHttpResponse = builder.add(Partition[(HttpResponse, ContextWithRequest[T])](2, {
+      // Successful return code
+      case (httpResponse, flowContext) if httpResponse.status.isSuccess() || !shouldRetry(httpResponse, flowContext) => 0
+      // Failed return code but retryable
+      case (_, _) => 1
+    }))
+    
+    // Merges requests coming from 3 input ports:
+    // - in(0) takes requests from the outside
+    // - in(1) takes retryable failed responses (with non successful and retryable status code)
+    val mergeRequests = builder.add(MergePreferred.create[(HttpRequest, ContextWithRequest[T])](1))
+
+    // Either success or non retryable unsuccessful http response, either way it won't be retried
+    val terminalResponse = partitionHttpResponse.out(0) map {
+      case (response, ContextWithRequest(flowContext, _, _, _)) => response -> flowContext
+    }
+    
+    // Retryable http response
+    val retryableResponse = partitionHttpResponse.out(1)
+      // Create a new request, yields a future that will complete after the appropriate backoff time
+      .map(toRetryableRequest)
+      // wait for the future to complete, retryBufferSize sets how many retryable future we can wait for in parallel
+      .mapAsync(retryBufferSize)(identity)
+    
+    // Outside requests coming in
+    source ~> mergeRequests.in(0)
+    
+    // Submit request to underlying http flow  -  Partition responses: Failure | Success
+    mergeRequests     ~>     http      ~>         partitionResponseTry.in
+    
+                                                  // Success -> analyze response and partition into 2 (see above)
+                                                  requestSuccessful ~> partitionHttpResponse
+    
+    // Retryable http response are sent back for retry
+    mergeRequests.preferred          <~           retryableResponse.outlet
+
+    new FanOutShape2(source.in, terminalResponse.outlet, requestFailed.outlet)
+  }
+  
+  private def shouldRetry(httpResponse: HttpResponse, contextWithRequest2: ContextWithRequest[T]) = {
+    isRetryable(httpResponse) && contextWithRequest2.currentAttempt <= maxAttempts
+  }
+
+  /**
+    * Create a re-submittable request from a failed retryable
+    * @return a future that will complete after the appropriate backoff time
+    */
+  private def toRetryableRequest(value: (HttpResponse, ContextWithRequest[T])) = value match {
+    case (response, contextWithRequest) => 
+      val nextRetryIn = contextWithRequest.retryIn
+      val nextRequest = (contextWithRequest.request, contextWithRequest.withNextAttempt)
+      // This response will never be consumed by anyone, so discard its content here to avoid pool freeze
+      // http://doc.akka.io/docs/akka-http/10.0.5/scala/http/client-side/request-level.html#using-the-future-based-api-in-actors
+      // https://github.com/akka/akka/issues/19538
+      // https://github.com/akka/akka-http/issues/183
+      // https://github.com/akka/akka-http/issues/117
+      response.discardEntityBytes().future() flatMap { _ =>
+        akka.pattern.after(nextRetryIn, scheduler) {
+          Future.successful(nextRequest)
+        }
+      } recover {
+        case failure =>
+          // Can't do much here except log the error and keep going with the next request
+          Logger.error(s"Failed to discard entity bytes for response $response", failure)
+          nextRequest
+      }
+  }
+}
diff --git a/core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/dockerhub/DockerHubFlow.scala b/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/dockerhub/DockerHubFlow.scala
similarity index 67%
rename from core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/dockerhub/DockerHubFlow.scala
rename to dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/dockerhub/DockerHubFlow.scala
index 3ac567f64..6bc8dc0d6 100644
--- a/core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/dockerhub/DockerHubFlow.scala
+++ b/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/dockerhub/DockerHubFlow.scala
@@ -1,14 +1,14 @@
-package cromwell.core.callcaching.docker.registryv2.flows.dockerhub
+package cromwell.docker.registryv2.flows.dockerhub
 
+import akka.actor.Scheduler
 import akka.http.scaladsl.model.headers.{Authorization, BasicHttpCredentials}
-import akka.http.scaladsl.model.{HttpMethod, HttpMethods}
 import akka.stream.ActorMaterializer
 import cromwell.core.DockerCredentials
-import cromwell.core.callcaching.docker.DockerHashActor.DockerHashContext
-import cromwell.core.callcaching.docker.DockerImageIdentifierWithoutHash
-import cromwell.core.callcaching.docker.registryv2.flows.dockerhub.DockerHubFlow._
-import cromwell.core.callcaching.docker.registryv2.DockerRegistryV2AbstractFlow
-import cromwell.core.callcaching.docker.registryv2.DockerRegistryV2AbstractFlow.HttpDockerFlow
+import cromwell.docker.DockerHashActor.DockerHashContext
+import cromwell.docker.DockerImageIdentifierWithoutHash
+import cromwell.docker.registryv2.DockerRegistryV2AbstractFlow
+import cromwell.docker.registryv2.DockerRegistryV2AbstractFlow.HttpDockerFlow
+import cromwell.docker.registryv2.flows.dockerhub.DockerHubFlow._
 
 import scala.concurrent.ExecutionContext
 
@@ -19,8 +19,8 @@ object DockerHubFlow {
   private val ServiceName = Option("registry.docker.io")
 }
 
-class DockerHubFlow(httpClientFlow: HttpDockerFlow)(implicit ec: ExecutionContext, materializer: ActorMaterializer) 
-  extends DockerRegistryV2AbstractFlow(httpClientFlow)(ec, materializer) {
+class DockerHubFlow(httpClientFlow: HttpDockerFlow)(implicit ec: ExecutionContext, materializer: ActorMaterializer, scheduler: Scheduler) 
+  extends DockerRegistryV2AbstractFlow(httpClientFlow)(ec, materializer, scheduler) {
 
   override val registryHostName = RegistryHostName
   override val authorizationServerHostName = AuthorizationServerHostName
@@ -44,6 +44,4 @@ class DockerHubFlow(httpClientFlow: HttpDockerFlow)(implicit ec: ExecutionContex
       case _ => false
     }
   }
-
-   override def manifestRequestHttpMethod: HttpMethod = HttpMethods.HEAD
 }
diff --git a/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/gcr/GcrAbstractFlow.scala b/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/gcr/GcrAbstractFlow.scala
new file mode 100644
index 000000000..2982c4a86
--- /dev/null
+++ b/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/gcr/GcrAbstractFlow.scala
@@ -0,0 +1,42 @@
+package cromwell.docker.registryv2.flows.gcr
+
+import akka.actor.Scheduler
+import akka.http.scaladsl.model.headers.{Authorization, OAuth2BearerToken}
+import akka.stream.ActorMaterializer
+import com.google.auth.oauth2.{AccessToken, OAuth2Credentials}
+import cromwell.docker.DockerHashActor.DockerHashContext
+import cromwell.docker.registryv2.DockerRegistryV2AbstractFlow
+import cromwell.docker.registryv2.DockerRegistryV2AbstractFlow.HttpDockerFlow
+
+import scala.concurrent.ExecutionContext
+import scala.concurrent.duration._
+
+abstract class GcrAbstractFlow(httpClientFlow: HttpDockerFlow, host: String)(implicit ec: ExecutionContext, materializer: ActorMaterializer, scheduler: Scheduler) extends DockerRegistryV2AbstractFlow(httpClientFlow)(ec, materializer, scheduler) {
+  
+  private val AccessTokenAcceptableTTL = 1.minute
+  
+  override val registryHostName = host
+  override val authorizationServerHostName = s"$host/v2"
+  
+  /**
+    * Builds the list of headers for the token request
+    */
+   def buildTokenRequestHeaders(dockerHashContext: DockerHashContext) = {
+    dockerHashContext.credentials collect {
+      case credentials: OAuth2Credentials => Authorization(OAuth2BearerToken(freshAccessToken(credentials)))
+    }
+  }
+  
+  private def freshAccessToken(credential: OAuth2Credentials) = {
+    def accessTokenTTLIsAcceptable(accessToken: AccessToken) = {
+      (accessToken.getExpirationTime.getTime - System.currentTimeMillis()).millis.gteq(AccessTokenAcceptableTTL)
+    }
+    
+    Option(credential.getAccessToken) match {
+      case Some(accessToken) if accessTokenTTLIsAcceptable(accessToken) => accessToken.getTokenValue
+      case _ =>
+        credential.refresh()
+        credential.getAccessToken.getTokenValue
+    }
+  }
+}
diff --git a/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/gcr/GcrEuFlow.scala b/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/gcr/GcrEuFlow.scala
new file mode 100644
index 000000000..f51c1c0a5
--- /dev/null
+++ b/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/gcr/GcrEuFlow.scala
@@ -0,0 +1,9 @@
+package cromwell.docker.registryv2.flows.gcr
+
+import akka.actor.Scheduler
+import akka.stream.ActorMaterializer
+import cromwell.docker.registryv2.DockerRegistryV2AbstractFlow.HttpDockerFlow
+
+import scala.concurrent.ExecutionContext
+
+private class GcrEuFlow(httpClientFlow: HttpDockerFlow)(implicit ec: ExecutionContext, materializer: ActorMaterializer, scheduler: Scheduler) extends GcrAbstractFlow(httpClientFlow, "eu.gcr.io")
diff --git a/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/gcr/GcrFlow.scala b/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/gcr/GcrFlow.scala
new file mode 100644
index 000000000..1d4ba17c9
--- /dev/null
+++ b/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/gcr/GcrFlow.scala
@@ -0,0 +1,9 @@
+package cromwell.docker.registryv2.flows.gcr
+
+import akka.actor.Scheduler
+import akka.stream.ActorMaterializer
+import cromwell.docker.registryv2.DockerRegistryV2AbstractFlow.HttpDockerFlow
+
+import scala.concurrent.ExecutionContext
+
+private class GcrFlow(httpClientFlow: HttpDockerFlow)(implicit ec: ExecutionContext, materializer: ActorMaterializer, scheduler: Scheduler) extends GcrAbstractFlow(httpClientFlow, "gcr.io")
diff --git a/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/gcr/GcrUsFlow.scala b/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/gcr/GcrUsFlow.scala
new file mode 100644
index 000000000..0194913fa
--- /dev/null
+++ b/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/gcr/GcrUsFlow.scala
@@ -0,0 +1,9 @@
+package cromwell.docker.registryv2.flows.gcr
+
+import akka.actor.Scheduler
+import akka.stream.ActorMaterializer
+import cromwell.docker.registryv2.DockerRegistryV2AbstractFlow.HttpDockerFlow
+
+import scala.concurrent.ExecutionContext
+
+private class GcrUsFlow(httpClientFlow: HttpDockerFlow)(implicit ec: ExecutionContext, materializer: ActorMaterializer, scheduler: Scheduler) extends GcrAbstractFlow(httpClientFlow, "us.gcr.io")
diff --git a/core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/gcr/GoogleFlow.scala b/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/gcr/GoogleFlow.scala
similarity index 81%
rename from core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/gcr/GoogleFlow.scala
rename to dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/gcr/GoogleFlow.scala
index d66bf8eba..94c00a549 100644
--- a/core/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/gcr/GoogleFlow.scala
+++ b/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/gcr/GoogleFlow.scala
@@ -1,16 +1,17 @@
-package cromwell.core.callcaching.docker.registryv2.flows.gcr
+package cromwell.docker.registryv2.flows.gcr
 
+import akka.actor.Scheduler
 import akka.stream.scaladsl.{Flow, GraphDSL, Merge, Partition}
 import akka.stream.{ActorMaterializer, FlowShape, ThrottleMode}
-import cromwell.core.callcaching.docker.DockerHashActor.{DockerHashContext, DockerHashResponse, DockerHashUnknownRegistry}
-import cromwell.core.callcaching.docker.registryv2.DockerRegistryV2AbstractFlow.HttpDockerFlow
-import cromwell.core.callcaching.docker.{DockerFlow, DockerImageIdentifierWithoutHash}
+import cromwell.docker.DockerHashActor.{DockerHashContext, DockerHashResponse, DockerHashUnknownRegistry}
+import cromwell.docker.registryv2.DockerRegistryV2AbstractFlow.HttpDockerFlow
+import cromwell.docker.{DockerFlow, DockerImageIdentifierWithoutHash}
 
 import scala.concurrent.ExecutionContext
 import scala.concurrent.duration._
 import scala.language.postfixOps
 
-class GoogleFlow(httpClientFlow: HttpDockerFlow, queriesPer100Sec: Int)(implicit ec: ExecutionContext, materializer: ActorMaterializer) extends DockerFlow {
+class GoogleFlow(httpClientFlow: HttpDockerFlow, queriesPer100Sec: Int)(implicit ec: ExecutionContext, materializer: ActorMaterializer, scheduler: Scheduler) extends DockerFlow {
   
   private val gcrFlow = new GcrFlow(httpClientFlow)
   private val usGcrFlow = new GcrUsFlow(httpClientFlow)
@@ -35,7 +36,7 @@ class GoogleFlow(httpClientFlow: HttpDockerFlow, queriesPer100Sec: Int)(implicit
     // If we ever get an image that none of the above flow can process,
     // which shouldn't happen because the caller should have called "accepts" before
     // to make sure this flow can handle the image
-    val failFlow = builder.add(Flow[DockerHashContext].map(dockerContext => (DockerHashUnknownRegistry(dockerContext.dockerImageID), dockerContext)))
+    val failFlow = builder.add(Flow[DockerHashContext].map(dockerContext => (DockerHashUnknownRegistry(dockerContext.request), dockerContext)))
     
     val partition = builder.add(Partition[DockerHashContext](googleFlows.length + 1, mapContextToFlow))
     val merge = builder.add(Merge[(DockerHashResponse, DockerHashContext)](googleFlows.length + 1))
diff --git a/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/quay/QuayFlow.scala b/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/quay/QuayFlow.scala
new file mode 100644
index 000000000..113ba5ca8
--- /dev/null
+++ b/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/quay/QuayFlow.scala
@@ -0,0 +1,28 @@
+package cromwell.docker.registryv2.flows.quay
+
+import akka.actor.Scheduler
+import akka.http.scaladsl.model.HttpHeader
+import akka.stream.scaladsl.{Flow, GraphDSL, Source}
+import akka.stream.{ActorMaterializer, FanOutShape2}
+import cromwell.docker.DockerHashActor.{DockerHashContext, DockerHashResponse}
+import cromwell.docker.registryv2.DockerRegistryV2AbstractFlow
+import cromwell.docker.registryv2.DockerRegistryV2AbstractFlow.HttpDockerFlow
+
+import scala.concurrent.ExecutionContext
+
+class QuayFlow(httpClientFlow: HttpDockerFlow)(implicit ec: ExecutionContext, materializer: ActorMaterializer, scheduler: Scheduler) extends DockerRegistryV2AbstractFlow(httpClientFlow) {
+  override protected def registryHostName: String = "quay.io"
+  // Not used for now because we bypass the token part as quay doesn't require one for public images 
+  override protected def authorizationServerHostName: String = "quay.io"
+  // Not used for now, same reason as above
+  override protected def buildTokenRequestHeaders(dockerHashContext: DockerHashContext): List[HttpHeader] = List.empty
+
+  override private [registryv2] val tokenFlow = GraphDSL.create() { implicit b =>
+    // Flow that always returns a successful empty token
+    val noTokenFlow = b.add(Flow[DockerHashContext].map((None, _)))
+    // this token flow never fails as it returns an empty token. 
+    // However it still needs a failure port. This creates an empty source that will act as the failure output port.
+    val emptyFailurePort = b.add(Source.empty[(DockerHashResponse, DockerHashContext)])
+    new FanOutShape2(noTokenFlow.in, noTokenFlow.out, emptyFailurePort.out)
+  }
+}
diff --git a/dockerHashing/src/test/scala/cromwell/docker/DockerEmptyFlowSpec.scala b/dockerHashing/src/test/scala/cromwell/docker/DockerEmptyFlowSpec.scala
new file mode 100644
index 000000000..621dffc37
--- /dev/null
+++ b/dockerHashing/src/test/scala/cromwell/docker/DockerEmptyFlowSpec.scala
@@ -0,0 +1,45 @@
+package cromwell.docker
+
+import cromwell.docker.DockerHashActor.DockerHashUnknownRegistry
+import org.scalatest.{FlatSpecLike, Matchers}
+
+import scala.concurrent.duration._
+
+class DockerEmptyFlowSpec extends DockerFlowSpec("DockerEmptyFlowSpec") with FlatSpecLike with Matchers {
+  behavior of "An empty docker flow"
+
+  override protected def registryFlows: Seq[DockerFlow] = Seq()
+
+  it should "send an unrecognized host message back for a public docker hash" in {
+    dockerActor ! makeRequest("ubuntu:latest")
+
+    expectMsgClass(5.seconds, classOf[DockerHashUnknownRegistry])
+  }
+
+  it should "send an unrecognized host message back for a public docker on gcr" in {
+    dockerActor ! makeRequest("gcr.io/google-containers/alpine-with-bash:1.0")
+
+    expectMsgClass(5.seconds, classOf[DockerHashUnknownRegistry])
+  }
+
+  it should "send an unrecognized host message back if the image does not exist" in {
+    val notFound = makeRequest("ubuntu:nonexistingtag")
+    dockerActor ! notFound
+
+    expectMsgClass(5.seconds, classOf[DockerHashUnknownRegistry])
+  }
+
+  it should "send an unrecognized host message back if the user doesn't have permission on this image" in {
+    val unauthorized = makeRequest("tjeandet/sinatra:v1")
+    dockerActor ! unauthorized
+
+    expectMsgClass(5.seconds, classOf[DockerHashUnknownRegistry])
+  }
+
+  it should "send an unrecognized host message back for an unrecognized host" in {
+    val unauthorized = makeRequest("unknown.io/image:v1")
+    dockerActor ! unauthorized
+
+    expectMsgClass(5.seconds, classOf[DockerHashUnknownRegistry])
+  }
+}
diff --git a/dockerHashing/src/test/scala/cromwell/docker/DockerFlowSpec.scala b/dockerHashing/src/test/scala/cromwell/docker/DockerFlowSpec.scala
new file mode 100644
index 000000000..afe9cd556
--- /dev/null
+++ b/dockerHashing/src/test/scala/cromwell/docker/DockerFlowSpec.scala
@@ -0,0 +1,36 @@
+package cromwell.docker
+
+import akka.http.scaladsl.Http
+import akka.stream.ActorMaterializer
+import akka.testkit.ImplicitSender
+import cromwell.core.TestKitSuite
+import cromwell.docker.DockerHashActor._
+import cromwell.docker.registryv2.flows.HttpFlowWithRetry.ContextWithRequest
+
+import scala.concurrent.duration._
+
+abstract class DockerFlowSpec(actorSystemName: String) extends TestKitSuite(actorSystemName) with ImplicitSender {
+  implicit val materializer = ActorMaterializer()
+  implicit val ex = system.dispatcher
+  implicit val scheduler = system.scheduler
+
+  lazy val httpPool = Http().superPool[ContextWithRequest[DockerHashContext]]()
+
+  protected def registryFlows: Seq[DockerFlow]
+
+  // Disable cache by setting a cache size of 0 - A separate test tests the cache
+  lazy val dockerActor = system.actorOf(DockerHashActor.props(registryFlows, 1000, 20.minutes, 0)(materializer))
+
+  def dockerImage(string: String) = DockerImageIdentifier.fromString(string).get.asInstanceOf[DockerImageIdentifierWithoutHash]
+
+  def makeRequest(string: String) = {
+    DockerHashRequest(dockerImage(string))
+  }
+
+  override protected def afterAll() = {
+    system.stop(dockerActor)
+    Http().shutdownAllConnectionPools()
+    materializer.shutdown()
+    super.afterAll()
+  }
+}
diff --git a/core/src/test/scala/cromwell/core/callcaching/docker/DockerHashActorSpec.scala b/dockerHashing/src/test/scala/cromwell/docker/DockerHashActorSpec.scala
similarity index 64%
rename from core/src/test/scala/cromwell/core/callcaching/docker/DockerHashActorSpec.scala
rename to dockerHashing/src/test/scala/cromwell/docker/DockerHashActorSpec.scala
index a7676e18d..f6d2b6958 100644
--- a/core/src/test/scala/cromwell/core/callcaching/docker/DockerHashActorSpec.scala
+++ b/dockerHashing/src/test/scala/cromwell/docker/DockerHashActorSpec.scala
@@ -1,49 +1,24 @@
-package cromwell.core.callcaching.docker
+package cromwell.docker
 
-import akka.http.scaladsl.Http
-import akka.http.scaladsl.model.HttpRequest
-import akka.stream.ActorMaterializer
-import akka.testkit.ImplicitSender
 import cromwell.core.Tags.IntegrationTest
-import cromwell.core.TestKitSuite
-import cromwell.core.callcaching.docker.DockerHashActor._
-import cromwell.core.callcaching.docker.registryv2.flows.dockerhub.DockerHubFlow
-import cromwell.core.callcaching.docker.registryv2.flows.gcr.GoogleFlow
+import cromwell.docker.DockerHashActor._
+import cromwell.docker.registryv2.flows.dockerhub.DockerHubFlow
+import cromwell.docker.registryv2.flows.gcr.GoogleFlow
 import org.scalatest.{FlatSpecLike, Matchers}
 
 import scala.concurrent.duration._
 import scala.language.postfixOps
 
-class DockerHashActorSpec extends TestKitSuite with FlatSpecLike with Matchers with ImplicitSender {
-
+class DockerHashActorSpec extends DockerFlowSpec("DockerHashActorSpec") with FlatSpecLike with Matchers {
   behavior of "DockerRegistryActor"
-  
-  implicit val materializer = ActorMaterializer()
-  implicit val ex = system.dispatcher
-  
-  val httpPool = Http().superPool[(DockerHashContext, HttpRequest)]()
-  val registryFlows = Seq(new DockerHubFlow(httpPool), new GoogleFlow(httpPool, 50000))
-  // Disable cache by setting a cache size of 0 - A separate test tests the cache
-  val dockerActor = system.actorOf(DockerHashActor.props(registryFlows, 1000, 20 minutes, 0)(materializer))
-  
-  private def dockerImage(string: String) = DockerImageIdentifier.fromString(string).get.asInstanceOf[DockerImageIdentifierWithoutHash]
-  
-  private def makeRequest(string: String) = {
-    DockerHashRequest(dockerImage(string))
-  }
 
-  override protected def afterAll() = {
-    system.stop(dockerActor)
-    Http().shutdownAllConnectionPools()
-    materializer.shutdown()
-    super.afterAll()
-  }
-  
+  override protected lazy val registryFlows = Seq(new DockerHubFlow(httpPool), new GoogleFlow(httpPool, 50000))
+
   it should "retrieve a public docker hash" taggedAs IntegrationTest in {
     dockerActor ! makeRequest("ubuntu:latest")
     
     expectMsgPF(5 second) {
-      case DockerHashResponseSuccess(DockerHashResult(alg, hash)) => 
+      case DockerHashSuccessResponse(DockerHashResult(alg, hash), _) => 
         alg shouldBe "sha256"
         hash should not be empty
     }
@@ -53,7 +28,7 @@ class DockerHashActorSpec extends TestKitSuite with FlatSpecLike with Matchers w
     dockerActor ! makeRequest("gcr.io/google-containers/alpine-with-bash:1.0")
 
     expectMsgPF(5 second) {
-      case DockerHashResponseSuccess(DockerHashResult(alg, hash)) =>
+      case DockerHashSuccessResponse(DockerHashResult(alg, hash), _) =>
         alg shouldBe "sha256"
         hash should not be empty
     }
@@ -81,14 +56,15 @@ class DockerHashActorSpec extends TestKitSuite with FlatSpecLike with Matchers w
   }
   
   it should "cache results" in {
-    val hashSuccess = DockerHashResult("sha256", "hashvalue")
-    val responseSuccess = DockerHashResponseSuccess(hashSuccess)
-    val mockResponseSuccess = MockHashResponse(responseSuccess, 1)
     
     val image1 = dockerImage("ubuntu:latest")
     val request = DockerHashRequest(image1)
     
-    val responseFailure = DockerHashFailedResponse(new Exception("Docker hash failed - part of test flow"), image1)
+    val hashSuccess = DockerHashResult("sha256", "hashvalue")
+    val responseSuccess = DockerHashSuccessResponse(hashSuccess, request)
+    val mockResponseSuccess = MockHashResponse(responseSuccess, 1)
+    
+    val responseFailure = DockerHashFailedResponse(new Exception("Docker hash failed - part of test flow"), request)
     val mockResponseFailure = MockHashResponse(responseFailure, 1)
     
     // Send back success, failure, success, failure, ...
@@ -96,7 +72,7 @@ class DockerHashActorSpec extends TestKitSuite with FlatSpecLike with Matchers w
     val dockerActorWithCache = system.actorOf(DockerHashActor.props(Seq(mockHttpFlow), 1000, 3 seconds, 10)(materializer))
     
     dockerActorWithCache ! request
-    expectMsg(DockerHashResponseSuccess(hashSuccess))
+    expectMsg(DockerHashSuccessResponse(hashSuccess, request))
     // Necessary to give some time to the cache to be updated - as it's decoupled from sending back the response
     Thread.sleep(1000)
     
@@ -113,5 +89,16 @@ class DockerHashActorSpec extends TestKitSuite with FlatSpecLike with Matchers w
     expectMsg(responseFailure)
     mockHttpFlow.count() shouldBe 2
   }
+
+
+  it should "not deadlock" taggedAs IntegrationTest in {
+    lazy val dockerActorScale = system.actorOf(DockerHashActor.props(registryFlows, 1000, 20.minutes, 0)(materializer))
+    0 until 400 foreach { _ =>
+      dockerActorScale ! makeRequest("gcr.io/google-containers/alpine-with-bash:1.0")
+    }
+
+    val received = receiveN(400, 1 minute)
+    received foreach { _ shouldBe a[DockerHashSuccessResponse] }
+  }
   
 }
diff --git a/core/src/test/scala/cromwell/core/callcaching/docker/DockerHashMocks.scala b/dockerHashing/src/test/scala/cromwell/docker/DockerHashMocks.scala
similarity index 80%
rename from core/src/test/scala/cromwell/core/callcaching/docker/DockerHashMocks.scala
rename to dockerHashing/src/test/scala/cromwell/docker/DockerHashMocks.scala
index ee86cc49e..805fe4401 100644
--- a/core/src/test/scala/cromwell/core/callcaching/docker/DockerHashMocks.scala
+++ b/dockerHashing/src/test/scala/cromwell/docker/DockerHashMocks.scala
@@ -1,8 +1,9 @@
-package cromwell.core.callcaching.docker
+package cromwell.docker
 
 import akka.http.scaladsl.model._
 import akka.stream.scaladsl.Flow
-import cromwell.core.callcaching.docker.DockerHashActor.{DockerHashContext, DockerHashResponse}
+import cromwell.docker.DockerHashActor.{DockerHashContext, DockerHashResponse}
+import cromwell.docker.registryv2.flows.HttpFlowWithRetry.ContextWithRequest
 
 import scala.util.Try
 
@@ -11,8 +12,8 @@ case class MockHttpResponse(httpResponse: Try[HttpResponse], nb: Int)
 class HttpMock[T](responses: MockHttpResponse*) {
   private var responsesLeft = responses.toBuffer
   
-  private def nextResponse(value: (HttpRequest, (T, HttpRequest))): (Try[HttpResponse], (T, HttpRequest)) = value match {
-    case (request, context) =>
+  private def nextResponse(value: (HttpRequest, ContextWithRequest[T])): (Try[HttpResponse], ContextWithRequest[T]) = value match {
+    case (request @ _, context) =>
       responsesLeft.headOption match {
         case Some(mockResponse) =>
           if (mockResponse.nb > 1)
@@ -27,7 +28,7 @@ class HttpMock[T](responses: MockHttpResponse*) {
       }
   }
   
-  def httpMock() = Flow[(HttpRequest, (T, HttpRequest))] map nextResponse
+  def httpMock() = Flow[(HttpRequest, ContextWithRequest[T])] map nextResponse
 }
 
 case class MockHashResponse(hashResponse: DockerHashResponse, nb: Int)
diff --git a/core/src/test/scala/cromwell/core/callcaching/docker/DockerImageIdentifierSpec.scala b/dockerHashing/src/test/scala/cromwell/docker/DockerImageIdentifierSpec.scala
similarity index 62%
rename from core/src/test/scala/cromwell/core/callcaching/docker/DockerImageIdentifierSpec.scala
rename to dockerHashing/src/test/scala/cromwell/docker/DockerImageIdentifierSpec.scala
index 9d0fb6218..649e16930 100644
--- a/core/src/test/scala/cromwell/core/callcaching/docker/DockerImageIdentifierSpec.scala
+++ b/dockerHashing/src/test/scala/cromwell/docker/DockerImageIdentifierSpec.scala
@@ -1,4 +1,4 @@
-package cromwell.core.callcaching.docker
+package cromwell.docker
 
 import org.scalatest.prop.TableDrivenPropertyChecks
 import org.scalatest.{FlatSpec, Matchers}
@@ -8,19 +8,17 @@ class DockerImageIdentifierSpec extends FlatSpec with Matchers with TableDrivenP
   
   it should "parse valid docker images" in {
     val valid = Table(
-      ("sourceString",                            "host",              "repo",          "image",     "reference"),
-      
+      ("sourceString",                            "host",             "repo",                 "image",     "reference"),
       // Without tags -> latest
-      ("ubuntu",                                  None,               "library",        "ubuntu",     "latest"),
-      ("broad/cromwell",                          None,               "broad",          "cromwell",   "latest"),
-      ("index.docker.io/ubuntu",         Option("index.docker.io"),   "library",        "ubuntu",     "latest"),
-      ("broad/cromwell/submarine",                None,               "broad/cromwell", "submarine",  "latest"),
-      ("gcr.io/google/alpine",              Option("gcr.io"),         "google",         "alpine",     "latest"),
-
+      ("ubuntu",                                  None,               None,                   "ubuntu",     "latest"),
+      ("broad/cromwell",                          None,               Some("broad"),          "cromwell",   "latest"),
+      ("index.docker.io/ubuntu",         Option("index.docker.io"),   None,                   "ubuntu",     "latest"),
+      ("broad/cromwell/submarine",                None,               Some("broad/cromwell"), "submarine",  "latest"),
+      ("gcr.io/google/alpine",              Option("gcr.io"),         Some("google"),         "alpine",     "latest"),
       // With tags
-      ("ubuntu:latest",                           None,               "library",        "ubuntu",     "latest"),
-      ("ubuntu:1235-SNAP",                        None,               "library",        "ubuntu",     "1235-SNAP"),
-      ("ubuntu:V3.8-5_1",                         None,               "library",        "ubuntu",     "V3.8-5_1")
+      ("ubuntu:latest",                           None,               None,                   "ubuntu",     "latest"),
+      ("ubuntu:1235-SNAP",                        None,               None,                   "ubuntu",     "1235-SNAP"),
+      ("ubuntu:V3.8-5_1",                         None,               None,                   "ubuntu",     "V3.8-5_1")
     )
     
     forAll(valid) { (dockerString, host, repo, image, reference) =>
@@ -38,11 +36,22 @@ class DockerImageIdentifierSpec extends FlatSpec with Matchers with TableDrivenP
     withDigest.isSuccess shouldBe true
     val successfulDigest = withDigest.get
     successfulDigest.host shouldBe None
-    successfulDigest.repository shouldBe "library"
+    successfulDigest.repository shouldBe None
     successfulDigest.image shouldBe "ubuntu"
     successfulDigest.reference shouldBe "sha256:45168651"
     successfulDigest.isInstanceOf[DockerImageIdentifierWithHash] shouldBe true
     successfulDigest.asInstanceOf[DockerImageIdentifierWithHash].hash shouldBe DockerHashResult("sha256", "45168651")
+
+    // With tag + digest
+    val withTagDigest = DockerImageIdentifier.fromString("ubuntu:latest@sha256:45168651")
+    withTagDigest.isSuccess shouldBe true
+    val successfulTagDigest = withTagDigest.get
+    successfulTagDigest.host shouldBe None
+    successfulTagDigest.repository shouldBe None
+    successfulTagDigest.image shouldBe "ubuntu"
+    successfulTagDigest.reference shouldBe "latest@sha256:45168651"
+    successfulTagDigest.isInstanceOf[DockerImageIdentifierWithHash] shouldBe true
+    successfulTagDigest.asInstanceOf[DockerImageIdentifierWithHash].hash shouldBe DockerHashResult("sha256", "45168651")
   }
 
   it should "not parse invalid docker images" in {
@@ -51,7 +60,9 @@ class DockerImageIdentifierSpec extends FlatSpec with Matchers with TableDrivenP
       "NotValid:latest",
       "not:_valid",
       "/not:valid",
-      "not%valid"
+      "not%valid",
+      "not@sha256:digest:tag",
+      "not:sha256:digest@tag"
     )
     
     invalid foreach { image =>
diff --git a/dockerHashing/src/test/scala/cromwell/docker/local/DockerCliClientSpec.scala b/dockerHashing/src/test/scala/cromwell/docker/local/DockerCliClientSpec.scala
new file mode 100644
index 000000000..a9ce1b9fa
--- /dev/null
+++ b/dockerHashing/src/test/scala/cromwell/docker/local/DockerCliClientSpec.scala
@@ -0,0 +1,54 @@
+package cromwell.docker.local
+
+import org.scalatest.prop.TableDrivenPropertyChecks
+import org.scalatest.{FlatSpecLike, Matchers}
+
+import scala.util.{Failure, Success}
+
+class DockerCliClientSpec extends FlatSpecLike with Matchers with TableDrivenPropertyChecks {
+  behavior of "DockerCliClient"
+
+  private val lookupSuccessStdout = Seq(
+    "<none>\t<none>\t<none>",
+    "fauxbuntu\tlatest\tsha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
+    "fauxbuntu\tmytag\tsha256:00001111222233334444555566667777888899990000aaaabbbbccccddddeeee")
+
+  private val lookupSuccessHashes = Table(
+    ("dockerCliKey", "hashValue"),
+    (DockerCliKey("fauxbuntu", "latest"), "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"),
+    (DockerCliKey("fauxbuntu", "mytag"), "00001111222233334444555566667777888899990000aaaabbbbccccddddeeee"))
+
+  forAll(lookupSuccessHashes) { (dockerCliKey, hashValue) =>
+    it should s"successfully lookup simulated hash for ${dockerCliKey.fullName}" in {
+      val client = new TestDockerCliClient(DockerCliResult(0, lookupSuccessStdout, Nil))
+      val response = client.lookupHash(dockerCliKey)
+      response should be(Success(Option(s"sha256:$hashValue")))
+    }
+  }
+
+  val lookupFailStderr = Seq("Error response from daemon: Bad response from Docker engine")
+
+  it should "return not found for simulated hash fauxbuntu:unknowntag" in {
+    val client = new TestDockerCliClient(DockerCliResult(0, lookupSuccessStdout, Nil))
+    val dockerCliKey = DockerCliKey("fauxbuntu", "unknowntag")
+    val response = client.lookupHash(dockerCliKey)
+    response should be(Success(None))
+  }
+
+  it should "return expected error messages" in {
+    val client = new TestDockerCliClient(DockerCliResult(1, Nil, lookupFailStderr))
+    val dockerCliKey = DockerCliKey("fauxbuntu", "dockernotrunning")
+    val response = client.lookupHash(dockerCliKey)
+    response should be(a[Failure[_]])
+    val exception = response.asInstanceOf[Failure[_]].exception
+    exception.getMessage should be(
+      """|Error running: docker images --digests --format {{printf "%s\t%s\t%s" .Repository .Tag .Digest}}
+         |Exit code: 1
+         |Error response from daemon: Bad response from Docker engine
+         |""".stripMargin)
+  }
+}
+
+class TestDockerCliClient(dockerCliResult: DockerCliResult) extends DockerCliClient {
+  override private[local] def run(cmd: Seq[String]) = dockerCliResult
+}
diff --git a/dockerHashing/src/test/scala/cromwell/docker/local/DockerCliFlowSpec.scala b/dockerHashing/src/test/scala/cromwell/docker/local/DockerCliFlowSpec.scala
new file mode 100644
index 000000000..9d0777f87
--- /dev/null
+++ b/dockerHashing/src/test/scala/cromwell/docker/local/DockerCliFlowSpec.scala
@@ -0,0 +1,55 @@
+package cromwell.docker.local
+
+import cromwell.core.Tags.IntegrationTest
+import cromwell.docker.DockerHashActor.{DockerHashNotFound, DockerHashSuccessResponse}
+import cromwell.docker.{DockerFlow, DockerFlowSpec, DockerHashResult}
+import org.scalatest.{FlatSpecLike, Matchers}
+
+import scala.concurrent.duration._
+
+class DockerCliFlowSpec extends DockerFlowSpec("DockerCliFlowSpec") with FlatSpecLike with Matchers {
+  behavior of "DockerCliFlow"
+
+  override protected def registryFlows: Seq[DockerFlow] = Seq(new DockerCliFlow)
+
+  it should "retrieve a public docker hash" taggedAs IntegrationTest in {
+    dockerActor ! makeRequest("ubuntu:latest")
+
+    expectMsgPF(30.seconds) {
+      case DockerHashSuccessResponse(DockerHashResult(alg, hash), _) =>
+        alg shouldBe "sha256"
+        hash should not be empty
+    }
+  }
+
+  it should "retrieve a public docker hash on gcr" taggedAs IntegrationTest in {
+    dockerActor ! makeRequest("gcr.io/google-containers/alpine-with-bash:1.0")
+
+    expectMsgPF(30.seconds) {
+      case DockerHashSuccessResponse(DockerHashResult(alg, hash), _) =>
+        alg shouldBe "sha256"
+        hash should not be empty
+    }
+  }
+
+  it should "send image not found message back if the image does not exist" taggedAs IntegrationTest in {
+    val notFound = makeRequest("ubuntu:nonexistingtag")
+    dockerActor ! notFound
+
+    expectMsgClass(5.seconds, classOf[DockerHashNotFound])
+  }
+
+  it should "send image not found if the user doesn't have permission on this image" taggedAs IntegrationTest in {
+    val unauthorized = makeRequest("tjeandet/sinatra:v1")
+    dockerActor ! unauthorized
+
+    expectMsgClass(5.seconds, classOf[DockerHashNotFound])
+  }
+
+  it should "send image not found for an unrecognized host" taggedAs IntegrationTest in {
+    val unauthorized = makeRequest("unknown.io/image:v1")
+    dockerActor ! unauthorized
+
+    expectMsgClass(5.seconds, classOf[DockerHashNotFound])
+  }
+}
diff --git a/dockerHashing/src/test/scala/cromwell/docker/local/DockerCliTimeoutFlowSpec.scala b/dockerHashing/src/test/scala/cromwell/docker/local/DockerCliTimeoutFlowSpec.scala
new file mode 100644
index 000000000..383b06966
--- /dev/null
+++ b/dockerHashing/src/test/scala/cromwell/docker/local/DockerCliTimeoutFlowSpec.scala
@@ -0,0 +1,81 @@
+package cromwell.docker.local
+
+import cromwell.core.Tags.IntegrationTest
+import cromwell.docker.DockerHashActor.DockerHashFailedResponse
+import cromwell.docker.{DockerFlow, DockerFlowSpec}
+import org.scalatest.{FlatSpecLike, Matchers}
+
+import scala.concurrent.TimeoutException
+import scala.concurrent.duration._
+
+class DockerCliTimeoutFlowSpec extends DockerFlowSpec("DockerCliTimeoutFlowSpec") with FlatSpecLike with Matchers {
+  behavior of "A DockerCliFlow that times out"
+
+  override protected def registryFlows: Seq[DockerFlow] = Seq(new DockerCliFlow {
+    override lazy val firstLookupTimeout = 0.seconds
+  })
+
+  it should "timeout retrieving a public docker hash" taggedAs IntegrationTest in {
+    dockerActor ! makeRequest("ubuntu:latest")
+
+    expectMsgPF(5.seconds) {
+      case DockerHashFailedResponse(exception: TimeoutException, _) =>
+        exception.getMessage should be(
+          """|Timeout while looking up hash of ubuntu:latest.
+             |Ensure that docker is running correctly.
+             |""".stripMargin)
+    }
+  }
+
+  it should "timeout retrieving a public docker hash on gcr" taggedAs IntegrationTest in {
+    dockerActor ! makeRequest("gcr.io/google-containers/alpine-with-bash:1.0")
+
+    expectMsgPF(5.seconds) {
+      case DockerHashFailedResponse(exception: TimeoutException, _) =>
+        exception.getMessage should be(
+          """|Timeout while looking up hash of gcr.io/google-containers/alpine-with-bash:1.0.
+             |Ensure that docker is running correctly.
+             |""".stripMargin)
+    }
+
+  }
+
+  it should "timeout retrieving an image that does not exist" taggedAs IntegrationTest in {
+    val notFound = makeRequest("ubuntu:nonexistingtag")
+    dockerActor ! notFound
+
+    expectMsgPF(5.seconds) {
+      case DockerHashFailedResponse(exception: TimeoutException, _) =>
+        exception.getMessage should be(
+          """|Timeout while looking up hash of ubuntu:nonexistingtag.
+             |Ensure that docker is running correctly.
+             |""".stripMargin)
+    }
+  }
+
+  it should "timeout retrieving an image if the user doesn't have permission" taggedAs IntegrationTest in {
+    val unauthorized = makeRequest("tjeandet/sinatra:v1")
+    dockerActor ! unauthorized
+
+    expectMsgPF(5.seconds) {
+      case DockerHashFailedResponse(exception: TimeoutException, _) =>
+        exception.getMessage should be(
+          """|Timeout while looking up hash of tjeandet/sinatra:v1.
+             |Ensure that docker is running correctly.
+             |""".stripMargin)
+    }
+  }
+
+  it should "timeout retrieving an image for an unrecognized host" taggedAs IntegrationTest in {
+    val unauthorized = makeRequest("unknown.io/image:v1")
+    dockerActor ! unauthorized
+
+    expectMsgPF(5.seconds) {
+      case DockerHashFailedResponse(exception: TimeoutException, _) =>
+        exception.getMessage should be(
+          """|Timeout while looking up hash of unknown.io/library/image:v1.
+             |Ensure that docker is running correctly.
+             |""".stripMargin)
+    }
+  }
+}
diff --git a/core/src/test/scala/cromwell/core/callcaching/docker/registryv2/flows/HttpFlowWithRetrySpec.scala b/dockerHashing/src/test/scala/cromwell/docker/registryv2/flows/HttpFlowWithRetrySpec.scala
similarity index 62%
rename from core/src/test/scala/cromwell/core/callcaching/docker/registryv2/flows/HttpFlowWithRetrySpec.scala
rename to dockerHashing/src/test/scala/cromwell/docker/registryv2/flows/HttpFlowWithRetrySpec.scala
index 67d69bce7..6807de46c 100644
--- a/core/src/test/scala/cromwell/core/callcaching/docker/registryv2/flows/HttpFlowWithRetrySpec.scala
+++ b/dockerHashing/src/test/scala/cromwell/docker/registryv2/flows/HttpFlowWithRetrySpec.scala
@@ -1,4 +1,4 @@
-package cromwell.core.callcaching.docker.registryv2.flows
+package cromwell.docker.registryv2.flows
 
 import akka.NotUsed
 import akka.http.scaladsl.model.{HttpRequest, HttpResponse, StatusCodes}
@@ -6,7 +6,7 @@ import akka.stream._
 import akka.stream.scaladsl.{GraphDSL, RunnableGraph, Sink, Source}
 import akka.testkit.{ImplicitSender, TestProbe}
 import cromwell.core.TestKitSuite
-import cromwell.core.callcaching.docker.{HttpMock, MockHttpResponse}
+import cromwell.docker.{HttpMock, MockHttpResponse}
 import org.scalatest.{FlatSpecLike, Matchers}
 
 import scala.concurrent.duration._
@@ -17,6 +17,8 @@ class HttpFlowWithRetrySpec extends TestKitSuite with FlatSpecLike with Matchers
   behavior of "HttpFlowWithRetry"
 
   implicit val materializer = ActorMaterializer()
+  implicit val scheduler = system.scheduler
+  implicit val ec = system.dispatcher
 
   override protected def afterAll() = {
     materializer.shutdown()
@@ -67,7 +69,7 @@ class HttpFlowWithRetrySpec extends TestKitSuite with FlatSpecLike with Matchers
     stopProbe(probeFailure)
   }
 
-  it should "emit an Http response on failure port if request is not successful and not retryable" in {
+  it should "send back the http response if it is not successful and not retryable" in {
     val mockResponse = HttpResponse(status = StatusCodes.BadRequest)
     
     val httpMock = new HttpMock[Any](MockHttpResponse(Success(mockResponse), 1)).httpMock()
@@ -77,8 +79,8 @@ class HttpFlowWithRetrySpec extends TestKitSuite with FlatSpecLike with Matchers
 
     graph.run()
 
-    probeSuccess.expectNoMsg(1 second)
-    probeFailure.expectMsgPF(1 second) {
+    probeFailure.expectNoMsg(1 second)
+    probeSuccess.expectMsgPF(1 second) {
       case (response: HttpResponse, _) => response.status shouldBe StatusCodes.BadRequest
     }
 
@@ -86,7 +88,7 @@ class HttpFlowWithRetrySpec extends TestKitSuite with FlatSpecLike with Matchers
     stopProbe(probeFailure)
   }
 
-  it should "retry a request if it fails" in {
+  it should "fail a request if it fails" in {
     val mockFailure = Failure[HttpResponse](new Exception("Request failed - part of test flow"))
     val mockOK = HttpResponse(status = StatusCodes.OK)
     
@@ -97,9 +99,9 @@ class HttpFlowWithRetrySpec extends TestKitSuite with FlatSpecLike with Matchers
 
     graph.run()
 
-    probeFailure.expectNoMsg(1 second)
-    probeSuccess.expectMsgPF(1 second) {
-      case (response: HttpResponse, _) => response.status shouldBe StatusCodes.OK
+    probeSuccess.expectNoMsg(1 second)
+    probeFailure.expectMsgPF(1 second) {
+      case (failure: Throwable, _) => failure shouldBe mockFailure.failed.get
     }
 
     stopProbe(probeSuccess)
@@ -107,12 +109,10 @@ class HttpFlowWithRetrySpec extends TestKitSuite with FlatSpecLike with Matchers
   }
   
   it should "retry a request if the response is not a success and is retryable" in {
-    def isRetryable(httpResponse: HttpResponse) = httpResponse.status == StatusCodes.Unauthorized
-    
-    val mockFailure = HttpResponse(status = StatusCodes.Unauthorized)
+    val mockFailure = HttpResponse(status = StatusCodes.RequestTimeout)
     val mockOK = HttpResponse(status = StatusCodes.OK)
     val httpMock = new HttpMock[Any](MockHttpResponse(Success(mockFailure), 1), MockHttpResponse(Success(mockOK), 1)).httpMock()
-    val httpRetryMock = HttpFlowWithRetry[Any](httpMock, isRetryable = isRetryable)
+    val httpRetryMock = HttpFlowWithRetry[Any](httpMock)
 
     val (graph, probeSuccess, probeFailure) = makeRunnableGraph(httpRetryMock)
 
@@ -127,73 +127,51 @@ class HttpFlowWithRetrySpec extends TestKitSuite with FlatSpecLike with Matchers
     stopProbe(probeFailure)
   }
 
-  it should "handle flaky connections with a sufficiently large retry buffer" in {
+  it should "exponentially backoff and succeed if response is successful before max attempts" in {
     val mock200 = HttpResponse(status = StatusCodes.OK)
-    val mock429 = HttpResponse(status = StatusCodes.TooManyRequests)
-    
-    // only 20% of requests come back successful, rest are out of quota errors
-    val mock200s = MockHttpResponse(Success(mock200), 1)
-    val mock429s = MockHttpResponse(Success(mock429), 5)
+    val mock408 = HttpResponse(status = StatusCodes.RequestTimeout)
 
-    val httpMock = new HttpMock[Any](mock200s, mock429s).httpMock()
-    // Give a retry buffer of 1000 (= queues up to 1000 retry requests before starting dropping them) 
-    val httpRetryMock = new HttpFlowWithRetry[Any](httpMock, retryBufferSize = 1000)
+    val mock408s = MockHttpResponse(Success(mock408), 3)
+    val mock200s = MockHttpResponse(Success(mock200), 1)
 
-    // Send 1000 requests
-    val nbMessagesToSend = 1000
-    val mockRequest = HttpRequest()
-    val sourceIterator = 1 to nbMessagesToSend map { _ => (mockRequest, ()) } toIterator
-    val source = Source.fromIterator { () => sourceIterator }
-    val (graph, probeSuccess, probeFailure) = makeRunnableGraph(httpRetryMock, source)
+    val httpMock = new HttpMock[Any](mock408s, mock200s).httpMock()
+    // Give a retry buffer of 10 (= queues up to 10 retry requests before starting dropping them)
+    val httpRetryMock = new HttpFlowWithRetry[Any](httpMock, retryBufferSize = 10)
+    
+    val (graph, probeSuccess, probeFailure) = makeRunnableGraph(httpRetryMock)
 
     graph.run()
 
-    probeFailure.expectNoMsg(5 seconds)
-    var successes: Int = 0
-    
-    probeSuccess.receiveWhile(5 seconds, 500 millis, nbMessagesToSend) {
-      case _ => successes += 1
+    probeFailure.expectNoMsg(1 second)
+    // backoff starts at 1 second, multiplier is 3 and default randomization is 20%
+    // which means, worst case scenario, we need to wait
+    // (1 + 0.2) + (3 + 0.6) + (9 + 1.8) = 15.06 -> 20 to account for test latency
+    probeSuccess.expectMsgPF(20 seconds) {
+      case (response: HttpResponse, _) => response.status shouldBe StatusCodes.OK
     }
 
-    // All responses should come back successfully
-    successes == 1000 shouldBe true
-
     stopProbe(probeSuccess)
     stopProbe(probeFailure)
   }
 
-  it should "not deadlock even under high retry pressure" in {
-    val mock200 = HttpResponse(status = StatusCodes.OK)
-    val mock429 = HttpResponse(status = StatusCodes.TooManyRequests)
+  it should "exponentially backoff and fail if response is still unsuccessful after max attempts" in {
+    val mock408 = HttpResponse(status = StatusCodes.RequestTimeout)
 
-    // only 20% of requests come back successful, rest are out of quota errors
-    val mock200s = MockHttpResponse(Success(mock200), 1)
-    val mock429s = MockHttpResponse(Success(mock429), 5)
+    val mock408s = MockHttpResponse(Success(mock408), 1)
 
-    val httpMock = new HttpMock[Any](mock200s, mock429s).httpMock()
+    val httpMock = new HttpMock[Any](mock408s).httpMock()
     // Give a retry buffer of 10 (= queues up to 10 retry requests before starting dropping them)
     val httpRetryMock = new HttpFlowWithRetry[Any](httpMock, retryBufferSize = 10)
 
-    // Send 1000 requests, the retry buffer will overflow, and start dropping requests
-    // Note that this is an extreme case, 80% failure rate with 100 * retry buffer size elements and no throttling
-    val nbMessagesToSend = 1000
-    val mockRequest = HttpRequest()
-    val sourceIterator = 1 to nbMessagesToSend map { _ => (mockRequest, ()) } toIterator
-    val source = Source.fromIterator { () => sourceIterator }
-    val (graph, probeSuccess, probeFailure) = makeRunnableGraph(httpRetryMock, source)
+    val (graph, probeSuccess, probeFailure) = makeRunnableGraph(httpRetryMock)
 
     graph.run()
 
-    probeFailure.expectNoMsg(5 seconds)
-    var successes: Int = 0
-    // The second parameter makes sure that the stream keeps emitting responses at least every 500 milliseconds
-    probeSuccess.receiveWhile(5 seconds, 500 millis, nbMessagesToSend) {
-      case _ => successes += 1
+    probeFailure.expectNoMsg(1 second)
+    probeSuccess.expectMsgPF(20 seconds) {
+      case (response: HttpResponse, _) => response.status shouldBe StatusCodes.RequestTimeout
     }
 
-    // A minimum amount of requests should come back
-    successes > 200 shouldBe true
-
     stopProbe(probeSuccess)
     stopProbe(probeFailure)
   }
diff --git a/engine/src/main/resources/swagger/cromwell.yaml b/engine/src/main/resources/swagger/cromwell.yaml
index 962711bde..012681daa 100644
--- a/engine/src/main/resources/swagger/cromwell.yaml
+++ b/engine/src/main/resources/swagger/cromwell.yaml
@@ -11,9 +11,8 @@ info:
   version: ''
 produces:
   - application/json
-basePath: /api
 paths:
-  '/workflows/{version}/{id}/abort':
+  '/api/workflows/{version}/{id}/abort':
     post:
       summary: Abort a workflow based on workflow id
       parameters:
@@ -46,7 +45,7 @@ paths:
       security:
         - google_oauth:
             - openid
-  '/workflows/{version}':
+  '/api/workflows/{version}':
     post:
       summary: Submit a new workflow for execution
       consumes:
@@ -58,33 +57,33 @@ paths:
           type: string
           in: path
           default: v1
-        - name: wdlSource
-          description: WDL Source
+        - name: workflowSource
+          description: Workflow source file
           required: true
           type: file
           in: formData
         - name: workflowInputs
-          description: WDL Inputs JSON, 1
+          description: Workflow Inputs JSON, 1
           required: false
           type: file
           in: formData
         - name: workflowInputs_2
-          description: WDL Inputs JSON, 2
+          description: Workflow Inputs JSON, 2
           required: false
           type: file
           in: formData
         - name: workflowInputs_3
-          description: WDL Inputs JSON, 3
+          description: Workflow Inputs JSON, 3
           required: false
           type: file
           in: formData
         - name: workflowInputs_4
-          description: WDL Inputs JSON, 4
+          description: Workflow Inputs JSON, 4
           required: false
           type: file
           in: formData
         - name: workflowInputs_5
-          description: WDL Inputs JSON, 5
+          description: Workflow Inputs JSON, 5
           required: false
           type: file
           in: formData
@@ -93,13 +92,23 @@ paths:
           required: false
           type: file
           in: formData
+        - name: workflowType
+          description: Type of workflow
+          required: false
+          type: string
+          in: formData
+        - name: workflowTypeVersion
+          description: Version of workflow type
+          required: false
+          type: string
+          in: formData
         - name: customLabels
           description: Custom Labels JSON
           required: false
           type: file
           in: formData
-        - name: wdlDependencies
-          description: Zip file containing WDL dependencies
+        - name: workflowDependencies
+          description: Zip file containing workflow dependencies
           required: false
           type: file
           in: formData
@@ -117,7 +126,7 @@ paths:
       security:
         - google_oauth:
             - openid
-  '/workflows/{version}/batch':
+  '/api/workflows/{version}/batch':
     post:
       summary: Submit a batch of new workflows for execution
       consumes:
@@ -129,13 +138,13 @@ paths:
           type: string
           in: path
           default: v1
-        - name: wdlSource
-          description: WDL Source
+        - name: workflowSource
+          description: Workflow Source
           required: true
           type: file
           in: formData
         - name: workflowInputs
-          description: WDL Inputs JSON as an array
+          description: Workflow Inputs JSON as an array
           required: true
           type: file
           in: formData
@@ -144,13 +153,23 @@ paths:
           required: false
           type: file
           in: formData
+        - name: workflowType
+          description: Type of workflow
+          required: false
+          type: string
+          in: formData
+        - name: workflowTypeVersion
+          description: Version of workflow type
+          required: false
+          type: string
+          in: formData
         - name: customLabels
           description: Custom Labels JSON
           required: false
           type: file
           in: formData
-        - name: wdlDependencies
-          description: Zip file containing WDL dependencies
+        - name: workflowDependencies
+          description: Zip file containing workflow dependencies
           required: false
           type: file
           in: formData
@@ -170,7 +189,7 @@ paths:
       security:
         - google_oauth:
             - openid
-  '/workflows/{version}/{id}/outputs':
+  '/api/workflows/{version}/{id}/outputs':
     get:
       summary: Query for workflow outputs based on workflow id
       parameters:
@@ -199,9 +218,48 @@ paths:
       security:
         - google_oauth:
             - openid
-  '/workflows/{version}/query':
+  '/api/workflows/{version}/{id}/labels':
+    patch:
+      summary: Add new labels or update values for existing label keys by workflow id.
+      parameters:
+        - name: version
+          description: API Version
+          required: true
+          type: string
+          in: path
+          default: v1
+        - name: id
+          description: Workflow ID
+          required: true
+          type: string
+          in: path
+        - name: labels
+          description: >
+            Custom labels submitted as JSON.
+            Example: {"key-1":"value-1","key-2":"value-2"}
+          required: true
+          in: body
+          schema:
+            type: object
+      tags:
+        - Workflows
+      responses:
+        '200':
+          description: Successful Request
+          schema:
+            $ref: '#/definitions/LabelsResponse'
+        '400':
+          description: Malformed Workflow ID
+        '404':
+          description: Workflow ID Not Found
+        '500':
+          description: Internal Error
+      security:
+        - google_oauth:
+            - openid
+  '/api/workflows/{version}/query':
     get:
-      summary: Query workflows by start dates, end dates, names, ids, or statuses.
+      summary: Query workflows by start dates, end dates, names, ids, statuses, or labels.
       parameters:
         - name: version
           description: API Version
@@ -264,6 +322,19 @@ paths:
           description: >
             Returns only workflows with the specified workflow id.  If specified multiple times,
             returns workflows with any of the specified workflow ids.
+        - name: label
+          required: false
+          in: query
+          type: array
+          items:
+            type: string
+          collectionFormat: multi
+          pattern: ^([a-z][-a-z0-9]*[a-z0-9])?[:]([a-z][-a-z0-9]*[a-z0-9])?$
+          description: >
+            Returns workflows with the specified label keys.  If specified multiple times,
+            returns workflows with all of the specified label keys. Specify the label key
+            and label value pair as separated with
+            "label-key:label-value"
       tags:
         - Workflows
       responses:
@@ -311,7 +382,7 @@ paths:
       security:
         - google_oauth:
             - openid
-  '/workflows/{version}/{id}/status':
+  '/api/workflows/{version}/{id}/status':
     get:
       summary: Query for workflow status based on workflow id
       parameters:
@@ -342,7 +413,7 @@ paths:
       security:
         - google_oauth:
             - openid
-  '/workflows/{version}/{id}/logs':
+  '/api/workflows/{version}/{id}/logs':
     get:
       summary: Query for the standard output and error of all calls in a workflow
       parameters:
@@ -371,7 +442,61 @@ paths:
       security:
         - google_oauth:
             - openid
-  '/workflows/{version}/{id}/metadata':
+  '/api/workflows/{version}/callcaching/diff':
+    get:
+      summary: Return the hash differential between two calls
+      parameters:
+        - name: version
+          description: API Version
+          required: true
+          type: string
+          in: path
+          default: v1
+        - name: workflowA
+          description: Workflow Id of the first workflow
+          required: true
+          type: string
+          in: query
+        - name: callA
+          description: Call FQN (including workflow name) of the first call
+          required: true
+          type: string
+          in: query
+        - name: indexA
+          description: Index of the first call
+          required: false
+          type: int
+          in: query
+        - name: workflowB
+          description: Workflow Id of the second workflow
+          required: true
+          type: string
+          in: query
+        - name: callB
+          description: Call FQN (including workflow name) of the second call
+          required: true
+          type: string
+          in: query
+        - name: indexB
+          description: Index of the second call
+          required: false
+          type: int
+          in: query
+      tags:
+        - Workflows
+      responses:
+        '200':
+          description: Successful Request
+        '400':
+          description: Malformed query parameters
+        '404':
+          description: No matching cache entry
+        '500':
+          description: Internal Error
+      security:
+        - google_oauth:
+            - openid
+  '/api/workflows/{version}/{id}/metadata':
     get:
       summary: Query for workflow and call-level metadata for a specified workflow
       parameters:
@@ -556,7 +681,7 @@ paths:
       security:
         - google_oauth:
             - openid
-  '/workflows/{version}/backends':
+  '/api/workflows/{version}/backends':
     get:
       summary: Returns the backends supported by this Cromwell.
       parameters:
@@ -593,9 +718,6 @@ paths:
           description: Successful Request
           schema:
             $ref: '#/definitions/StatsResponse'
-      security:
-        - google_oauth:
-            - openid
   '/engine/{version}/version':
     get:
       summary: Returns the version of the Cromwell Engine
@@ -613,9 +735,6 @@ paths:
           description: Successful Request
           schema:
             $ref: '#/definitions/VersionResponse'
-      security:
-        - google_oauth:
-            - openid
 securityDefinitions:
   google_oauth:
     type: oauth2
@@ -855,3 +974,14 @@ definitions:
       cromwell:
         type: string
         description: The version of the Cromwell Engine
+  LabelsResponse:
+    required:
+      - id
+      - labels
+    properties:
+      id:
+        type: string
+        description: The identifier of the workflow
+      labels:
+        type: string
+        description: The custom labels as json
diff --git a/engine/src/main/resources/workflowTimings/workflowTimings.html b/engine/src/main/resources/workflowTimings/workflowTimings.html
index 3e1df152c..368c7bf5d 100644
--- a/engine/src/main/resources/workflowTimings/workflowTimings.html
+++ b/engine/src/main/resources/workflowTimings/workflowTimings.html
@@ -69,7 +69,7 @@
                     var firstEventStart = null;
                     var finalEventEnd = null;
 
-                    if(callStatus == "Done" || callStatus == "Failed" || callStatus == "Preempted") {
+                    if(callStatus == "Done" || callStatus == "Failed" || callStatus == "RetryableFailure") {
                         executionCallsCount++;
                         for (var executionEventIndex in executionEvents) {
                             var executionEvent = callList[callIndex].executionEvents[executionEventIndex];
diff --git a/engine/src/main/scala/cromwell/Simpletons.scala b/engine/src/main/scala/cromwell/Simpletons.scala
index 1d30e6c6b..895f76f1a 100644
--- a/engine/src/main/scala/cromwell/Simpletons.scala
+++ b/engine/src/main/scala/cromwell/Simpletons.scala
@@ -3,8 +3,8 @@ package cromwell
 import cromwell.core.simpleton.WdlValueSimpleton
 import cromwell.database.sql.SqlConverters._
 import cromwell.database.sql.tables.{CallCachingSimpletonEntry, JobStoreSimpletonEntry}
-import wdl4s.types.{WdlBooleanType, WdlFloatType, WdlIntegerType, WdlStringType}
-import wdl4s.values.{WdlPrimitive, WdlSingleFile, WdlValue}
+import wdl4s.wdl.types.{WdlBooleanType, WdlFloatType, WdlIntegerType, WdlStringType}
+import wdl4s.wdl.values.{WdlPrimitive, WdlSingleFile, WdlValue}
 
 import scala.util.Try
 
diff --git a/engine/src/main/scala/cromwell/engine/EngineFilesystems.scala b/engine/src/main/scala/cromwell/engine/EngineFilesystems.scala
index a738984dd..d95ee8ed7 100644
--- a/engine/src/main/scala/cromwell/engine/EngineFilesystems.scala
+++ b/engine/src/main/scala/cromwell/engine/EngineFilesystems.scala
@@ -2,57 +2,49 @@ package cromwell.engine
 
 import akka.actor.ActorSystem
 import cats.data.Validated.{Invalid, Valid}
-import com.google.api.client.http.HttpResponseException
-import com.typesafe.config.ConfigFactory
+import cats.instances.future._
+import cats.instances.list._
+import cats.syntax.traverse._
+import com.typesafe.config.{Config, ConfigFactory}
 import cromwell.core.WorkflowOptions
-import cromwell.core.path.{CustomRetryParams, DefaultPathBuilder, PathBuilder}
-import cromwell.core.retry.SimpleExponentialBackoff
-import cromwell.filesystems.gcs.{GoogleConfiguration, RetryableGcsPathBuilderFactory}
+import cromwell.core.path.{DefaultPathBuilder, PathBuilder}
+import cromwell.filesystems.gcs.auth.GoogleAuthMode
+import cromwell.filesystems.gcs.{GcsPathBuilderFactory, GoogleConfiguration}
 import lenthall.exception.MessageAggregation
+import lenthall.validation.ErrorOr.ErrorOr
 import net.ceedubs.ficus.Ficus._
 
-import scala.concurrent.duration._
-import scala.language.postfixOps
-
-case class EngineFilesystems(actorSystem: ActorSystem) {
-
-  private def isFatalGcsException(t: Throwable): Boolean = t match {
-    case e: HttpResponseException if e.getStatusCode == 403 => true
-    case e: HttpResponseException if e.getStatusCode == 400 && e.getContent.contains("INVALID_ARGUMENT") => true
-    case _ => false
-  }
-
-  private def isTransientGcsException(t: Throwable): Boolean = t match {
-    // Quota exceeded
-    case e: HttpResponseException if e.getStatusCode == 429 => true
-    case _ => false
-  }
-
-  private val GcsRetryParams = CustomRetryParams(
-    timeout = Duration.Inf,
-    maxRetries = Option(3),
-    backoff = SimpleExponentialBackoff(1 seconds, 3 seconds, 1.5D),
-    isTransient = isTransientGcsException,
-    isFatal = isFatalGcsException
-  )
-
-  private val config = ConfigFactory.load
-  private val googleConf: GoogleConfiguration = GoogleConfiguration(config)
-  private val googleAuthMode = config.as[Option[String]]("engine.filesystems.gcs.auth") map { confMode =>
-    googleConf.auth(confMode) match {
-      case Valid(mode) => mode
+import scala.concurrent.{ExecutionContext, Future}
+import scala.util.{Failure, Success, Try}
+
+object EngineFilesystems {
+  private val config: Config = ConfigFactory.load
+  
+  private val gcsPathBuilderFactory: Try[Option[GcsPathBuilderFactory]] = Try {
+    // Parse the configuration and create a GoogleConfiguration
+    val googleConf: GoogleConfiguration = GoogleConfiguration(config)
+    // Extract the specified authentication mode for engine gcs filesystem, if any
+    val engineAuthModeAsString: Option[String] = config.as[Option[String]]("engine.filesystems.gcs.auth")
+    // Validate it agasint the google configuration
+    val engineAuthModeValidation: Option[ErrorOr[GoogleAuthMode]] = engineAuthModeAsString map googleConf.auth
+    
+    engineAuthModeValidation map {
+      // If the authentication mode is recognized, create a GcsPathBuilderFactory for the engine
+      case Valid(mode) => GcsPathBuilderFactory(mode, googleConf.applicationName)
+      // Otherwise fail
       case Invalid(errors) => throw new RuntimeException() with MessageAggregation {
-        override def exceptionContext: String = s"Failed to create authentication mode for $confMode"
+        override def exceptionContext: String = s"Failed to create authentication mode for $engineAuthModeAsString"
         override def errorMessages: Traversable[String] = errors.toList
       }
     }
   }
 
-  private val gcsPathBuilderFactory = googleAuthMode map { mode =>
-    RetryableGcsPathBuilderFactory(mode, customRetryParams = GcsRetryParams)
-  }
+  private val defaultFileSystem =
+    Option(DefaultPathBuilder).filter(_ => config.as[Boolean]("engine.filesystems.local.enabled"))
 
-  def pathBuildersForWorkflow(workflowOptions: WorkflowOptions): List[PathBuilder] = {
-    List(gcsPathBuilderFactory map { _.withOptions(workflowOptions)(actorSystem) }, Option(DefaultPathBuilder)).flatten
+  def pathBuildersForWorkflow(workflowOptions: WorkflowOptions)(implicit as: ActorSystem, ec: ExecutionContext): Future[List[PathBuilder]] = gcsPathBuilderFactory match {
+    case Success(maybeBuilderFactory) => maybeBuilderFactory.toList.traverse(_.withOptions(workflowOptions)).map(_ ++ defaultFileSystem)
+    case Failure(failure) => Future.failed(failure)
   }
+
 }
diff --git a/engine/src/main/scala/cromwell/engine/EngineWorkflowDescriptor.scala b/engine/src/main/scala/cromwell/engine/EngineWorkflowDescriptor.scala
index c6fd821ae..6850e1681 100644
--- a/engine/src/main/scala/cromwell/engine/EngineWorkflowDescriptor.scala
+++ b/engine/src/main/scala/cromwell/engine/EngineWorkflowDescriptor.scala
@@ -4,20 +4,22 @@ import cromwell.backend.BackendWorkflowDescriptor
 import cromwell.core.WorkflowOptions.WorkflowOption
 import cromwell.core.callcaching.CallCachingMode
 import cromwell.core.path.PathBuilder
-import wdl4s._
+import wdl4s.wdl._
 
 case class EngineWorkflowDescriptor(namespace: WdlNamespaceWithWorkflow,
-                                          backendDescriptor: BackendWorkflowDescriptor,
-                                          backendAssignments: Map[TaskCall, String],
-                                          failureMode: WorkflowFailureMode,
-                                          pathBuilders: List[PathBuilder],
-                                          callCachingMode: CallCachingMode,
-                                          parentWorkflow: Option[EngineWorkflowDescriptor] = None) {
+                                    backendDescriptor: BackendWorkflowDescriptor,
+                                    backendAssignments: Map[WdlTaskCall, String],
+                                    failureMode: WorkflowFailureMode,
+                                    pathBuilders: List[PathBuilder],
+                                    callCachingMode: CallCachingMode,
+                                    parentWorkflow: Option[EngineWorkflowDescriptor] = None) {
   
   val rootWorkflow: EngineWorkflowDescriptor = parentWorkflow match {
     case Some(parent) => parent.rootWorkflow
     case None => this
   }
+
+  def isRootWorkflow = rootWorkflow.parentWorkflow.isEmpty
   
   lazy val id = backendDescriptor.id
   lazy val workflow = backendDescriptor.workflow
diff --git a/engine/src/main/scala/cromwell/engine/WdlFunctions.scala b/engine/src/main/scala/cromwell/engine/WdlFunctions.scala
index 9fe346c50..73f928cac 100644
--- a/engine/src/main/scala/cromwell/engine/WdlFunctions.scala
+++ b/engine/src/main/scala/cromwell/engine/WdlFunctions.scala
@@ -1,9 +1,9 @@
 package cromwell.engine
 
 import cromwell.backend.wdl.ReadLikeFunctions
-import wdl4s.expression.PureStandardLibraryFunctionsLike
+import wdl4s.wdl.expression.PureStandardLibraryFunctionsLike
 import cromwell.core.path.PathBuilder
-import wdl4s.values.{WdlFile, WdlValue}
+import wdl4s.wdl.values.{WdlFile, WdlValue}
 
 import scala.util.{Failure, Try}
 
diff --git a/engine/src/main/scala/cromwell/engine/engine.scala b/engine/src/main/scala/cromwell/engine/engine.scala
index 7a65770b1..5bdcd166f 100644
--- a/engine/src/main/scala/cromwell/engine/engine.scala
+++ b/engine/src/main/scala/cromwell/engine/engine.scala
@@ -1,15 +1,12 @@
 package cromwell.engine
 
-import java.time.OffsetDateTime
-
-import wdl4s._
+import wdl4s.wdl._
 
 import scala.util.{Failure, Success, Try}
 
 final case class AbortFunction(function: () => Unit)
 final case class AbortRegistrationFunction(register: AbortFunction => Unit)
 
-final case class FailureEventEntry(failure: String, timestamp: OffsetDateTime)
 final case class CallAttempt(fqn: FullyQualifiedName, attempt: Int)
 
 object WorkflowFailureMode {
diff --git a/engine/src/main/scala/cromwell/engine/io/IoActor.scala b/engine/src/main/scala/cromwell/engine/io/IoActor.scala
new file mode 100644
index 000000000..64eae2536
--- /dev/null
+++ b/engine/src/main/scala/cromwell/engine/io/IoActor.scala
@@ -0,0 +1,173 @@
+package cromwell.engine.io
+
+import java.net.{SocketException, SocketTimeoutException}
+import javax.net.ssl.SSLException
+
+import akka.NotUsed
+import akka.actor.{Actor, ActorLogging, ActorRef, Props}
+import akka.stream._
+import akka.stream.scaladsl.{Flow, GraphDSL, Merge, Partition, Source}
+import com.google.cloud.storage.StorageException
+import com.typesafe.config.ConfigFactory
+import cromwell.core.Dispatcher
+import cromwell.core.actor.StreamActorHelper
+import cromwell.core.actor.StreamIntegration.StreamContext
+import cromwell.core.io.{IoAck, IoCommand, Throttle}
+import cromwell.engine.io.IoActor._
+import cromwell.engine.io.gcs.GcsBatchFlow.BatchFailedException
+import cromwell.engine.io.gcs.{GcsBatchCommandContext, ParallelGcsBatchFlow}
+import cromwell.engine.io.nio.NioFlow
+import cromwell.filesystems.gcs.batch.GcsBatchIoCommand
+import cromwell.core.Dispatcher.IoDispatcher
+
+/**
+  * Actor that performs IO operations asynchronously using akka streams
+  * 
+  * @param queueSize size of the queue
+  * @param throttle optional throttler to control the throughput of requests.
+  *                 Applied to ALL incoming requests
+  * @param materializer actor materializer to run the stream
+  */
+final class IoActor(queueSize: Int, throttle: Option[Throttle])(implicit val materializer: ActorMaterializer) extends Actor with ActorLogging with StreamActorHelper[IoCommandContext[_]] {
+  
+  implicit private val system = context.system
+  implicit val ec = context.dispatcher
+  
+  private [io] lazy val defaultFlow = new NioFlow(parallelism = 100, context.system.scheduler).flow.withAttributes(ActorAttributes.dispatcher(Dispatcher.IoDispatcher))
+  private [io] lazy val gcsBatchFlow = new ParallelGcsBatchFlow(parallelism = 10, batchSize = 100, context.system.scheduler).flow.withAttributes(ActorAttributes.dispatcher(Dispatcher.IoDispatcher))
+  
+  protected val source = Source.queue[IoCommandContext[_]](queueSize, OverflowStrategy.dropNew)
+
+  protected val flow = GraphDSL.create() { implicit builder =>
+    import GraphDSL.Implicits._
+    
+    val input = builder.add(Flow[IoCommandContext[_]])
+    
+    // Partitions requests between gcs batch, and single nio requests
+    val batchPartitioner = builder.add(Partition[IoCommandContext[_]](2, {
+      case _: GcsBatchCommandContext[_, _] => 0
+      case other @ _ => 1
+    }))
+    
+    // Sub flow for batched gcs requests
+    val batches = batchPartitioner.out(0) collect { case batch: GcsBatchCommandContext[_, _] => batch }
+    
+    // Sub flow for single nio requests
+    val defaults = batchPartitioner.out(1) collect { case default: DefaultCommandContext[_] => default }
+    
+    // Merge results from both flows back together
+    val merger = builder.add(Merge[IoResult](2))
+    
+    // Flow processing nio requests
+    val defaultFlowPorts = builder.add(defaultFlow)
+    
+    // Flow processing gcs batch requests
+    val batchFlowPorts = builder.add(gcsBatchFlow)
+
+    input ~> batchPartitioner
+             defaults.outlet ~> defaultFlowPorts ~> merger
+             batches.outlet ~> batchFlowPorts ~> merger
+    
+    FlowShape[IoCommandContext[_], IoResult](input.in, merger.out)
+  }
+
+  protected val throttledFlow = throttle map { t => 
+    Flow[IoCommandContext[_]]
+      .throttle(t.elements, t.per, t.maximumBurst, ThrottleMode.Shaping)
+      .via(flow)
+  } getOrElse flow
+  
+  override protected lazy val streamSource = source.via(throttledFlow).withAttributes(ActorAttributes.dispatcher(Dispatcher.IoDispatcher))
+  
+  override def actorReceive: Receive = {
+    /* GCS Batch command with context */
+    case (clientContext: Any, gcsBatchCommand: GcsBatchIoCommand[_, _]) =>
+      val replyTo = sender()
+      val commandContext= GcsBatchCommandContext(gcsBatchCommand, replyTo, Option(clientContext))
+      sendToStream(commandContext)
+
+    /* GCS Batch command without context */
+    case gcsBatchCommand: GcsBatchIoCommand[_, _] =>
+      val replyTo = sender()
+      val commandContext= GcsBatchCommandContext(gcsBatchCommand, replyTo)
+      sendToStream(commandContext)
+
+    /* Default command with context */
+    case (clientContext: Any, command: IoCommand[_]) =>
+      val replyTo = sender()
+      val commandContext= DefaultCommandContext(command, replyTo, Option(clientContext))
+      sendToStream(commandContext)
+      
+    /* Default command without context */
+    case command: IoCommand[_] => 
+      val replyTo = sender()
+      val commandContext= DefaultCommandContext(command, replyTo)
+      sendToStream(commandContext)
+  }
+}
+
+trait IoCommandContext[T] extends StreamContext {
+  def request: IoCommand[T]
+  def replyTo: ActorRef
+  def fail(failure: Throwable): IoResult = (request.fail(failure), this)
+  def success(value: T): IoResult = (request.success(value), this)
+}
+
+object IoActor {
+  import net.ceedubs.ficus.Ficus._
+  
+  /** Flow that can consume an IoCommandContext and produce an IoResult */
+  type IoFlow = Flow[IoCommandContext[_], IoResult, NotUsed]
+  
+  /** Result type of an IoFlow, contains the original command context and the final IoAck response. */
+  type IoResult = (IoAck[_], IoCommandContext[_])
+  
+  private val ioConfig = ConfigFactory.load().getConfig("system.io")
+  
+  /** Maximum number of times a command will be attempted: First attempt + 5 retries */
+  val MaxAttemptsNumber = ioConfig.getOrElse[Int]("number-of-attempts", 5)
+
+  case class DefaultCommandContext[T](request: IoCommand[T], replyTo: ActorRef, override val clientContext: Option[Any] = None) extends IoCommandContext[T]
+
+  /**
+    * ATTENTION: Transient failures are retried *forever* 
+    * Be careful when adding error codes to this method.
+    * Currently only 429 (= quota exceeded are considered truly transient)
+    */
+  def isTransient(failure: Throwable): Boolean = failure match {
+    case gcs: StorageException => gcs.getCode == 429
+    case _ => false
+  }
+
+  val AdditionalRetryableHttpCodes = List(
+    // HTTP 410: Gone
+    // From Google doc (https://cloud.google.com/storage/docs/json_api/v1/status-codes):
+    // "You have attempted to use a resumable upload session that is no longer available.
+    // If the reported status code was not successful and you still wish to upload the file, you must start a new session."
+    410,
+    // Some 503 errors seem to yield "false" on the "isRetryable" method because they are not retried.
+    // The CloudStorage exception mechanism is not flawless yet (https://github.com/GoogleCloudPlatform/google-cloud-java/issues/1545)
+    // so that could be the cause.
+    // For now explicitly lists 503 as a retryable code here to work around that.
+    503
+  )
+  
+  /**
+    * Failures that are considered retryable.
+    * Retrying them should increase the "retry counter"
+    */
+  def isRetryable(failure: Throwable): Boolean = failure match {
+    case gcs: StorageException => gcs.isRetryable || AdditionalRetryableHttpCodes.contains(gcs.getCode) || isRetryable(gcs.getCause)
+    case _: SSLException => true
+    case _: BatchFailedException => true
+    case _: SocketException => true
+    case _: SocketTimeoutException => true
+    case other => isTransient(other)
+  }
+
+  def isFatal(failure: Throwable) = !isRetryable(failure)
+  
+  def props(queueSize: Int, throttle: Option[Throttle])(implicit materializer: ActorMaterializer) = {
+    Props(new IoActor(queueSize, throttle)).withDispatcher(IoDispatcher)
+  }
+}
diff --git a/engine/src/main/scala/cromwell/engine/io/gcs/GcsBatchCommandContext.scala b/engine/src/main/scala/cromwell/engine/io/gcs/GcsBatchCommandContext.scala
new file mode 100644
index 000000000..097649fc1
--- /dev/null
+++ b/engine/src/main/scala/cromwell/engine/io/gcs/GcsBatchCommandContext.scala
@@ -0,0 +1,95 @@
+package cromwell.engine.io.gcs
+
+import akka.actor.ActorRef
+import com.google.api.client.googleapis.batch.BatchRequest
+import com.google.api.client.googleapis.batch.json.JsonBatchCallback
+import com.google.api.client.googleapis.json.GoogleJsonError
+import com.google.api.client.http.HttpHeaders
+import com.google.api.client.util.ExponentialBackOff
+import com.google.cloud.storage.StorageException
+import cromwell.core.retry.{Backoff, SimpleExponentialBackoff}
+import cromwell.engine.io.IoActor.IoResult
+import cromwell.engine.io.gcs.GcsBatchCommandContext.BatchResponse
+import cromwell.engine.io.{IoActor, IoCommandContext}
+import cromwell.filesystems.gcs.batch.GcsBatchIoCommand
+
+import scala.concurrent.Promise
+import scala.concurrent.duration._
+import scala.language.postfixOps
+
+object GcsBatchCommandContext {
+  def defaultBackoff = SimpleExponentialBackoff(
+    new ExponentialBackOff.Builder()
+      .setInitialIntervalMillis(1.second.toMillis.toInt)
+      .setMultiplier(4)
+      .setMaxIntervalMillis(30.seconds.toMillis.toInt)
+      .setRandomizationFactor(0.2D)
+      .setMaxElapsedTimeMillis(30.minutes.toMillis.toInt)
+      .build()
+  )
+  type BatchResponse = Either[IoResult, GcsBatchCommandContext[_, _]]
+}
+
+final case class GcsBatchCommandContext[T, U](request: GcsBatchIoCommand[T, U],
+                                              replyTo: ActorRef,
+                                              override val clientContext: Option[Any] = None,
+                                              backoff: Backoff = GcsBatchCommandContext.defaultBackoff,
+                                              currentAttempt: Int = 1,
+                                              promise: Promise[BatchResponse] = Promise[BatchResponse]
+                                    ) extends IoCommandContext[T] {
+
+  /**
+    * None if no retry should be attempted, Some(timeToWaitBeforeNextAttempt) otherwise
+    */
+  lazy val retryIn = if (currentAttempt >= IoActor.MaxAttemptsNumber) None else Option(backoff.backoffMillis milliseconds)
+
+  /**
+    * Json batch call back for a batched request
+    */
+  lazy val callback: JsonBatchCallback[U] = new JsonBatchCallback[U]() {
+    def onSuccess(response: U, httpHeaders: HttpHeaders) = onSuccessCallback(response, httpHeaders)
+    def onFailure(googleJsonError: GoogleJsonError, httpHeaders: HttpHeaders) = onFailureCallback(googleJsonError, httpHeaders)
+  }
+
+  /**
+    * Increment backoff time and attempt count
+    */
+  lazy val next: GcsBatchCommandContext[T, U] = {
+    this.copy(backoff = backoff.next, currentAttempt = currentAttempt + 1, promise = Promise[BatchResponse])
+  }
+
+  /**
+    * Only increment backoff. To be used for failure thas should be retried infinitely
+    */
+  lazy val nextTransient: GcsBatchCommandContext[T, U] = {
+    this.copy(backoff = backoff.next, promise = Promise[BatchResponse])
+  }
+
+  /**
+    * Queue the request for batching
+    */
+  def queue(batchRequest: BatchRequest) = request.operation.queue(batchRequest, callback)
+
+  /**
+    * On success callback. Transform the request response to a stream-ready response that can complete the promise
+    */
+  private def onSuccessCallback(response: U, httpHeaders: HttpHeaders) = {
+    val promiseResponse: BatchResponse = request.onSuccess(response, httpHeaders) match {
+        // Left means the command is complete, so just create the corresponding IoSuccess with the value
+      case Left(responseValue) => Left(success(responseValue))
+        // Right means there is a subsequent request to be executed, clone this context with the new request and a new promise
+      case Right(nextCommand) => Right(this.copy(request = nextCommand, promise = Promise[BatchResponse]))
+    }
+    
+    promise.trySuccess(promiseResponse)
+    ()
+  }
+
+  /**
+    * On failure callback. Fail the promise with a StorageException
+    */
+  private def onFailureCallback(googleJsonError: GoogleJsonError, httpHeaders: HttpHeaders) = {
+    promise.tryFailure(new StorageException(googleJsonError))
+    ()
+  }
+}
diff --git a/engine/src/main/scala/cromwell/engine/io/gcs/GcsBatchFlow.scala b/engine/src/main/scala/cromwell/engine/io/gcs/GcsBatchFlow.scala
new file mode 100644
index 000000000..646a79799
--- /dev/null
+++ b/engine/src/main/scala/cromwell/engine/io/gcs/GcsBatchFlow.scala
@@ -0,0 +1,152 @@
+package cromwell.engine.io.gcs
+
+import java.io.IOException
+
+import akka.actor.Scheduler
+import akka.stream._
+import akka.stream.scaladsl.{Flow, GraphDSL, MergePreferred, Partition}
+import com.google.api.client.googleapis.batch.BatchRequest
+import com.google.api.client.http.{HttpRequest, HttpRequestInitializer}
+import cromwell.engine.io.IoActor
+import cromwell.engine.io.IoActor.IoResult
+import cromwell.engine.io.gcs.GcsBatchFlow.BatchFailedException
+import cromwell.filesystems.gcs.{GcsPathBuilder, GoogleConfiguration}
+
+import scala.concurrent.duration._
+import scala.concurrent.{ExecutionContext, Future}
+import scala.language.postfixOps
+import scala.util.{Failure, Try}
+
+object GcsBatchFlow {
+
+  /**
+    * Exception used to fail the request promises when the batch request itself fails.
+    * Is considered retryable.
+    */
+  case class BatchFailedException(failure: Throwable) extends IOException(failure)
+}
+
+class GcsBatchFlow(batchSize: Int, scheduler: Scheduler)(implicit ec: ExecutionContext) {
+
+  // Does not carry any authentication, assumes all underlying requests are properly authenticated
+  private val httpRequestInitializer = new HttpRequestInitializer {
+    override def initialize(request: HttpRequest): Unit = {
+      request.setConnectTimeout(GoogleConfiguration.DefaultConnectionTimeout.toMillis.toInt)
+      request.setReadTimeout(GoogleConfiguration.DefaultReadTimeout.toMillis.toInt)
+      ()
+    }
+  }
+  
+  private val batch: BatchRequest = new BatchRequest(GcsPathBuilder.HttpTransport, httpRequestInitializer)
+  
+  val flow = GraphDSL.create() { implicit builder =>
+    import GraphDSL.Implicits._
+
+    // Source where batch commands are coming from. This is the input port of this flow
+    val source = builder.add(Flow[GcsBatchCommandContext[_, _]])
+    
+    // Merge commands from source (above), and commands that need to be retried (see retries below)
+    val sourceMerger = builder.add(MergePreferred[GcsBatchCommandContext[_, _]](1))
+    
+    // Process a batch and spit atomic GcsBatchResponses out for each internal request
+    val batchProcessor = builder.add(
+      Flow[GcsBatchCommandContext[_, _]]
+        // Group commands together in batches so they can be processed as such
+      .groupedWithin(batchSize, 5 seconds)
+        // execute the batch and outputs each sub-response individually, as a Future
+      .mapConcat[Future[GcsBatchResponse[_]]](executeBatch)
+        // Wait for each Future to complete
+      .mapAsyncUnordered[GcsBatchResponse[_]](batchSize) { identity }
+    )
+    
+    // Partitions the responses: Terminal responses exit the flow, others go back to the sourceMerger
+    val responseHandler = builder.add(responseHandlerFlow)
+      
+    // Buffer commands to be retried to avoid backpressuring too rapidly
+    val nextRequestBuffer = builder.add(Flow[GcsBatchCommandContext[_, _]].buffer(batchSize, OverflowStrategy.backpressure))
+    
+    source ~> sourceMerger ~> batchProcessor ~> responseHandler.in
+              sourceMerger.preferred <~ nextRequestBuffer <~ responseHandler.out1
+
+    FlowShape[GcsBatchCommandContext[_, _], IoResult](source.in, responseHandler.out0)
+  }
+  
+  /**
+    * Fan out shape splitting GcsBatchResponse into 2:
+    *   First port emits terminal result that can exit the GcsBatch flow
+    *   Second port emits request to be re-injected to be executed in a later batch
+    */
+  private lazy val responseHandlerFlow = GraphDSL.create() { implicit builder =>
+    import GraphDSL.Implicits._
+
+    val source = builder.add(Partition[GcsBatchResponse[_]](2, {
+      case _: GcsBatchTerminal[_] => 0
+      case _ => 1
+    }))
+
+    // Terminal responses: output of this flow
+    val terminals = source.out(0) collect { case terminal: GcsBatchTerminal[_] => terminal.ioResult }
+
+    // Next command context, can be a retry or another request needed by the command
+    val nextRequest = source.out(1).collect {
+      case retry: GcsBatchRetry[_] => retry.context
+      case nextRequest: GcsBatchNextRequest[_] => nextRequest.context
+    }
+
+    new FanOutShape2[GcsBatchResponse[_], IoResult, GcsBatchCommandContext[_, _]](source.in, terminals.outlet, nextRequest.outlet)
+  }
+  
+  private def executeBatch(contexts: Seq[GcsBatchCommandContext[_, _]]): List[Future[GcsBatchResponse[_]]] = {
+    def failAllPromisesWith(failure: Throwable) = contexts foreach { context =>
+      context.promise.tryFailure(failure)
+      ()
+    }
+
+    // Add all requests to the batch
+    contexts foreach { _.queue(batch) }
+    
+    // Try to execute the batch request. 
+    // If it fails with an IO Exception, fail all the underlying promises with a retyrable BatchFailedException
+    // Otherwise fail with the original exception
+    Try(batch.execute()) match {
+      case Failure(failure: IOException) => failAllPromisesWith(BatchFailedException(failure))
+      case Failure(failure) => failAllPromisesWith(failure)
+      case _ =>
+    }
+
+    // Map all promise responses to a GcsBatchResponse to be either sent back as a response or retried in the next batch
+    contexts.toList map { context =>
+      context.promise.future map {
+        case Left(response) => GcsBatchTerminal(response)
+        case Right(nextRequest) => GcsBatchNextRequest(nextRequest)
+      } recoverWith recoverCommand(context)
+    }
+  }
+
+  /**
+    * Handles a failed future.
+    *   If the failure is retryable, and the command hasn't reached its max attempts:
+    *     schedule the command to be retried in a later batch after waiting for the appropriate amount of time
+    *   Otherwise create a GcsBatchTerminal response with the IoFailure
+    *   In both cases, returns a successful Future to avoid failing the stream or dropping elements
+    */
+  private def recoverCommand(context: GcsBatchCommandContext[_, _]): PartialFunction[Throwable, Future[GcsBatchResponse[_]]] = {
+    // If the failure is retryable - recover with a GcsBatchRetry so it can be retried in the next batch
+    case failure if IoActor.isRetryable(failure) =>
+      context.retryIn match {
+        case Some(waitTime) if IoActor.isTransient(failure) => 
+          akka.pattern.after(waitTime, scheduler)(Future.successful(GcsBatchRetry(context.nextTransient, failure)))
+        case Some(waitTime) => 
+          akka.pattern.after(waitTime, scheduler)(Future.successful(GcsBatchRetry(context.next, failure)))
+        case None => fail(context, failure)
+      }
+      
+    // Otherwise just fail the command
+    case failure => fail(context, failure)
+  }
+
+  /**
+    * Fail a command context with a failure.
+    */
+  private def fail(context: GcsBatchCommandContext[_, _], failure: Throwable) = Future.successful(GcsBatchTerminal(context.fail(failure)))
+}
diff --git a/engine/src/main/scala/cromwell/engine/io/gcs/GcsResponse.scala b/engine/src/main/scala/cromwell/engine/io/gcs/GcsResponse.scala
new file mode 100644
index 000000000..937e2b660
--- /dev/null
+++ b/engine/src/main/scala/cromwell/engine/io/gcs/GcsResponse.scala
@@ -0,0 +1,12 @@
+package cromwell.engine.io.gcs
+
+import cromwell.engine.io.IoActor._
+
+/**
+  * ADT used only inside the batch stream
+  * @tparam T final type of the result of the Command
+  */
+private [gcs] sealed trait GcsBatchResponse[T]
+private [gcs] case class GcsBatchTerminal[T](ioResult: IoResult) extends GcsBatchResponse[T]
+private [gcs] case class GcsBatchRetry[T](context: GcsBatchCommandContext[T, _], failure: Throwable) extends GcsBatchResponse[T]
+private [gcs] case class GcsBatchNextRequest[T](context: GcsBatchCommandContext[T, _]) extends GcsBatchResponse[T]
diff --git a/engine/src/main/scala/cromwell/engine/io/gcs/ParallelGcsBatchFlow.scala b/engine/src/main/scala/cromwell/engine/io/gcs/ParallelGcsBatchFlow.scala
new file mode 100644
index 000000000..633e270d8
--- /dev/null
+++ b/engine/src/main/scala/cromwell/engine/io/gcs/ParallelGcsBatchFlow.scala
@@ -0,0 +1,29 @@
+package cromwell.engine.io.gcs
+
+import akka.actor.Scheduler
+import akka.stream.FlowShape
+import akka.stream.scaladsl.{Balance, GraphDSL, Merge}
+import cromwell.engine.io.IoActor.IoResult
+
+import scala.concurrent.ExecutionContext
+
+/**
+  * Balancer that distributes requests to multiple batch flows in parallel
+  */
+class ParallelGcsBatchFlow(parallelism: Int, batchSize: Int, scheduler: Scheduler)(implicit ec: ExecutionContext) {
+  
+  val flow = GraphDSL.create() { implicit builder =>
+    import GraphDSL.Implicits._
+    val balancer = builder.add(Balance[GcsBatchCommandContext[_, _]](parallelism, waitForAllDownstreams = false))
+    val merge = builder.add(Merge[IoResult](parallelism))
+
+    for (_ <- 1 to parallelism) {
+      val workerFlow = new GcsBatchFlow(batchSize, scheduler).flow
+      // for each worker, add an edge from the balancer to the worker, then wire
+      // it to the merge element
+      balancer ~> workerFlow.async ~> merge
+    }
+
+    FlowShape(balancer.in, merge.out)
+  }
+}
diff --git a/engine/src/main/scala/cromwell/engine/io/nio/NioFlow.scala b/engine/src/main/scala/cromwell/engine/io/nio/NioFlow.scala
new file mode 100644
index 000000000..0ffd549e2
--- /dev/null
+++ b/engine/src/main/scala/cromwell/engine/io/nio/NioFlow.scala
@@ -0,0 +1,93 @@
+package cromwell.engine.io.nio
+
+import akka.actor.{ActorSystem, Scheduler}
+import akka.stream.scaladsl.Flow
+import cromwell.core.io._
+import cromwell.core.path.{DefaultPath, Path}
+import cromwell.core.retry.Retry
+import cromwell.engine.io.IoActor
+import cromwell.engine.io.IoActor.{DefaultCommandContext, IoResult}
+import cromwell.filesystems.gcs.GcsPath
+import cromwell.util.TryWithResource._
+
+import scala.concurrent.{ExecutionContext, Future}
+import scala.io.Codec
+
+/**
+  * Flow that executes IO operations by calling java.nio.Path methods
+  */
+class NioFlow(parallelism: Int, scheduler: Scheduler, nbAttempts: Int = IoActor.MaxAttemptsNumber)(implicit ec: ExecutionContext, actorSystem: ActorSystem) {
+  private val processCommand: DefaultCommandContext[_] => Future[IoResult] = commandContext => {
+    val operationResult = Retry.withRetry(
+      () => handleSingleCommand(commandContext.request),
+      maxRetries = Option(nbAttempts),
+      backoff = IoCommand.defaultBackoff,
+      isTransient = IoActor.isTransient,
+      isFatal = IoActor.isFatal
+    )
+    
+    operationResult map { (_, commandContext) } recoverWith {
+      case failure => Future.successful(commandContext.fail(failure))
+    }
+  }
+  
+  private [nio] def handleSingleCommand(ioSingleCommand: IoCommand[_]) = {
+    ioSingleCommand match {
+      case copyCommand: IoCopyCommand => copy(copyCommand) map copyCommand.success
+      case writeCommand: IoWriteCommand => write(writeCommand) map writeCommand.success
+      case deleteCommand: IoDeleteCommand => delete(deleteCommand) map deleteCommand.success
+      case sizeCommand: IoSizeCommand => size(sizeCommand) map sizeCommand.success
+      case readAsStringCommand: IoContentAsStringCommand => readAsString(readAsStringCommand) map readAsStringCommand.success
+      case hashCommand: IoHashCommand => hash(hashCommand) map hashCommand.success
+      case touchCommand: IoTouchCommand => touch(touchCommand) map touchCommand.success
+      case _ => Future.failed(new NotImplementedError("Method not implemented"))
+    }
+  }
+  
+  val flow = Flow[DefaultCommandContext[_]].mapAsyncUnordered[IoResult](parallelism)(processCommand)
+  
+  private def copy(copy: IoCopyCommand) = Future {
+    createDirectoriesForSFSPath(copy.destination)
+    copy.source.copyTo(copy.destination, copy.overwrite)
+    ()
+  }
+
+  private def write(write: IoWriteCommand) = Future {
+    createDirectoriesForSFSPath(write.file)
+    write.file.write(write.content)(write.openOptions, Codec.UTF8)
+    ()
+  }
+  
+  private def delete(delete: IoDeleteCommand) = Future {
+    delete.file.delete(delete.swallowIOExceptions)
+    ()
+  }
+
+  private def readAsString(read: IoContentAsStringCommand) = Future {
+    read.file.contentAsString
+  }
+  
+  private def size(size: IoSizeCommand) = Future {
+    size.file.size
+  }
+
+  private def hash(hash: IoHashCommand) = {
+    hash.file match {
+      case gcsPath: GcsPath => Future { gcsPath.cloudStorage.get(gcsPath.blob).getCrc32c }
+      case path => Future.fromTry(
+        tryWithResource(() => path.newInputStream) { inputStream =>
+          org.apache.commons.codec.digest.DigestUtils.md5Hex(inputStream)
+        }
+      )
+    }
+  }
+
+  private def touch(touch: IoTouchCommand) = Future {
+    touch.file.touch()
+  }
+  
+  private def createDirectoriesForSFSPath(path: Path) = path match {
+    case _: DefaultPath => path.parent.createPermissionedDirectories()
+    case _ =>
+  }
+}
diff --git a/engine/src/main/scala/cromwell/engine/package.scala b/engine/src/main/scala/cromwell/engine/package.scala
index 6b2df1cc7..198645503 100644
--- a/engine/src/main/scala/cromwell/engine/package.scala
+++ b/engine/src/main/scala/cromwell/engine/package.scala
@@ -1,8 +1,8 @@
 package cromwell
 
 import cromwell.core.JobOutput
-import wdl4s._
-import wdl4s.values.WdlValue
+import wdl4s.wdl._
+import wdl4s.wdl.values.WdlValue
 
 package object engine {
 
diff --git a/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala b/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala
index ebf5df088..995dd9551 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala
@@ -4,6 +4,7 @@ import java.util.UUID
 
 import akka.actor.FSM.{CurrentState, Transition}
 import akka.actor._
+import akka.stream.ActorMaterializer
 import cats.instances.try_._
 import cats.syntax.functor._
 import cromwell.core.Dispatcher.EngineDispatcher
@@ -17,10 +18,10 @@ import cromwell.engine.workflow.workflowstore.{InMemoryWorkflowStore, WorkflowSt
 import cromwell.jobstore.EmptyJobStoreActor
 import cromwell.server.CromwellRootActor
 import cromwell.services.metadata.MetadataService.{GetSingleWorkflowMetadataAction, GetStatus, WorkflowOutputs}
+import cromwell.services.metadata.impl.WriteMetadataActor.{CheckPendingWrites, HasPendingWrites, NoPendingWrites}
 import cromwell.subworkflowstore.EmptySubWorkflowStoreActor
-import cromwell.webservice.PerRequest.RequestComplete
 import cromwell.webservice.metadata.MetadataBuilderActor
-import spray.http.StatusCodes
+import cromwell.webservice.metadata.MetadataBuilderActor.{BuiltMetadataResponse, FailedMetadataResponse}
 import spray.json._
 
 import scala.concurrent.ExecutionContext.Implicits.global
@@ -32,15 +33,18 @@ import scala.util.{Failure, Try}
  * Designed explicitly for the use case of the 'run' functionality in Main. This Actor will start a workflow,
  * print out the outputs when complete and reply with a result.
  */
-class SingleWorkflowRunnerActor(source: WorkflowSourceFilesCollection, metadataOutputPath: Option[Path])
-  extends CromwellRootActor with LoggingFSM[RunnerState, SwraData] {
+class SingleWorkflowRunnerActor(source: WorkflowSourceFilesCollection,
+                                metadataOutputPath: Option[Path],
+                                gracefulShutdown: Boolean,
+                                abortJobsOnTerminate: Boolean
+                                )(implicit materializer: ActorMaterializer)
+  extends CromwellRootActor(gracefulShutdown, abortJobsOnTerminate) with LoggingFSM[RunnerState, SwraData] {
 
   override val serverMode = false
 
   import SingleWorkflowRunnerActor._
   private val backoff = SimpleExponentialBackoff(1 second, 1 minute, 1.2)
 
-  override val abortJobsOnTerminate = true
   override lazy val workflowStore = new InMemoryWorkflowStore()
   override lazy val jobStoreActor = context.actorOf(EmptyJobStoreActor.props)
   override lazy val subWorkflowStoreActor = context.actorOf(EmptySubWorkflowStoreActor.props)
@@ -67,31 +71,44 @@ class SingleWorkflowRunnerActor(source: WorkflowSourceFilesCollection, metadataO
     case Event(IssuePollRequest, RunningSwraData(_, id)) =>
       requestStatus(id)
       stay()
-    case Event(RequestComplete((StatusCodes.OK, jsObject: JsObject)), RunningSwraData(_, _)) if !jsObject.state.isTerminal =>
+    case Event(BuiltMetadataResponse(jsObject: JsObject), RunningSwraData(_, _)) if !jsObject.state.isTerminal =>
       schedulePollRequest()
       stay()
-    case Event(RequestComplete((StatusCodes.OK, jsObject: JsObject)), RunningSwraData(replyTo, id)) if jsObject.state == WorkflowSucceeded =>
-      val metadataBuilder = context.actorOf(MetadataBuilderActor.props(serviceRegistryActor),
-        s"CompleteRequest-Workflow-$id-request-${UUID.randomUUID()}")
-      metadataBuilder ! WorkflowOutputs(id)
+    case Event(BuiltMetadataResponse(jsObject: JsObject), RunningSwraData(replyTo, id)) if jsObject.state == WorkflowSucceeded =>
       log.info(s"$Tag workflow finished with status '$WorkflowSucceeded'.")
-      goto(RequestingOutputs) using SucceededSwraData(replyTo, id)
-    case Event(RequestComplete((StatusCodes.OK, jsObject: JsObject)), RunningSwraData(replyTo, id)) if jsObject.state == WorkflowFailed =>
+      serviceRegistryActor ! CheckPendingWrites
+      goto(WaitingForFlushedMetadata) using SucceededSwraData(replyTo, id)
+    case Event(BuiltMetadataResponse(jsObject: JsObject), RunningSwraData(replyTo, id)) if jsObject.state == WorkflowFailed =>
       log.info(s"$Tag workflow finished with status '$WorkflowFailed'.")
-      requestMetadataOrIssueReply(FailedSwraData(replyTo, id, new RuntimeException(s"Workflow $id transitioned to state $WorkflowFailed")))
-    case Event(RequestComplete((StatusCodes.OK, jsObject: JsObject)), RunningSwraData(replyTo, id)) if jsObject.state == WorkflowAborted =>
+      serviceRegistryActor ! CheckPendingWrites
+      goto(WaitingForFlushedMetadata) using FailedSwraData(replyTo, id, new RuntimeException(s"Workflow $id transitioned to state $WorkflowFailed"))
+    case Event(BuiltMetadataResponse(jsObject: JsObject), RunningSwraData(replyTo, id)) if jsObject.state == WorkflowAborted =>
       log.info(s"$Tag workflow finished with status '$WorkflowAborted'.")
-      requestMetadataOrIssueReply(AbortedSwraData(replyTo, id))
+      serviceRegistryActor ! CheckPendingWrites
+      goto(WaitingForFlushedMetadata) using AbortedSwraData(replyTo, id)
+  }
+  
+  when (WaitingForFlushedMetadata) {
+    case Event(HasPendingWrites, _) => 
+      context.system.scheduler.scheduleOnce(1 second, serviceRegistryActor, CheckPendingWrites)(context.system.dispatcher, self)
+      stay()
+    case Event(NoPendingWrites, data: SucceededSwraData) =>
+      val metadataBuilder = context.actorOf(MetadataBuilderActor.props(serviceRegistryActor),
+        s"CompleteRequest-Workflow-${data.id}-request-${UUID.randomUUID()}")
+      metadataBuilder ! WorkflowOutputs(data.id)
+      goto(RequestingOutputs)
+    case Event(NoPendingWrites, data : TerminalSwraData) =>
+      requestMetadataOrIssueReply(data)
   }
 
   when (RequestingOutputs) {
-    case Event(RequestComplete((StatusCodes.OK, outputs: JsObject)), data: TerminalSwraData) =>
+    case Event(BuiltMetadataResponse(outputs: JsObject), data: TerminalSwraData) =>
       outputOutputs(outputs)
       requestMetadataOrIssueReply(data)
   }
 
   when (RequestingMetadata) {
-    case Event(RequestComplete((StatusCodes.OK, metadata: JsObject)), data: TerminalSwraData) =>
+    case Event(BuiltMetadataResponse(metadata: JsObject), data: TerminalSwraData) =>
       outputMetadata(metadata)
       issueReply(data)
   }
@@ -105,7 +122,7 @@ class SingleWorkflowRunnerActor(source: WorkflowSourceFilesCollection, metadataO
     case Event(r: WorkflowStoreEngineActor.WorkflowAbortFailed, data) => failAndFinish(r.reason, data)
     case Event(Failure(e), data) => failAndFinish(e, data)
     case Event(Status.Failure(e), data) => failAndFinish(e, data)
-    case Event(RequestComplete((_, snap)), data) => failAndFinish(new RuntimeException(s"Unexpected API completion message: $snap"), data)
+    case Event(FailedMetadataResponse(e), data) => failAndFinish(e, data)
     case Event((CurrentState(_, _) | Transition(_, _, _)), _) =>
       // ignore uninteresting current state and transition messages
       stay()
@@ -194,8 +211,11 @@ class SingleWorkflowRunnerActor(source: WorkflowSourceFilesCollection, metadataO
 }
 
 object SingleWorkflowRunnerActor {
-  def props(source: WorkflowSourceFilesCollection, metadataOutputFile: Option[Path]): Props = {
-    Props(new SingleWorkflowRunnerActor(source, metadataOutputFile)).withDispatcher(EngineDispatcher)
+  def props(source: WorkflowSourceFilesCollection,
+            metadataOutputFile: Option[Path],
+            gracefulShutdown: Boolean,
+            abortJobsOnTerminate: Boolean)(implicit materializer: ActorMaterializer): Props = {
+    Props(new SingleWorkflowRunnerActor(source, metadataOutputFile, gracefulShutdown, abortJobsOnTerminate)).withDispatcher(EngineDispatcher)
   }
 
   sealed trait RunnerMessage
@@ -208,6 +228,7 @@ object SingleWorkflowRunnerActor {
   case object NotStarted extends RunnerState
   case object SubmittedWorkflow extends RunnerState
   case object RunningWorkflow extends RunnerState
+  case object WaitingForFlushedMetadata extends RunnerState
   case object RequestingOutputs extends RunnerState
   case object RequestingMetadata extends RunnerState
 
@@ -228,7 +249,7 @@ object SingleWorkflowRunnerActor {
                                    id: WorkflowId) extends TerminalSwraData { override val terminalState = WorkflowAborted }
 
   implicit class EnhancedJsObject(val jsObject: JsObject) extends AnyVal {
-    def state: WorkflowState = WorkflowState.fromString(jsObject.fields("status").asInstanceOf[JsString].value)
+    def state: WorkflowState = WorkflowState.withName(jsObject.fields("status").asInstanceOf[JsString].value)
   }
 
   private val Tag = "SingleWorkflowRunnerActor"
diff --git a/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala b/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala
index ad97eb0f5..16e8f08f0 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala
@@ -16,12 +16,12 @@ import cromwell.engine.workflow.lifecycle.MaterializeWorkflowDescriptorActor.{Ma
 import cromwell.engine.workflow.lifecycle.WorkflowFinalizationActor.{StartFinalizationCommand, WorkflowFinalizationFailedResponse, WorkflowFinalizationSucceededResponse}
 import cromwell.engine.workflow.lifecycle.WorkflowInitializationActor.{StartInitializationCommand, WorkflowInitializationFailedResponse, WorkflowInitializationSucceededResponse}
 import cromwell.engine.workflow.lifecycle._
-import cromwell.engine.workflow.lifecycle.execution.{WorkflowExecutionActor, WorkflowMetadataHelper}
 import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor._
+import cromwell.engine.workflow.lifecycle.execution.{WorkflowExecutionActor, WorkflowMetadataHelper}
 import cromwell.subworkflowstore.SubWorkflowStoreActor.WorkflowComplete
 import cromwell.webservice.EngineStatsActor
-import wdl4s.{LocallyQualifiedName => _}
 
+import scala.concurrent.Future
 import scala.util.Failure
 
 object WorkflowActor {
@@ -134,19 +134,36 @@ object WorkflowActor {
 
   def props(workflowId: WorkflowId,
             startMode: StartMode,
-            wdlSource: WorkflowSourceFilesCollection,
+            workflowSourceFilesCollection: WorkflowSourceFilesCollection,
             conf: Config,
+            ioActor: ActorRef,
             serviceRegistryActor: ActorRef,
             workflowLogCopyRouter: ActorRef,
             jobStoreActor: ActorRef,
             subWorkflowStoreActor: ActorRef,
             callCacheReadActor: ActorRef,
+            callCacheWriteActor: ActorRef,
             dockerHashActor: ActorRef,
             jobTokenDispenserActor: ActorRef,
             backendSingletonCollection: BackendSingletonCollection,
             serverMode: Boolean): Props = {
-    Props(new WorkflowActor(workflowId, startMode, wdlSource, conf, serviceRegistryActor, workflowLogCopyRouter,
-      jobStoreActor, subWorkflowStoreActor, callCacheReadActor, dockerHashActor, jobTokenDispenserActor, backendSingletonCollection, serverMode)).withDispatcher(EngineDispatcher)
+    Props(
+      new WorkflowActor(
+        workflowId = workflowId,
+        startMode = startMode,
+        workflowSourceFilesCollection = workflowSourceFilesCollection,
+        conf = conf,
+        ioActor = ioActor,
+        serviceRegistryActor = serviceRegistryActor,
+        workflowLogCopyRouter = workflowLogCopyRouter,
+        jobStoreActor = jobStoreActor,
+        subWorkflowStoreActor = subWorkflowStoreActor,
+        callCacheReadActor = callCacheReadActor,
+        callCacheWriteActor = callCacheWriteActor,
+        dockerHashActor = dockerHashActor,
+        jobTokenDispenserActor = jobTokenDispenserActor,
+        backendSingletonCollection = backendSingletonCollection,
+        serverMode = serverMode)).withDispatcher(EngineDispatcher)
   }
 }
 
@@ -155,13 +172,15 @@ object WorkflowActor {
   */
 class WorkflowActor(val workflowId: WorkflowId,
                     startMode: StartMode,
-                    workflowSources: WorkflowSourceFilesCollection,
+                    workflowSourceFilesCollection: WorkflowSourceFilesCollection,
                     conf: Config,
+                    ioActor: ActorRef,
                     override val serviceRegistryActor: ActorRef,
                     workflowLogCopyRouter: ActorRef,
                     jobStoreActor: ActorRef,
                     subWorkflowStoreActor: ActorRef,
                     callCacheReadActor: ActorRef,
+                    callCacheWriteActor: ActorRef,
                     dockerHashActor: ActorRef,
                     jobTokenDispenserActor: ActorRef,
                     backendSingletonCollection: BackendSingletonCollection,
@@ -171,10 +190,18 @@ class WorkflowActor(val workflowId: WorkflowId,
   implicit val ec = context.dispatcher
   override val workflowIdForLogging = workflowId
 
+  private val restarting = startMode match {
+    case StartNewWorkflow => false
+    case RestartExistingWorkflow => true
+  }
+
+  private val workflowDockerLookupActor = context.actorOf(
+    WorkflowDockerLookupActor.props(workflowId, dockerHashActor, startMode), s"WorkflowDockerLookupActor-$workflowId")
+
   startWith(WorkflowUnstartedState, WorkflowActorData.empty)
 
   pushCurrentStateToMetadataService(workflowId, WorkflowUnstartedState.workflowState)
-  
+
   override def supervisorStrategy: SupervisorStrategy = OneForOneStrategy() { case _ => Escalate }
 
   when(WorkflowUnstartedState) {
@@ -182,39 +209,36 @@ class WorkflowActor(val workflowId: WorkflowId,
       val actor = context.actorOf(MaterializeWorkflowDescriptorActor.props(serviceRegistryActor, workflowId, importLocalFilesystem = !serverMode),
         "MaterializeWorkflowDescriptorActor")
       pushWorkflowStart(workflowId)
-      actor ! MaterializeWorkflowDescriptorCommand(workflowSources, conf)
+      actor ! MaterializeWorkflowDescriptorCommand(workflowSourceFilesCollection, conf)
       goto(MaterializingWorkflowDescriptorState) using stateData.copy(currentLifecycleStateActor = Option(actor))
-    case Event(AbortWorkflowCommand, stateData) => goto(WorkflowAbortedState)
+    case Event(AbortWorkflowCommand, _) => goto(WorkflowAbortedState)
   }
 
   when(MaterializingWorkflowDescriptorState) {
     case Event(MaterializeWorkflowDescriptorSuccessResponse(workflowDescriptor), data) =>
-      val initializerActor = context.actorOf(WorkflowInitializationActor.props(workflowId, workflowDescriptor, serviceRegistryActor),
+      val initializerActor = context.actorOf(WorkflowInitializationActor.props(workflowId, workflowDescriptor, ioActor, serviceRegistryActor, restarting),
         name = s"WorkflowInitializationActor-$workflowId")
       initializerActor ! StartInitializationCommand
       goto(InitializingWorkflowState) using data.copy(currentLifecycleStateActor = Option(initializerActor), workflowDescriptor = Option(workflowDescriptor))
     case Event(MaterializeWorkflowDescriptorFailureResponse(reason: Throwable), data) =>
       goto(WorkflowFailedState) using data.copy(lastStateReached = StateCheckpoint(MaterializingWorkflowDescriptorState, Option(List(reason))))
-    case Event(AbortWorkflowCommand, stateData) =>
+    case Event(AbortWorkflowCommand, _) =>
       // No lifecycle sub-actors exist yet, so no indirection via WorkflowAbortingState is necessary:
       goto(WorkflowAbortedState)
   }
 
   when(InitializingWorkflowState) {
     case Event(WorkflowInitializationSucceededResponse(initializationData), data @ WorkflowActorData(_, Some(workflowDescriptor), _, _)) =>
-      val restarting = startMode match {
-        case StartNewWorkflow => false
-        case RestartExistingWorkflow => true
-      }
-
       val executionActor = context.actorOf(WorkflowExecutionActor.props(
         workflowDescriptor,
-        serviceRegistryActor,
-        jobStoreActor,
-        subWorkflowStoreActor,
-        callCacheReadActor,
-        dockerHashActor,
-        jobTokenDispenserActor,
+        ioActor = ioActor,
+        serviceRegistryActor = serviceRegistryActor,
+        jobStoreActor = jobStoreActor,
+        subWorkflowStoreActor = subWorkflowStoreActor,
+        callCacheReadActor = callCacheReadActor,
+        callCacheWriteActor = callCacheWriteActor,
+        workflowDockerLookupActor = workflowDockerLookupActor,
+        jobTokenDispenserActor = jobTokenDispenserActor,
         backendSingletonCollection,
         initializationData,
         restarting = restarting), name = s"WorkflowExecutionActor-$workflowId")
@@ -250,7 +274,7 @@ class WorkflowActor(val workflowId: WorkflowId,
   }
 
   when(WorkflowAbortingState) {
-    case Event(x: EngineLifecycleStateCompleteResponse, data @ WorkflowActorData(_, Some(workflowDescriptor), _, _)) =>
+    case Event(_: EngineLifecycleStateCompleteResponse, data @ WorkflowActorData(_, Some(workflowDescriptor), _, _)) =>
       finalizeWorkflow(data, workflowDescriptor, Map.empty, Map.empty, failures = None)
     case _ => stay()
   }
@@ -273,17 +297,6 @@ class WorkflowActor(val workflowId: WorkflowId,
   }
 
   onTransition {
-    case fromState -> toState =>
-      workflowLogger.debug(s"transitioning from {} to {}", arg1 = fromState, arg2 = toState)
-      // This updates the workflow status
-      // Only publish "External" state to metadata service
-      // workflowState maps a state to an "external" state (e.g all states extending WorkflowActorRunningState map to WorkflowRunning)
-      if (fromState.workflowState != toState.workflowState) {
-        pushCurrentStateToMetadataService(workflowId, toState.workflowState)
-      }
-  }
-
-  onTransition {
     case (oldState, terminalState: WorkflowActorTerminalState) =>
       workflowLogger.debug(s"transition from {} to {}. Stopping self.", arg1 = oldState, arg2 = terminalState)
       pushWorkflowEnd(workflowId)
@@ -303,39 +316,65 @@ class WorkflowActor(val workflowId: WorkflowId,
           * being recreated so that in case MaterializeWorkflowDescriptor fails, the workflow logs can still
           * be copied by accessing the workflow options outside of the EngineWorkflowDescriptor.
           */
-        def bruteForceWorkflowOptions: WorkflowOptions = WorkflowOptions.fromJsonString(workflowSources.workflowOptionsJson).getOrElse(WorkflowOptions.fromJsonString("{}").get)
-        def bruteForcePathBuilders: List[PathBuilder] = EngineFilesystems(context.system).pathBuildersForWorkflow(bruteForceWorkflowOptions)
+        def bruteForceWorkflowOptions: WorkflowOptions = WorkflowOptions.fromJsonString(workflowSourceFilesCollection.workflowOptionsJson).getOrElse(WorkflowOptions.fromJsonString("{}").get)
+        val system = context.system
+        val ec = context.system.dispatcher
+        def bruteForcePathBuilders: Future[List[PathBuilder]] = EngineFilesystems.pathBuildersForWorkflow(bruteForceWorkflowOptions)(system, ec)
 
         val (workflowOptions, pathBuilders) = stateData.workflowDescriptor match {
-          case Some(wd) => (wd.backendDescriptor.workflowOptions, wd.pathBuilders)
+          case Some(wd) => (wd.backendDescriptor.workflowOptions, Future.successful(wd.pathBuilders))
           case None => (bruteForceWorkflowOptions, bruteForcePathBuilders)
         }
 
         workflowOptions.get(FinalWorkflowLogDir).toOption match {
-            case Some(destinationDir) =>
-              workflowLogCopyRouter ! CopyWorkflowLogsActor.Copy(workflowId, PathFactory.buildPath(destinationDir, pathBuilders))
-            case None if WorkflowLogger.isTemporary => workflowLogger.deleteLogFile() match {
-              case Failure(f) => log.error(f, "Failed to delete workflow log")
-              case _ =>
-            }
+          case Some(destinationDir) =>
+            pathBuilders.map(pb => workflowLogCopyRouter ! CopyWorkflowLogsActor.Copy(workflowId, PathFactory.buildPath(destinationDir, pb)))(ec)
+          case None if WorkflowLogger.isTemporary => workflowLogger.deleteLogFile() match {
+            case Failure(f) => log.error(f, "Failed to delete workflow log")
             case _ =>
           }
+          case _ =>
         }
+      }
       context stop self
   }
 
+  onTransition {
+    case fromState -> toState =>
+      workflowLogger.debug(s"transitioning from {} to {}", arg1 = fromState, arg2 = toState)
+      // This updates the workflow status
+      // Only publish "External" state to metadata service
+      // workflowState maps a state to an "external" state (e.g all states extending WorkflowActorRunningState map to WorkflowRunning)
+      if (fromState.workflowState != toState.workflowState) {
+        pushCurrentStateToMetadataService(workflowId, toState.workflowState)
+      }
+  }
+
   private def finalizationSucceeded(data: WorkflowActorData) = {
     val finalState = data.lastStateReached match {
       case StateCheckpoint(WorkflowAbortingState, None) => WorkflowAbortedState
-      case StateCheckpoint(state, Some(failures)) => WorkflowFailedState
-      case StateCheckpoint(state, None) => WorkflowSucceededState
+      case StateCheckpoint(_, Some(_)) => WorkflowFailedState
+      case StateCheckpoint(_, None) => WorkflowSucceededState
     }
 
     goto(finalState) using data.copy(currentLifecycleStateActor = None)
   }
 
   private[workflow] def makeFinalizationActor(workflowDescriptor: EngineWorkflowDescriptor, jobExecutionMap: JobExecutionMap, workflowOutputs: CallOutputs) = {
-    context.actorOf(WorkflowFinalizationActor.props(workflowId, workflowDescriptor, jobExecutionMap, workflowOutputs, stateData.initializationData), name = s"WorkflowFinalizationActor")
+    val copyWorkflowOutputsActorProps = stateName match {
+      case InitializingWorkflowState => None
+      case _ => Option(CopyWorkflowOutputsActor.props(workflowIdForLogging, ioActor, workflowDescriptor, workflowOutputs, stateData.initializationData))
+    }
+    
+    context.actorOf(WorkflowFinalizationActor.props(
+      workflowId = workflowId,
+      workflowDescriptor = workflowDescriptor,
+      ioActor = ioActor,
+      jobExecutionMap = jobExecutionMap,
+      workflowOutputs = workflowOutputs,
+      initializationData = stateData.initializationData,
+      copyWorkflowOutputsActor = copyWorkflowOutputsActorProps
+    ), name = s"WorkflowFinalizationActor")
   }
   /**
     * Run finalization actor and transition to FinalizingWorkflowState.
diff --git a/engine/src/main/scala/cromwell/engine/workflow/WorkflowDockerLookupActor.scala b/engine/src/main/scala/cromwell/engine/workflow/WorkflowDockerLookupActor.scala
new file mode 100644
index 000000000..54c033a7a
--- /dev/null
+++ b/engine/src/main/scala/cromwell/engine/workflow/WorkflowDockerLookupActor.scala
@@ -0,0 +1,290 @@
+package cromwell.engine.workflow
+
+import akka.actor.{ActorRef, LoggingFSM, Props}
+import cromwell.core.{Dispatcher, WorkflowId}
+import cromwell.database.sql.SqlDatabase
+import cromwell.database.sql.tables.DockerHashStoreEntry
+import cromwell.docker.DockerHashActor.{DockerHashFailureResponse, DockerHashSuccessResponse}
+import cromwell.docker.{DockerClientHelper, DockerHashRequest, DockerHashResult, DockerImageIdentifier}
+import cromwell.engine.workflow.WorkflowActor.{RestartExistingWorkflow, StartMode}
+import cromwell.engine.workflow.WorkflowDockerLookupActor._
+import cromwell.services.SingletonServicesStore
+import cromwell.core.Dispatcher.EngineDispatcher
+
+import lenthall.util.TryUtil
+
+import scala.concurrent.duration._
+import scala.language.postfixOps
+import scala.util.{Failure, Success}
+
+/**
+  * Ensures docker hash consistency throughout a workflow.
+  *
+  * Caches successful docker hash lookups and serves them to subsequent identical requests.
+  * Persists those hashes in the database to be resilient to server restarts.
+  *
+  * Failure modes:
+  * 1) Failure to load hashes from the DB upon restart.
+  * 2) Failure to parse hashes from the DB upon restart.
+  * 3) Failure to write a hash result to the DB.
+  * 4) Failure to look up a docker hash.
+  * 5) Timeout from DockerHashActor.
+  *
+  * Behavior:
+  * 1-2) Return a terminal lookup failure for all pending requests, transition to a permanently Failed state in which any
+  *      future requests will immediately return lookup failure.  The JobPreparation actor should fail in response to this
+  *      lookup termination message, which in turn should fail the workflow.
+  * 3-5) Return lookup failure for the current request and all pending requests for the same tag.  Any future requests
+  *      for this tag will be attempted again.
+  */
+
+class WorkflowDockerLookupActor private[workflow](workflowId: WorkflowId, val dockerHashingActor: ActorRef, startMode: StartMode, databaseInterface: SqlDatabase)
+  extends LoggingFSM[WorkflowDockerLookupActorState, WorkflowDockerLookupActorData] with DockerClientHelper {
+
+  implicit val ec = context.system.dispatchers.lookup(Dispatcher.EngineDispatcher)
+
+  // Amount of time after which the docker request should be considered lost and sent again.
+  override protected def backpressureTimeout: FiniteDuration = 10 seconds
+  // A multiplier for the amount of time to wait when we get a Backpressure response before sending the request again.
+  // This effectively bounds the jitter.
+  override protected def backpressureRandomizerFactor: Double = 0.5D
+
+  context.become(dockerReceive orElse receive)
+
+  startWith(
+    stateName = if (startMode == RestartExistingWorkflow) AwaitingFirstRequestOnRestart else Running,
+    stateData = WorkflowDockerLookupActorData.empty
+  )
+
+  // `AwaitingFirstRequestOnRestart` is only used in restart scenarios.  This state waits until there's at least one hash
+  // request before trying to load the docker hash mappings.  This is so we'll have at least one `JobPreparationActor`
+  // reference available to message with a terminal failure in case the reading or parsing of these mappings fails.
+  when(AwaitingFirstRequestOnRestart) {
+    case Event(request: DockerHashRequest, data) =>
+      loadDockerHashStoreEntries()
+      goto(LoadingCache) using data.addHashRequest(request, sender())
+  }
+
+  // Waiting for a response from the database with the hash mapping for this workflow.
+  when(LoadingCache) {
+    case Event(DockerHashStoreLoadingSuccess(dockerHashEntries), data) =>
+      loadCacheAndHandleHashRequests(dockerHashEntries, data)
+    case Event(request: DockerHashRequest, data) =>
+      stay using data.addHashRequest(request, sender())
+  }
+
+  // This is the normal operational mode.
+  when(Running) {
+    // This tag has already been looked up and its hash is in the mappings cache.
+    case Event(request: DockerHashRequest, data) if data.mappings.contains(request.dockerImageID) =>
+      sender ! DockerHashSuccessResponse(data.mappings(request.dockerImageID), request)
+      stay()
+    // A request for the hash for this tag has already been made to the hashing actor.  Don't request the hash again,
+    // just add this sender to the list of replyTos for when the hash arrives.
+    case Event(request: DockerHashRequest, data) if data.hashRequests.contains(request) =>
+      stay using data.addHashRequest(request, sender())
+    // This tag has not (successfully) been looked up before, so look it up now.
+    case Event(request: DockerHashRequest, data) =>
+      requestDockerHash(request, data)
+    case Event(dockerResponse: DockerHashSuccessResponse, data) =>
+      persistDockerHash(dockerResponse, data)
+      stay()
+    case Event(dockerResponse: DockerHashFailureResponse, data) =>
+      handleLookupFailure(dockerResponse, data)
+    case Event(DockerHashStoreSuccess(response), data) =>
+      recordMappingAndRespond(response, data)
+    case Event(DockerHashStoreFailure(request, e), data) =>
+      handleStoreFailure(request, new Exception(s"Failure storing docker hash for ${request.dockerImageID.fullName}", e), data)
+  }
+
+  // In state Terminal we reject all requests with the cause set in the state data.
+  when(Terminal) {
+    case Event(request: DockerHashRequest, data) =>
+      sender() ! WorkflowDockerLookupFailure(data.failureCause.orNull, request)
+      stay()
+  }
+
+  private def fail(reason: Throwable): State = {
+    self ! TransitionToFailed(reason)
+    stay()
+  }
+
+  whenUnhandled {
+    case Event(DockerHashActorTimeout(request), data) =>
+      val reason = new Exception(s"Timeout looking up docker hash")
+      data.hashRequests(request) foreach { _ ! WorkflowDockerLookupFailure(reason, request) }
+      val updatedData = data.copy(hashRequests = data.hashRequests - request)
+      stay() using updatedData
+    case Event(TransitionToFailed(cause), data) =>
+      log.error(cause, s"Workflow Docker lookup actor for $workflowId transitioning to Failed")
+      val updatedData = respondToAllRequestsWithTerminalFailure(FailedException, data)
+      goto(Terminal) using updatedData.withFailureCause(FailedException)
+  }
+
+  /**
+    * Load mappings from the database into the state data, reply to queued requests which have mappings, and initiate
+    * hash lookups for requests which don't have mappings.
+    */
+  private def loadCacheAndHandleHashRequests(hashEntries: Map[String, String], data: WorkflowDockerLookupActorData): State = {
+    val dockerMappingsTry = hashEntries map {
+      case (dockerTag, dockerHash) => DockerImageIdentifier.fromString(dockerTag) -> DockerHashResult.fromString(dockerHash)
+    }
+
+    TryUtil.sequenceKeyValues(dockerMappingsTry) match {
+      case Success(dockerMappings) =>
+        // Figure out which of the queued requests already have established mappings.
+        val (hasMappings, doesNotHaveMappings) = data.hashRequests.partition { case (request, _) => dockerMappings.contains(request.dockerImageID) }
+
+        // The requests which have mappings receive success responses.
+        hasMappings foreach { case (request, replyTos) =>
+          val result = dockerMappings(request.dockerImageID)
+          replyTos foreach { _ ! DockerHashSuccessResponse(result, request)}
+        }
+
+        // The requests without mappings need to be looked up.
+        doesNotHaveMappings.keys foreach { sendDockerCommand(_) }
+
+        // Update state data accordingly.
+        val newData = data.copy(hashRequests = doesNotHaveMappings, mappings = dockerMappings, failureCause = None)
+        goto(Running) using newData
+
+      case Failure(e) =>
+        fail(new Exception("Failed to parse docker tag -> hash mappings from DB", e))
+    }
+  }
+
+  private def requestDockerHash(request: DockerHashRequest, data: WorkflowDockerLookupActorData): State = {
+    sendDockerCommand(request)
+    val replyTo = sender()
+    val updatedData = data.copy(hashRequests = data.hashRequests + (request -> List(replyTo)))
+    stay using updatedData
+  }
+
+  private def recordMappingAndRespond(response: DockerHashSuccessResponse, data: WorkflowDockerLookupActorData): State = {
+    // Add the new label to hash mapping to the current set of mappings.
+    val request = response.request
+    data.hashRequests(request) foreach { _ ! DockerHashSuccessResponse(response.dockerHash, request) }
+    val updatedData = data.copy(hashRequests = data.hashRequests - request, mappings = data.mappings + (request.dockerImageID -> response.dockerHash))
+    stay using updatedData
+  }
+
+  private def respondToAllRequests(reason: Throwable,
+                                   data: WorkflowDockerLookupActorData,
+                                   messageBuilder: (Throwable, DockerHashRequest) => WorkflowDockerLookupResponse): WorkflowDockerLookupActorData = {
+    data.hashRequests foreach { case (request, replyTos) =>
+      replyTos foreach { _ ! messageBuilder(reason, request) }
+    }
+    data.clearHashRequests
+  }
+
+  private def respondToAllRequestsWithTerminalFailure(reason: Throwable, data: WorkflowDockerLookupActorData): WorkflowDockerLookupActorData = {
+    respondToAllRequests(reason, data, WorkflowDockerTerminalFailure.apply)
+  }
+
+  private def persistDockerHash(response: DockerHashSuccessResponse, data: WorkflowDockerLookupActorData): Unit = {
+    val dockerHashStoreEntry = DockerHashStoreEntry(workflowId.toString, response.request.dockerImageID.fullName, response.dockerHash.algorithmAndHash)
+    databaseInterface.addDockerHashStoreEntry(dockerHashStoreEntry) onComplete {
+      case Success(_) => self ! DockerHashStoreSuccess(response)
+      case Failure(ex) => self ! DockerHashStoreFailure(response.request, ex)
+    }
+  }
+
+  private def handleLookupFailure(dockerResponse: DockerHashFailureResponse, data: WorkflowDockerLookupActorData): State = {
+    // Fail all pending requests.  This logic does not blacklist the tag, which will allow lookups to be attempted
+    // again in the future.
+    val failureResponse = WorkflowDockerLookupFailure(new Exception(dockerResponse.reason), dockerResponse.request)
+    val request = dockerResponse.request
+    data.hashRequests(request) foreach { _ ! failureResponse }
+
+    val updatedData = data.copy(hashRequests = data.hashRequests - request)
+    stay using updatedData
+  }
+
+  private def handleStoreFailure(dockerHashRequest: DockerHashRequest, reason: Throwable, data: WorkflowDockerLookupActorData): State = {
+    data.hashRequests(dockerHashRequest) foreach { _ ! WorkflowDockerLookupFailure(reason, dockerHashRequest) }
+    // Remove these requesters from the collection of those awaiting hashes.
+    stay() using data.copy(hashRequests = data.hashRequests - dockerHashRequest)
+  }
+
+  def loadDockerHashStoreEntries(): Unit = {
+    databaseInterface.queryDockerHashStoreEntries(workflowId.toString) onComplete {
+      case Success(dockerHashEntries) =>
+        val dockerMappings = dockerHashEntries.map(entry => entry.dockerTag -> entry.dockerHash).toMap
+        self ! DockerHashStoreLoadingSuccess(dockerMappings)
+      case Failure(ex) =>
+        fail(new RuntimeException("Failed to load docker tag -> hash mappings from DB", ex))
+    }
+  }
+
+  override protected def onTimeout(message: Any, to: ActorRef): Unit = {
+    message match {
+      case r: DockerHashRequest => self ! DockerHashActorTimeout(r)
+    }
+  }
+}
+
+object WorkflowDockerLookupActor {
+  /* States */
+  sealed trait WorkflowDockerLookupActorState
+  case object AwaitingFirstRequestOnRestart extends WorkflowDockerLookupActorState
+  case object LoadingCache extends WorkflowDockerLookupActorState
+  case object Running extends WorkflowDockerLookupActorState
+  case object Terminal extends WorkflowDockerLookupActorState
+  private val FailedException =
+    new Exception(s"The WorkflowDockerLookupActor has failed. Subsequent docker tags for this workflow will not be resolved.")
+
+  /* Internal ADTs */
+  final case class DockerRequestContext(dockerHashRequest: DockerHashRequest, replyTo: ActorRef)
+  sealed trait DockerHashStoreResponse
+  final case class DockerHashStoreSuccess(successResponse: DockerHashSuccessResponse) extends DockerHashStoreResponse
+  final case class DockerHashStoreFailure(dockerHashRequest: DockerHashRequest, reason: Throwable) extends DockerHashStoreResponse
+  final case class DockerHashStoreLoadingSuccess(dockerMappings: Map[String, String])
+  final case class DockerHashActorTimeout(request: DockerHashRequest)
+
+  /* Messages */
+  sealed trait WorkflowDockerLookupActorMessage
+  private final case class TransitionToFailed(cause: Throwable) extends WorkflowDockerLookupActorMessage
+
+  /* Responses */
+  sealed trait WorkflowDockerLookupResponse
+  final case class WorkflowDockerLookupFailure(reason: Throwable, request: DockerHashRequest) extends WorkflowDockerLookupResponse
+  final case class WorkflowDockerTerminalFailure(reason: Throwable, request: DockerHashRequest) extends WorkflowDockerLookupResponse
+
+  def props(workflowId: WorkflowId, dockerHashingActor: ActorRef, startMode: StartMode, databaseInterface: SqlDatabase = SingletonServicesStore.databaseInterface) = {
+    Props(new WorkflowDockerLookupActor(workflowId, dockerHashingActor, startMode, databaseInterface)).withDispatcher(EngineDispatcher)
+  }
+
+  object WorkflowDockerLookupActorData {
+    def empty = WorkflowDockerLookupActorData(hashRequests = Map.empty, mappings = Map.empty, failureCause = None)
+  }
+
+  final case class WorkflowDockerLookupActorData(hashRequests: Map[DockerHashRequest, List[ActorRef]],
+                                                 mappings: Map[DockerImageIdentifier, DockerHashResult],
+                                                 failureCause: Option[Throwable]) {
+    /**
+      * Add the specified request and replyTo to this state data.
+      *
+      * @param request The request to be added.
+      * @param replyTo The actor to be informed of the hash or the failure to look up the hash.
+      * @return State data with the added request and replyTo.
+      */
+    def addHashRequest(request: DockerHashRequest, replyTo: ActorRef): WorkflowDockerLookupActorData = {
+      // Prepend this `ActorRef` to the list of `ActorRef`s awaiting the hash for this request, or to Nil if this is the first.
+      val alreadyAwaiting = hashRequests.getOrElse(request, Nil)
+      this.copy(hashRequests = hashRequests + (request -> (replyTo :: alreadyAwaiting)))
+    }
+
+    /**
+      * Empty the collection of hash requests.
+      * @return State data with all awaiting hash requests removed.
+      */
+    def clearHashRequests: WorkflowDockerLookupActorData = this.copy(hashRequests = Map.empty)
+
+    /**
+      * Add this failure cause to the state data.
+      * @param cause The failure cause.
+      * @return Updated state data.
+      */
+    def withFailureCause(cause: Throwable): WorkflowDockerLookupActorData = this.copy(failureCause = Option(cause))
+  }
+}
diff --git a/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala b/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala
index fb87406a1..ffff98559 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala
@@ -11,6 +11,7 @@ import cromwell.core.{WorkflowAborted, WorkflowId}
 import cromwell.engine.backend.BackendSingletonCollection
 import cromwell.engine.workflow.WorkflowActor._
 import cromwell.engine.workflow.WorkflowManagerActor._
+import cromwell.engine.workflow.lifecycle.execution.WorkflowMetadataHelper
 import cromwell.engine.workflow.workflowstore.{WorkflowStoreActor, WorkflowStoreEngineActor, WorkflowStoreState}
 import cromwell.jobstore.JobStoreActor.{JobStoreWriteFailure, JobStoreWriteSuccess, RegisterWorkflowCompleted}
 import cromwell.webservice.EngineStatsActor
@@ -19,7 +20,6 @@ import net.ceedubs.ficus.Ficus._
 import org.apache.commons.lang3.exception.ExceptionUtils
 
 import scala.concurrent.duration._
-import scala.sys.ShutdownHookThread
 
 object WorkflowManagerActor {
   val DefaultMaxWorkflowsToRun = 5000
@@ -33,6 +33,7 @@ object WorkflowManagerActor {
   /**
     * Commands
     */
+  case object PreventNewWorkflowsFromStarting extends WorkflowManagerActorMessage
   sealed trait WorkflowManagerActorCommand extends WorkflowManagerActorMessage
   case object RetrieveNewWorkflows extends WorkflowManagerActorCommand
   final case class AbortWorkflowCommand(id: WorkflowId, replyTo: ActorRef) extends WorkflowManagerActorCommand
@@ -41,19 +42,20 @@ object WorkflowManagerActor {
   case object EngineStatsCommand extends WorkflowManagerActorCommand
 
   def props(workflowStore: ActorRef,
+            ioActor: ActorRef,
             serviceRegistryActor: ActorRef,
             workflowLogCopyRouter: ActorRef,
             jobStoreActor: ActorRef,
             subWorkflowStoreActor: ActorRef,
             callCacheReadActor: ActorRef,
+            callCacheWriteActor: ActorRef,
             dockerHashActor: ActorRef,
             jobTokenDispenserActor: ActorRef,
             backendSingletonCollection: BackendSingletonCollection,
-            abortJobsOnTerminate: Boolean,
             serverMode: Boolean): Props = {
-    val params = WorkflowManagerActorParams(ConfigFactory.load, workflowStore, serviceRegistryActor,
-      workflowLogCopyRouter, jobStoreActor, subWorkflowStoreActor, callCacheReadActor, dockerHashActor, jobTokenDispenserActor, backendSingletonCollection,
-      abortJobsOnTerminate, serverMode)
+    val params = WorkflowManagerActorParams(ConfigFactory.load, workflowStore, ioActor, serviceRegistryActor,
+      workflowLogCopyRouter, jobStoreActor, subWorkflowStoreActor, callCacheReadActor, callCacheWriteActor,
+      dockerHashActor, jobTokenDispenserActor, backendSingletonCollection, serverMode)
     Props(new WorkflowManagerActor(params)).withDispatcher(EngineDispatcher)
   }
 
@@ -62,6 +64,7 @@ object WorkflowManagerActor {
     */
   sealed trait WorkflowManagerState
   case object Running extends WorkflowManagerState
+  case object RunningAndNotStartingNewWorkflows extends WorkflowManagerState
   case object Aborting extends WorkflowManagerState
   case object Done extends WorkflowManagerState
 
@@ -87,21 +90,23 @@ object WorkflowManagerActor {
 
 case class WorkflowManagerActorParams(config: Config,
                                       workflowStore: ActorRef,
+                                      ioActor: ActorRef,
                                       serviceRegistryActor: ActorRef,
                                       workflowLogCopyRouter: ActorRef,
                                       jobStoreActor: ActorRef,
                                       subWorkflowStoreActor: ActorRef,
                                       callCacheReadActor: ActorRef,
+                                      callCacheWriteActor: ActorRef,
                                       dockerHashActor: ActorRef,
                                       jobTokenDispenserActor: ActorRef,
                                       backendSingletonCollection: BackendSingletonCollection,
-                                      abortJobsOnTerminate: Boolean,
                                       serverMode: Boolean)
 
 class WorkflowManagerActor(params: WorkflowManagerActorParams)
-  extends LoggingFSM[WorkflowManagerState, WorkflowManagerData] {
+  extends LoggingFSM[WorkflowManagerState, WorkflowManagerData] with WorkflowMetadataHelper {
 
   private val config = params.config
+  override val serviceRegistryActor = params.serviceRegistryActor
 
   private val maxWorkflowsRunning = config.getConfig("system").as[Option[Int]]("max-concurrent-workflows").getOrElse(DefaultMaxWorkflowsToRun)
   private val maxWorkflowsToLaunch = config.getConfig("system").as[Option[Int]]("max-workflow-launch-count").getOrElse(DefaultMaxWorkflowsToLaunch)
@@ -111,51 +116,16 @@ class WorkflowManagerActor(params: WorkflowManagerActorParams)
   private val tag = self.path.name
 
   private var abortingWorkflowToReplyTo = Map.empty[WorkflowId, ActorRef]
-  private var shutdownHookThreadOption: Option[ShutdownHookThread] = None
+  private var nextPollCancellable: Option[Cancellable] = None
 
   override def preStart(): Unit = {
-    addShutdownHook()
     // Starts the workflow polling cycle
     self ! RetrieveNewWorkflows
   }
 
-  override def postStop() = {
-    // If the actor is stopping, especially during error tests, then there's nothing to wait for later at JVM shutdown.
-    tryRemoveShutdownHook()
-    super.postStop()
-  }
-
-  private def addShutdownHook() = {
-    // Only abort jobs on SIGINT if the config explicitly sets system.abort-jobs-on-terminate = true.
-    val abortJobsOnTerminate =
-    config.getConfig("system").as[Option[Boolean]]("abort-jobs-on-terminate").getOrElse(params.abortJobsOnTerminate)
-
-    if (abortJobsOnTerminate) {
-      val shutdownHookThread = sys.addShutdownHook {
-        logger.info(s"$tag: Received shutdown signal.")
-        self ! AbortAllWorkflowsCommand
-        while (stateData != null && stateData.workflows.nonEmpty) {
-          log.info(s"Waiting for ${stateData.workflows.size} workflows to abort...")
-          Thread.sleep(1000)
-        }
-      }
-      shutdownHookThreadOption = Option(shutdownHookThread)
-    }
-  }
-
-  private def tryRemoveShutdownHook() = {
-    try {
-      shutdownHookThreadOption.foreach(_.remove())
-    } catch {
-      case _: IllegalStateException => /* ignore, we're probably shutting down */
-      case exception: Exception => log.error(exception, "Error while removing shutdown hook: {}", exception.getMessage)
-    }
-    shutdownHookThreadOption = None
-  }
-
   startWith(Running, WorkflowManagerData(workflows = Map.empty))
 
-  when (Running) {
+  val runningAndNotStartingNewWorkflowsStateFunction: StateFunction = {
     /*
      Commands from clients
      */
@@ -169,12 +139,10 @@ class WorkflowManagerActor(params: WorkflowManagerActorParams)
       stay()
     case Event(WorkflowStoreEngineActor.NoNewWorkflowsToStart, _) =>
       log.debug("WorkflowStore provided no new workflows to start")
-      scheduleNextNewWorkflowPoll()
       stay()
     case Event(WorkflowStoreEngineActor.NewWorkflowsToStart(newWorkflows), stateData) =>
       val newSubmissions = newWorkflows map submitWorkflow
       log.info("Retrieved {} workflows from the WorkflowStoreActor", newSubmissions.toList.size)
-      scheduleNextNewWorkflowPoll()
       stay() using stateData.withAddition(newSubmissions)
     case Event(SubscribeToWorkflowCommand(id), data) =>
       data.workflows.get(id) foreach {_ ! SubscribeTransitionCallBack(sender())}
@@ -190,6 +158,7 @@ class WorkflowManagerActor(params: WorkflowManagerActorParams)
         case None =>
           // All cool, if we got this far the workflow ID was found in the workflow store so this workflow must have never
           // made it to the workflow manager.
+          pushCurrentStateToMetadataService(id, WorkflowAborted)
           replyTo ! WorkflowStoreEngineActor.WorkflowAborted(id)
           stay()
       }
@@ -217,13 +186,21 @@ class WorkflowManagerActor(params: WorkflowManagerActorParams)
           val replyTo = abortingWorkflowToReplyTo(workflowId)
           replyTo ! WorkflowStoreEngineActor.WorkflowAborted(workflowId)
           abortingWorkflowToReplyTo -= workflowId
-        } else {
-          params.workflowStore ! WorkflowStoreActor.RemoveWorkflow(workflowId)
         }
       }
       stay using data.without(workflowActor)
   }
 
+  val scheduleNextNewWorkflowPollStateFunction: StateFunction = {
+    case event @ Event(WorkflowStoreEngineActor.NoNewWorkflowsToStart | _: WorkflowStoreEngineActor.NewWorkflowsToStart, _) =>
+      scheduleNextNewWorkflowPoll()
+      runningAndNotStartingNewWorkflowsStateFunction(event)
+  }
+  
+  when (Running) (scheduleNextNewWorkflowPollStateFunction.orElse(runningAndNotStartingNewWorkflowsStateFunction))
+  
+  when (RunningAndNotStartingNewWorkflows) (runningAndNotStartingNewWorkflowsStateFunction)
+
   when (Aborting) {
     case Event(Transition(workflowActor, _, toState: WorkflowActorState), data) if toState.terminal =>
       // Remove this terminal actor from the workflowStore and log a progress message.
@@ -244,6 +221,10 @@ class WorkflowManagerActor(params: WorkflowManagerActorParams)
   when (Done) { FSM.NullFunction }
 
   whenUnhandled {
+    case Event(PreventNewWorkflowsFromStarting, _) =>
+      nextPollCancellable foreach { _.cancel() }
+      sender() ! akka.Done
+      goto(RunningAndNotStartingNewWorkflows)
     // Uninteresting transition and current state notifications.
     case Event((Transition(_, _, _) | CurrentState(_, _)), _) => stay()
     case Event(JobStoreWriteSuccess(_), _) => stay() // Snoozefest
@@ -264,7 +245,7 @@ class WorkflowManagerActor(params: WorkflowManagerActorParams)
   onTransition {
     case _ -> Done =>
       logger.info(s"$tag All workflows finished")
-      ()
+      context stop self
     case fromState -> toState =>
       logger.debug(s"$tag transitioning from $fromState to $toState")
   }
@@ -284,8 +265,8 @@ class WorkflowManagerActor(params: WorkflowManagerActorParams)
       StartNewWorkflow
     }
 
-    val wfProps = WorkflowActor.props(workflowId, startMode, workflow.sources, config, params.serviceRegistryActor,
-      params.workflowLogCopyRouter, params.jobStoreActor, params.subWorkflowStoreActor, params.callCacheReadActor,
+    val wfProps = WorkflowActor.props(workflowId, startMode, workflow.sources, config, params.ioActor, params.serviceRegistryActor,
+      params.workflowLogCopyRouter, params.jobStoreActor, params.subWorkflowStoreActor, params.callCacheReadActor, params.callCacheWriteActor,
       params.dockerHashActor, params.jobTokenDispenserActor,
       params.backendSingletonCollection, params.serverMode)
     val wfActor = context.actorOf(wfProps, name = s"WorkflowActor-$workflowId")
@@ -297,18 +278,18 @@ class WorkflowManagerActor(params: WorkflowManagerActorParams)
   }
 
   private def scheduleNextNewWorkflowPoll() = {
-    context.system.scheduler.scheduleOnce(newWorkflowPollRate, self, RetrieveNewWorkflows)(context.dispatcher)
+    nextPollCancellable = Option(context.system.scheduler.scheduleOnce(newWorkflowPollRate, self, RetrieveNewWorkflows)(context.dispatcher))
   }
 
   private def expandFailureReasons(reasons: Seq[Throwable]): String = {
-    
+
     reasons map {
       case reason: ThrowableAggregation => expandFailureReasons(reason.throwables.toSeq)
       case reason: KnownJobFailureException =>
         val stderrMessage = reason.stderrPath map { path => s"\nCheck the content of stderr for potential additional information: ${path.pathAsString}" } getOrElse ""
         reason.getMessage + stderrMessage
       case reason =>
-      reason.getMessage + "\n" + ExceptionUtils.getStackTrace(reason)
+        reason.getMessage + "\n" + ExceptionUtils.getStackTrace(reason)
     } mkString "\n"
   }
 }
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/CopyWorkflowLogsActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/CopyWorkflowLogsActor.scala
index 6963280ff..e98da8983 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/CopyWorkflowLogsActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/CopyWorkflowLogsActor.scala
@@ -6,8 +6,11 @@ import akka.actor.SupervisorStrategy.Restart
 import akka.actor.{Actor, ActorLogging, ActorRef, OneForOneStrategy, Props}
 import cromwell.core.Dispatcher.IoDispatcher
 import cromwell.core._
+import cromwell.core.io._
 import cromwell.core.logging.WorkflowLogger
 import cromwell.core.path.Path
+import cromwell.engine.workflow.lifecycle.execution.WorkflowMetadataHelper
+import cromwell.filesystems.gcs.batch.GcsBatchCommandBuilder
 import cromwell.services.metadata.MetadataService.PutMetadataAction
 import cromwell.services.metadata.{MetadataEvent, MetadataKey, MetadataValue}
 
@@ -19,25 +22,35 @@ object CopyWorkflowLogsActor {
     case _: IOException => Restart
   }
 
-  def props(serviceRegistryActor: ActorRef) = Props(new CopyWorkflowLogsActor(serviceRegistryActor)).withDispatcher(IoDispatcher)
+  def props(serviceRegistryActor: ActorRef, ioActor: ActorRef) = Props(new CopyWorkflowLogsActor(serviceRegistryActor, ioActor)).withDispatcher(IoDispatcher)
 }
 
 // This could potentially be turned into a more generic "Copy/Move something from A to B"
 // Which could be used for other copying work (outputs, call logs..)
-class CopyWorkflowLogsActor(serviceRegistryActor: ActorRef)
-    extends Actor
-    with ActorLogging {
+class CopyWorkflowLogsActor(override val serviceRegistryActor: ActorRef, override val ioActor: ActorRef) extends Actor 
+  with ActorLogging with GcsBatchCommandBuilder with IoClientHelper with WorkflowMetadataHelper with MonitoringCompanionHelper {
 
-  def copyAndClean(src: Path, dest: Path) = {
+  implicit val ec = context.dispatcher
+  
+  def copyLog(src: Path, dest: Path, workflowId: WorkflowId) = {
     dest.parent.createPermissionedDirectories()
+    // Send the workflowId as context along with the copy so we can update metadata when the response comes back
+    sendIoCommandWithContext(copyCommand(src, dest, overwrite = true), workflowId)
+    // In order to keep "copy and then delete" operations atomic as far as monitoring is concerned, removeWork will only be called
+    // when the delete is complete (successfully or not), or when the copy completes if WorkflowLogger.isTemporary is false
+    addWork()
+  }
 
-    src.copyTo(dest, overwrite = true)
-    if (WorkflowLogger.isTemporary) {
-      src.delete()
-    }
+  def deleteLog(src: Path) = if (WorkflowLogger.isTemporary) {
+    sendIoCommand(deleteCommand(src))
+  } else removeWork()
+  
+  def updateLogsPathInMetadata(workflowId: WorkflowId, path: Path) = {
+    val metadataEventMsg = MetadataEvent(MetadataKey(workflowId, None, WorkflowMetadataKeys.WorkflowLog), MetadataValue(path.pathAsString))
+    serviceRegistryActor ! PutMetadataAction(metadataEventMsg)
   }
 
-  override def receive = {
+  def copyLogsReceive: Receive = {
     case CopyWorkflowLogsActor.Copy(workflowId, destinationDir) =>
       val workflowLogger = new WorkflowLogger(self.path.name, workflowId, Option(log))
 
@@ -46,15 +59,40 @@ class CopyWorkflowLogsActor(serviceRegistryActor: ActorRef)
           val destPath = destinationDir.resolve(src.name)
           workflowLogger.info(s"Copying workflow logs from $src to $destPath")
 
-          copyAndClean(src, destPath)
-
-          val metadataEventMsg = MetadataEvent(MetadataKey(workflowId, None, WorkflowMetadataKeys.WorkflowLog), MetadataValue(destPath))
-          serviceRegistryActor ! PutMetadataAction(metadataEventMsg)
+          copyLog(src, destPath, workflowId)
         }
       }
+      
+    case (workflowId: WorkflowId, IoSuccess(copy: IoCopyCommand, _)) =>
+      updateLogsPathInMetadata(workflowId, copy.destination)
+      deleteLog(copy.source)
+      
+    case (workflowId: WorkflowId, IoFailure(copy: IoCopyCommand, failure)) =>
+      pushWorkflowFailures(workflowId, List(new IOException("Could not copy workflow logs", failure)))
+      log.error(failure, s"Failed to copy workflow logs from ${copy.source.pathAsString} to ${copy.destination.pathAsString}")
+      deleteLog(copy.source)
+      
+    case IoSuccess(_: IoDeleteCommand, _) => removeWork()
+      
+    case IoFailure(delete: IoDeleteCommand, failure) =>
+      removeWork()
+      log.error(failure, s"Failed to delete workflow logs from ${delete.file.pathAsString}")
+
+    case other => log.warning(s"CopyWorkflowLogsActor received an unexpected message: $other")
   }
+  
+  override def receive = monitoringReceive orElse ioReceive orElse copyLogsReceive
 
   override def preRestart(t: Throwable, message: Option[Any]) = {
     message foreach self.forward
   }
+
+  override protected def onTimeout(message: Any, to: ActorRef): Unit = message match {
+    case copy: IoCopyCommand =>
+      log.error(s"Failed to copy workflow logs from ${copy.source.pathAsString} to ${copy.destination.pathAsString}: Timeout")
+      deleteLog(copy.source)
+    case delete: IoDeleteCommand =>
+      log.error(s"Failed to delete workflow logs from ${delete.file.pathAsString}: Timeout")
+    case _ =>
+  }
 }
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/CopyWorkflowOutputsActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/CopyWorkflowOutputsActor.scala
index 55229ee1d..627053d0f 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/CopyWorkflowOutputsActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/CopyWorkflowOutputsActor.scala
@@ -1,41 +1,63 @@
 package cromwell.engine.workflow.lifecycle
 
-import akka.actor.Props
-import cromwell.backend.BackendWorkflowFinalizationActor.{FinalizationResponse, FinalizationSuccess}
+import akka.actor.{Actor, ActorLogging, ActorRef, Props}
+import akka.event.LoggingReceive
+import cromwell.backend.BackendLifecycleActor.BackendWorkflowLifecycleActorResponse
+import cromwell.backend.BackendWorkflowFinalizationActor.{FinalizationFailed, FinalizationResponse, FinalizationSuccess, Finalize}
 import cromwell.backend.{AllBackendInitializationData, BackendConfigurationDescriptor, BackendInitializationData, BackendLifecycleActorFactory}
 import cromwell.core.Dispatcher.IoDispatcher
 import cromwell.core.WorkflowOptions._
 import cromwell.core._
+import cromwell.core.io.AsyncIo
 import cromwell.core.path.{Path, PathCopier, PathFactory}
 import cromwell.engine.EngineWorkflowDescriptor
 import cromwell.engine.backend.{BackendConfiguration, CromwellBackends}
-import wdl4s.values.{WdlArray, WdlMap, WdlSingleFile, WdlValue}
+import cromwell.filesystems.gcs.batch.GcsBatchCommandBuilder
+import wdl4s.wdl.values.{WdlArray, WdlMap, WdlSingleFile, WdlValue}
 
 import scala.concurrent.{ExecutionContext, Future}
+import scala.util.{Failure, Success}
 
 object CopyWorkflowOutputsActor {
-  def props(workflowId: WorkflowId, workflowDescriptor: EngineWorkflowDescriptor, workflowOutputs: CallOutputs,
+  def props(workflowId: WorkflowId, ioActor: ActorRef, workflowDescriptor: EngineWorkflowDescriptor, workflowOutputs: CallOutputs,
             initializationData: AllBackendInitializationData) = Props(
-    new CopyWorkflowOutputsActor(workflowId, workflowDescriptor, workflowOutputs, initializationData)
+    new CopyWorkflowOutputsActor(workflowId, ioActor, workflowDescriptor, workflowOutputs, initializationData)
   ).withDispatcher(IoDispatcher)
 }
 
-class CopyWorkflowOutputsActor(workflowId: WorkflowId, val workflowDescriptor: EngineWorkflowDescriptor, workflowOutputs: CallOutputs,
+class CopyWorkflowOutputsActor(workflowId: WorkflowId, override val ioActor: ActorRef, val workflowDescriptor: EngineWorkflowDescriptor, workflowOutputs: CallOutputs,
                                initializationData: AllBackendInitializationData)
-  extends EngineWorkflowFinalizationActor with PathFactory {
+  extends Actor with ActorLogging with PathFactory with AsyncIo with GcsBatchCommandBuilder {
 
+  implicit val ec = context.dispatcher
   override val pathBuilders = workflowDescriptor.pathBuilders
 
-  private def copyWorkflowOutputs(workflowOutputsFilePath: String): Unit = {
+  override def receive = ioReceive orElse LoggingReceive {
+    case Finalize => performActionThenRespond(afterAll()(context.dispatcher), FinalizationFailed)(context.dispatcher)
+  }
+
+  private def performActionThenRespond(operation: => Future[BackendWorkflowLifecycleActorResponse],
+                                       onFailure: (Throwable) => BackendWorkflowLifecycleActorResponse)
+                                      (implicit ec: ExecutionContext) = {
+    val respondTo: ActorRef = sender
+    operation onComplete {
+      case Success(r) => respondTo ! r
+      case Failure(t) => respondTo ! onFailure(t)
+    }
+  }
+
+  private def copyWorkflowOutputs(workflowOutputsFilePath: String): Future[Seq[Unit]] = {
     val workflowOutputsPath = buildPath(workflowOutputsFilePath)
 
-    val outputFilePaths = getOutputFilePaths
+    val outputFilePaths = getOutputFilePaths(workflowOutputsPath)
 
-    outputFilePaths foreach {
-      case (workflowRootPath, srcPath) =>
-        // WARNING: PathCopier does not do atomic copies. The files may be partially written.
-        PathCopier.copy(workflowRootPath, srcPath, workflowOutputsPath)
+    val copies = outputFilePaths map {
+      case (srcPath, dstPath) => 
+        dstPath.createDirectories()
+        copyAsync(srcPath, dstPath)
     }
+    
+    Future.sequence(copies)
   }
 
   private def findFiles(values: Seq[WdlValue]): Seq[WdlSingleFile] = {
@@ -47,16 +69,23 @@ class CopyWorkflowOutputsActor(workflowId: WorkflowId, val workflowDescriptor: E
     }
   }
   
-  private def getOutputFilePaths: Seq[(Path, Path)] = {
-    for {
+  private def getOutputFilePaths(workflowOutputsPath: Path): List[(Path, Path)] = {
+    val rootAndFiles = for {
       // NOTE: Without .toSeq, outputs in arrays only yield the last output
       backend <- workflowDescriptor.backendAssignments.values.toSeq
       config <- BackendConfiguration.backendConfigurationDescriptor(backend).toOption.toSeq
       rootPath <- getBackendRootPath(backend, config).toSeq
-      outputFiles = findFiles(workflowOutputs.values.map(_.wdlValue).toSeq)
-      wdlFile <- outputFiles
-      wdlPath = PathFactory.buildPath(wdlFile.value, pathBuilders)
-    } yield (rootPath, wdlPath)
+      outputFiles = findFiles(workflowOutputs.values.map(_.wdlValue).toSeq).map(_.value)
+    } yield (rootPath, outputFiles)
+    
+    val outputFileDestinations = rootAndFiles flatMap {
+      case (workflowRoot, outputs) =>
+        outputs map { output => 
+          val outputPath = PathFactory.buildPath(output, pathBuilders)
+          outputPath -> PathCopier.getDestinationFilePath(workflowRoot, outputPath, workflowOutputsPath) 
+        }
+    }
+    outputFileDestinations.distinct.toList
   }
 
   private def getBackendRootPath(backend: String, config: BackendConfigurationDescriptor): Option[Path] = {
@@ -72,8 +101,13 @@ class CopyWorkflowOutputsActor(workflowId: WorkflowId, val workflowDescriptor: E
     backendFactory.getExecutionRootPath(workflowDescriptor.backendDescriptor, config.backendConfig, initializationData)
   }
 
-  final override def afterAll()(implicit ec: ExecutionContext): Future[FinalizationResponse] = Future {
-    workflowDescriptor.getWorkflowOption(FinalWorkflowOutputsDir) foreach copyWorkflowOutputs
-    FinalizationSuccess
+  /**
+    * Happens after everything else runs
+    */
+  final def afterAll()(implicit ec: ExecutionContext): Future[FinalizationResponse] = {
+    workflowDescriptor.getWorkflowOption(FinalWorkflowOutputsDir) match {
+      case Some(outputs) => copyWorkflowOutputs(outputs) map { _ => FinalizationSuccess }
+      case None => Future.successful(FinalizationSuccess)
+    }
   }
 }
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/EngineWorkflowFinalizationActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/EngineWorkflowFinalizationActor.scala
deleted file mode 100644
index 401396cae..000000000
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/EngineWorkflowFinalizationActor.scala
+++ /dev/null
@@ -1,30 +0,0 @@
-package cromwell.engine.workflow.lifecycle
-
-import akka.actor.{Actor, ActorLogging, ActorRef}
-import akka.event.LoggingReceive
-import cromwell.backend.BackendLifecycleActor.BackendWorkflowLifecycleActorResponse
-import cromwell.backend.BackendWorkflowFinalizationActor.{FinalizationFailed, FinalizationResponse, Finalize}
-
-import scala.concurrent.{ExecutionContext, Future}
-import scala.util.{Failure, Success}
-
-trait EngineWorkflowFinalizationActor extends Actor with ActorLogging {
-  def receive: Receive = LoggingReceive {
-    case Finalize => performActionThenRespond(afterAll()(context.dispatcher), FinalizationFailed)(context.dispatcher)
-  }
-
-  protected def performActionThenRespond(operation: => Future[BackendWorkflowLifecycleActorResponse],
-                                         onFailure: (Throwable) => BackendWorkflowLifecycleActorResponse)
-                                        (implicit ec: ExecutionContext) = {
-    val respondTo: ActorRef = sender
-    operation onComplete {
-      case Success(r) => respondTo ! r
-      case Failure(t) => respondTo ! onFailure(t)
-    }
-  }
-
-  /**
-    * Happens after everything else runs
-    */
-  def afterAll()(implicit ec: ExecutionContext): Future[FinalizationResponse]
-}
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala
index 278edc2d7..207f24063 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActor.scala
@@ -1,6 +1,7 @@
 package cromwell.engine.workflow.lifecycle
 
-import akka.actor.{ActorRef, FSM, LoggingFSM, Props}
+import akka.actor.{ActorRef, FSM, LoggingFSM, Props, Status}
+import akka.pattern.pipe
 import cats.data.NonEmptyList
 import cats.data.Validated._
 import cats.instances.list._
@@ -12,8 +13,8 @@ import com.typesafe.config.Config
 import com.typesafe.scalalogging.LazyLogging
 import cromwell.backend.BackendWorkflowDescriptor
 import cromwell.core.Dispatcher.EngineDispatcher
-import cromwell.core.WorkflowOptions.{ReadFromCache, WorkflowOption, WriteToCache}
 import cromwell.core._
+import cromwell.core.WorkflowOptions.{ReadFromCache, WorkflowOption, WriteToCache}
 import cromwell.core.callcaching._
 import cromwell.core.labels.{Label, Labels}
 import cromwell.core.logging.WorkflowLogging
@@ -21,17 +22,18 @@ import cromwell.core.path.BetterFileMethods.OpenOptions
 import cromwell.core.path.{DefaultPathBuilder, Path, PathBuilder}
 import cromwell.engine._
 import cromwell.engine.backend.CromwellBackends
-import cromwell.engine.workflow.lifecycle.MaterializeWorkflowDescriptorActor.{MaterializeWorkflowDescriptorActorData, MaterializeWorkflowDescriptorActorState}
+import cromwell.engine.workflow.lifecycle.MaterializeWorkflowDescriptorActor.MaterializeWorkflowDescriptorActorState
 import cromwell.services.metadata.MetadataService._
 import cromwell.services.metadata.{MetadataEvent, MetadataKey, MetadataValue}
 import lenthall.exception.MessageAggregation
 import lenthall.validation.ErrorOr._
 import net.ceedubs.ficus.Ficus._
 import spray.json._
-import wdl4s._
-import wdl4s.expression.NoFunctions
-import wdl4s.values.{WdlString, WdlValue}
+import wdl4s.wdl._
+import wdl4s.wdl.expression.NoFunctions
+import wdl4s.wdl.values.{WdlSingleFile, WdlString, WdlValue}
 
+import scala.concurrent.Future
 import scala.language.postfixOps
 import scala.util.{Failure, Success, Try}
 
@@ -72,15 +74,11 @@ object MaterializeWorkflowDescriptorActor {
     override val terminal = true
   }
   case object ReadyToMaterializeState extends MaterializeWorkflowDescriptorActorState
+  case object MaterializingState extends MaterializeWorkflowDescriptorActorState
   case object MaterializationSuccessfulState extends MaterializeWorkflowDescriptorActorTerminalState
   case object MaterializationFailedState extends MaterializeWorkflowDescriptorActorTerminalState
   case object MaterializationAbortedState extends MaterializeWorkflowDescriptorActorTerminalState
 
-  /*
-  Data
-   */
-  case class MaterializeWorkflowDescriptorActorData()
-
   private val DefaultWorkflowFailureMode = NoNewCalls.toString
 
   private[lifecycle] def validateCallCachingMode(workflowOptions: WorkflowOptions, conf: Config): ErrorOr[CallCachingMode] = {
@@ -116,34 +114,52 @@ object MaterializeWorkflowDescriptorActor {
 class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef,
                                          val workflowIdForLogging: WorkflowId,
                                          cromwellBackends: => CromwellBackends,
-                                         importLocalFilesystem: Boolean) extends LoggingFSM[MaterializeWorkflowDescriptorActorState, MaterializeWorkflowDescriptorActorData] with LazyLogging with WorkflowLogging {
+                                         importLocalFilesystem: Boolean) extends LoggingFSM[MaterializeWorkflowDescriptorActorState, Unit] with LazyLogging with WorkflowLogging {
 
   import MaterializeWorkflowDescriptorActor._
 
   val tag = self.path.name
 
   val iOExecutionContext = context.system.dispatchers.lookup("akka.dispatchers.io-dispatcher")
-
-  startWith(ReadyToMaterializeState, MaterializeWorkflowDescriptorActorData())
+  implicit val ec = context.dispatcher
+  
+  startWith(ReadyToMaterializeState, ())
 
   when(ReadyToMaterializeState) {
     case Event(MaterializeWorkflowDescriptorCommand(workflowSourceFiles, conf), _) =>
-      buildWorkflowDescriptor(workflowIdForLogging, workflowSourceFiles, conf) match {
-        case Valid(descriptor) =>
-          sender() ! MaterializeWorkflowDescriptorSuccessResponse(descriptor)
-          goto(MaterializationSuccessfulState)
-        case Invalid(error) =>
-          sender() ! MaterializeWorkflowDescriptorFailureResponse(
-            new IllegalArgumentException with MessageAggregation {
-              val exceptionContext = s"Workflow input processing failed"
-              val errorMessages = error.toList
-            })
+      val replyTo = sender()
+
+      workflowOptionsAndPathBuilders(workflowSourceFiles) match {
+        case Valid((workflowOptions, pathBuilders)) =>
+          val futureDescriptor = pathBuilders map {
+            buildWorkflowDescriptor(workflowIdForLogging, workflowSourceFiles, conf, workflowOptions, _)
+          }
+
+          // Pipe the response to self, but make it look like it comes from the sender of the command
+          // This way we can access it through sender() in the next state and don't have to store the value
+          // of replyTo in the data
+          pipe(futureDescriptor).to(self, replyTo)
+          goto(MaterializingState)
+        case Invalid(error) => 
+          workflowInitializationFailed(error, replyTo)
           goto(MaterializationFailedState)
       }
     case Event(MaterializeWorkflowDescriptorAbortCommand, _) =>
       goto(MaterializationAbortedState)
   }
 
+  when(MaterializingState) {
+    case Event(Valid(descriptor: EngineWorkflowDescriptor), _) =>
+      sender() ! MaterializeWorkflowDescriptorSuccessResponse(descriptor)
+      goto(MaterializationSuccessfulState)
+    case Event(Invalid(error: NonEmptyList[String]@unchecked), _) =>
+      workflowInitializationFailed(error, sender())
+      goto(MaterializationFailedState)
+    case Event(Status.Failure(failure), _) =>
+      workflowInitializationFailed(NonEmptyList.of(failure.getMessage), sender())
+      goto(MaterializationFailedState)
+  }
+
   // Let these fall through to the whenUnhandled handler:
   when(MaterializationSuccessfulState) { FSM.NullFunction }
   when(MaterializationFailedState) { FSM.NullFunction }
@@ -165,17 +181,33 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef,
       stay
   }
 
+  private def workflowInitializationFailed(errors: NonEmptyList[String], replyTo: ActorRef) = {
+    sender() ! MaterializeWorkflowDescriptorFailureResponse(
+      new IllegalArgumentException with MessageAggregation {
+        val exceptionContext = "Workflow input processing failed"
+        val errorMessages = errors.toList
+      })
+  }
+
+  private def workflowOptionsAndPathBuilders(sourceFiles: WorkflowSourceFilesCollection): ErrorOr[(WorkflowOptions, Future[List[PathBuilder]])] = {
+    val workflowOptionsValidation = validateWorkflowOptions(sourceFiles.workflowOptionsJson)
+    workflowOptionsValidation map { workflowOptions =>
+      val pathBuilders = EngineFilesystems.pathBuildersForWorkflow(workflowOptions)(context.system, context.dispatcher)
+      (workflowOptions, pathBuilders)
+    }
+  }
+
   private def buildWorkflowDescriptor(id: WorkflowId,
                                       sourceFiles: WorkflowSourceFilesCollection,
-                                      conf: Config): ErrorOr[EngineWorkflowDescriptor] = {
+                                      conf: Config,
+                                      workflowOptions: WorkflowOptions,
+                                      pathBuilders: List[PathBuilder]): ErrorOr[EngineWorkflowDescriptor] = {
     val namespaceValidation = validateNamespace(sourceFiles)
-    val workflowOptionsValidation = validateWorkflowOptions(sourceFiles.workflowOptionsJson)
     val labelsValidation = validateLabels(sourceFiles.labelsJson)
-    (namespaceValidation |@| workflowOptionsValidation |@| labelsValidation) map {
-      (_, _, _)
-    } flatMap { case (namespace, workflowOptions, labels) =>
+
+    (namespaceValidation |@| labelsValidation).tupled flatMap { case (namespace, labels) =>
       pushWfNameMetadataService(namespace.workflow.unqualifiedName)
-      val pathBuilders = EngineFilesystems(context.system).pathBuildersForWorkflow(workflowOptions)
+      publishLabelsToMetadata(id, namespace.workflow.unqualifiedName, labels)
       buildWorkflowDescriptor(id, sourceFiles, namespace, workflowOptions, labels, conf, pathBuilders)
     }
   }
@@ -188,6 +220,18 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef,
     serviceRegistryActor ! PutMetadataAction(nameEvent)
   }
 
+  private def publishLabelsToMetadata(rootWorkflowId: WorkflowId, unqualifiedName: String, labels: Labels): Unit = {
+    val defaultLabel = "cromwell-workflow-id" -> s"cromwell-$rootWorkflowId"
+    val customLabels = labels.asMap
+    labelsToMetadata(customLabels + defaultLabel, rootWorkflowId)
+  }
+
+  protected def labelsToMetadata(labels: Map[String, String], workflowId: WorkflowId): Unit = {
+    labels foreach { case (k, v) =>
+      serviceRegistryActor ! PutMetadataAction(MetadataEvent(MetadataKey(workflowId, None, s"${WorkflowMetadataKeys.Labels}:$k"), MetadataValue(v)))
+    }
+  }
+
   private def buildWorkflowDescriptor(id: WorkflowId,
                                       sourceFiles: WorkflowSourceFilesCollection,
                                       namespace: WdlNamespaceWithWorkflow,
@@ -202,17 +246,16 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef,
     val backendAssignmentsValidation = validateBackendAssignments(namespace.taskCalls, workflowOptions, defaultBackendName)
     val callCachingModeValidation = validateCallCachingMode(workflowOptions, conf)
 
-    (rawInputsValidation |@| failureModeValidation |@| backendAssignmentsValidation |@| callCachingModeValidation) map {
-      (_, _, _, _)
-    } flatMap { case (rawInputs, failureMode, backendAssignments, callCachingMode) =>
-      buildWorkflowDescriptor(id, namespace, rawInputs, backendAssignments, workflowOptions, labels, failureMode, pathBuilders, callCachingMode)
+    (rawInputsValidation |@| failureModeValidation |@| backendAssignmentsValidation |@| callCachingModeValidation).tupled flatMap {
+      case (rawInputs, failureMode, backendAssignments, callCachingMode) =>
+        buildWorkflowDescriptor(id, namespace, rawInputs, backendAssignments, workflowOptions, labels, failureMode, pathBuilders, callCachingMode)
     }
   }
 
   private def buildWorkflowDescriptor(id: WorkflowId,
                                       namespace: WdlNamespaceWithWorkflow,
                                       rawInputs: Map[String, JsValue],
-                                      backendAssignments: Map[TaskCall, String],
+                                      backendAssignments: Map[WdlTaskCall, String],
                                       workflowOptions: WorkflowOptions,
                                       labels: Labels,
                                       failureMode: WorkflowFailureMode,
@@ -235,9 +278,10 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef,
 
     for {
       coercedInputs <- validateCoercedInputs(rawInputs, namespace)
-      _ = pushWfInputsToMetadataService(coercedInputs)
-      evaluatedWorkflowsDeclarations <- validateDeclarations(namespace, workflowOptions, coercedInputs, pathBuilders)
-      declarationsAndInputs <- checkTypes(evaluatedWorkflowsDeclarations ++ coercedInputs)
+      coercedValidatedFileInputs <- validateWdlFiles(coercedInputs)
+      _ = pushWfInputsToMetadataService(coercedValidatedFileInputs)
+      evaluatedWorkflowsDeclarations <- validateDeclarations(namespace, workflowOptions, coercedValidatedFileInputs, pathBuilders)
+      declarationsAndInputs <- checkTypes(evaluatedWorkflowsDeclarations ++ coercedValidatedFileInputs)
       backendDescriptor = BackendWorkflowDescriptor(id, namespace.workflow, declarationsAndInputs, workflowOptions, labels)
     } yield EngineWorkflowDescriptor(namespace, backendDescriptor, backendAssignments, failureMode, pathBuilders, callCachingMode)
   }
@@ -256,7 +300,7 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef,
     serviceRegistryActor ! PutMetadataAction(inputEvents)
   }
 
-  private def validateBackendAssignments(calls: Set[TaskCall], workflowOptions: WorkflowOptions, defaultBackendName: Option[String]): ErrorOr[Map[TaskCall, String]] = {
+  private def validateBackendAssignments(calls: Set[WdlTaskCall], workflowOptions: WorkflowOptions, defaultBackendName: Option[String]): ErrorOr[Map[WdlTaskCall, String]] = {
     val callToBackendMap = Try {
       calls map { call =>
         val backendPriorities = Seq(
@@ -285,7 +329,7 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef,
   /**
     * Map a call to a backend name depending on the runtime attribute key
     */
-  private def assignBackendUsingRuntimeAttrs(call: TaskCall): Option[String] = {
+  private def assignBackendUsingRuntimeAttrs(call: WdlTaskCall): Option[String] = {
     val runtimeAttributesMap = call.task.runtimeAttributes.attrs
     runtimeAttributesMap.get(RuntimeBackendKey) map { wdlExpr => evaluateBackendNameExpression(call.fullyQualifiedName, wdlExpr) }
   }
@@ -313,7 +357,7 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef,
   private def validateImportsDirectory(zipContents: Array[Byte]): ErrorOr[Path] = {
 
     def makeZipFile(contents: Array[Byte]): Try[Path] = Try {
-      DefaultPathBuilder.createTempFile("", ".zip").write(contents)(OpenOptions.default)
+      DefaultPathBuilder.createTempFile("", ".zip").writeByteArray(contents)(OpenOptions.default)
     }
 
     def unZipFile(f: Path) = Try {
@@ -359,10 +403,10 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef,
       } else {
         List(WdlNamespace.directoryResolver(importsDirFile))
       }
-      val results = WdlNamespaceWithWorkflow.load(w.wdlSource, importResolvers)
+      val results = WdlNamespaceWithWorkflow.load(w.workflowSource, importResolvers)
       importsDir.delete(swallowIOExceptions = true)
       results match {
-        case Success(ns) => ns.validNel
+        case Success(ns) => validateWorkflowNameLengths(ns)
         case Failure(f) => f.getMessage.invalidNel
       }
     }
@@ -370,25 +414,31 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef,
     validateImportsDirectory(w.importsZip) flatMap importsAsNamespace
   }
 
-  private def validateNamespace(source: WorkflowSourceFilesCollection): ErrorOr[WdlNamespaceWithWorkflow] = {
-    try {
-      source match {
-        case w: WorkflowSourceFilesWithDependenciesZip => validateNamespaceWithImports(w)
-        case w: WorkflowSourceFilesWithoutImports =>
-          val importResolvers: Seq[ImportResolver] = if (importLocalFilesystem) {
-            List(WdlNamespace.fileResolver)
-          } else {
-            List.empty
-          }
-          // This .get is ok because we're already in a try/catch.
-          WdlNamespaceWithWorkflow.load(w.wdlSource, importResolvers).get.validNel
-      }
-    } catch {
-      case e: Exception => s"Unable to load namespace from workflow: ${e.getMessage}".invalidNel
+  private def validateWorkflowNameLengths(namespace: WdlNamespaceWithWorkflow): ErrorOr[WdlNamespaceWithWorkflow] = {
+    def allWorkflowNames(n: WdlNamespace): Seq[String] = n.workflows.map(_.unqualifiedName) ++ n.namespaces.flatMap(allWorkflowNames)
+    val tooLong = allWorkflowNames(namespace).filter(_.length >= 100)
+    if (tooLong.nonEmpty) {
+      ("Workflow names must be shorter than 100 characters: " + tooLong.mkString(" ")).invalidNel
+    } else {
+      namespace.validNel
     }
   }
 
-  private def validateRawInputs(json: WdlJson): ErrorOr[Map[String, JsValue]] = {
+  private def validateNamespace(source: WorkflowSourceFilesCollection): ErrorOr[WdlNamespaceWithWorkflow] = source match {
+    case w: WorkflowSourceFilesWithDependenciesZip => validateNamespaceWithImports(w)
+    case w: WorkflowSourceFilesWithoutImports =>
+      val importResolvers: Seq[ImportResolver] = if (importLocalFilesystem) {
+        List(WdlNamespace.fileResolver)
+      } else {
+        List.empty
+      }
+      WdlNamespaceWithWorkflow.load(w.workflowSource, importResolvers) match {
+        case Failure(e) => s"Unable to load namespace from workflow: ${e.getMessage}".invalidNel
+        case Success(namespace) => validateWorkflowNameLengths(namespace)
+      }
+  }
+
+  private def validateRawInputs(json: WorkflowJson): ErrorOr[Map[String, JsValue]] = {
     Try(json.parseJson) match {
       case Success(JsObject(inputs)) => inputs.validNel
       case Failure(reason: Throwable) => s"Workflow contains invalid inputs JSON: ${reason.getMessage}".invalidNel
@@ -396,12 +446,12 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef,
     }
   }
 
-  private def validateLabels(json: WdlJson): ErrorOr[Labels] = {
+  private def validateLabels(json: WorkflowJson): ErrorOr[Labels] = {
 
     def toLabels(inputs: Map[String, JsValue]): ErrorOr[Labels] = {
       val vectorOfValidatedLabel: Vector[ErrorOr[Label]] = inputs.toVector map {
         case (key, JsString(s)) => Label.validateLabel(key, s)
-        case (key, other) => s"Invalid label '$key: $other': Labels must be strings, and must match the regex ${Label.LabelRegexPattern}".invalidNel
+        case (key, other) => s"Invalid label $key: $other : Labels must be strings. ${Label.LabelExpectationsMessage}".invalidNel
       }
 
       vectorOfValidatedLabel.sequence[ErrorOr, Label] map { validatedVectorofLabel => Labels(validatedVectorofLabel) }
@@ -423,7 +473,17 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef,
     }
   }
 
-  private def validateWorkflowOptions(workflowOptions: WdlJson): ErrorOr[WorkflowOptions] = {
+  private def validateWdlFiles(workflowInputs: WorkflowCoercedInputs): ErrorOr[WorkflowCoercedInputs] = {
+    val failedFiles = workflowInputs.collect {
+      case (fullyQualifiedName , WdlSingleFile(value)) if value.startsWith("\"gs://") => s"""Invalid value for File input '$fullyQualifiedName': $value starts with a '\"' """
+    }
+    NonEmptyList.fromList(failedFiles.toList) match {
+      case Some(errors) => Invalid(errors)
+      case None => workflowInputs.validNel
+    }
+  }
+
+  private def validateWorkflowOptions(workflowOptions: WorkflowJson): ErrorOr[WorkflowOptions] = {
     WorkflowOptions.fromJsonString(workflowOptions) match {
       case Success(opts) => opts.validNel
       case Failure(e) => s"Workflow contains invalid options JSON: ${e.getMessage}".invalidNel
@@ -438,8 +498,8 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef,
     }
 
     modeString flatMap WorkflowFailureMode.tryParse match {
-        case Success(mode) => mode.validNel
-        case Failure(t) => t.getMessage.invalidNel
+      case Success(mode) => mode.validNel
+      case Failure(t) => t.getMessage.invalidNel
     }
   }
 }
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/WorkflowFinalizationActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/WorkflowFinalizationActor.scala
index 5c4b777a5..79f147a2b 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/WorkflowFinalizationActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/WorkflowFinalizationActor.scala
@@ -1,6 +1,7 @@
 package cromwell.engine.workflow.lifecycle
 
-import akka.actor.{FSM, Props}
+import akka.actor.SupervisorStrategy.Stop
+import akka.actor.{ActorRef, FSM, OneForOneStrategy, Props}
 import cromwell.backend.BackendWorkflowFinalizationActor.{FinalizationFailed, FinalizationSuccess, Finalize}
 import cromwell.backend._
 import cromwell.core.Dispatcher.EngineDispatcher
@@ -9,7 +10,7 @@ import cromwell.engine.EngineWorkflowDescriptor
 import cromwell.engine.backend.CromwellBackends
 import cromwell.engine.workflow.lifecycle.WorkflowFinalizationActor._
 import cromwell.engine.workflow.lifecycle.WorkflowLifecycleActor._
-import wdl4s.TaskCall
+import wdl4s.wdl.WdlTaskCall
 
 import scala.util.{Failure, Success, Try}
 
@@ -38,14 +39,19 @@ object WorkflowFinalizationActor {
   case object WorkflowFinalizationSucceededResponse extends WorkflowLifecycleSuccessResponse
   final case class WorkflowFinalizationFailedResponse(reasons: Seq[Throwable]) extends WorkflowLifecycleFailureResponse
 
-  def props(workflowId: WorkflowId, workflowDescriptor: EngineWorkflowDescriptor, jobExecutionMap: JobExecutionMap,
-            workflowOutputs: CallOutputs, initializationData: AllBackendInitializationData): Props = {
-    Props(new WorkflowFinalizationActor(workflowId, workflowDescriptor, jobExecutionMap, workflowOutputs, initializationData)).withDispatcher(EngineDispatcher)
+  def props(workflowId: WorkflowId, workflowDescriptor: EngineWorkflowDescriptor, ioActor: ActorRef, jobExecutionMap: JobExecutionMap,
+            workflowOutputs: CallOutputs, initializationData: AllBackendInitializationData, copyWorkflowOutputsActor: Option[Props]): Props = {
+    Props(new WorkflowFinalizationActor(workflowId, workflowDescriptor, ioActor, jobExecutionMap, workflowOutputs, initializationData, copyWorkflowOutputsActor)).withDispatcher(EngineDispatcher)
   }
 }
 
-case class WorkflowFinalizationActor(workflowIdForLogging: WorkflowId, workflowDescriptor: EngineWorkflowDescriptor,
-                                     jobExecutionMap: JobExecutionMap, workflowOutputs: CallOutputs, initializationData: AllBackendInitializationData)
+case class WorkflowFinalizationActor(workflowIdForLogging: WorkflowId,
+                                     workflowDescriptor: EngineWorkflowDescriptor,
+                                     ioActor: ActorRef,
+                                     jobExecutionMap: JobExecutionMap,
+                                     workflowOutputs: CallOutputs,
+                                     initializationData: AllBackendInitializationData,
+                                     copyWorkflowOutputsActorProps: Option[Props])
   extends WorkflowLifecycleActor[WorkflowFinalizationActorState] {
 
   val tag = self.path.name
@@ -57,6 +63,13 @@ case class WorkflowFinalizationActor(workflowIdForLogging: WorkflowId, workflowD
   override def successResponse(data: WorkflowLifecycleActorData) = WorkflowFinalizationSucceededResponse
   override def failureResponse(reasons: Seq[Throwable]) = WorkflowFinalizationFailedResponse(reasons)
 
+  // If an engine or backend finalization actor (children of this actor) dies, send ourselves the failure and stop the child actor
+  override def supervisorStrategy = OneForOneStrategy() {
+    case failure => 
+      self.tell(FinalizationFailed(failure), sender())
+      Stop
+  }
+
   startWith(FinalizationPendingState, WorkflowLifecycleActorData.empty)
 
   when(FinalizationPendingState) {
@@ -65,21 +78,18 @@ case class WorkflowFinalizationActor(workflowIdForLogging: WorkflowId, workflowD
         for {
           (backend, calls) <- workflowDescriptor.backendAssignments.groupBy(_._2).mapValues(_.keySet)
           props <- CromwellBackends.backendLifecycleFactoryActorByName(backend).map(
-            _.workflowFinalizationActorProps(workflowDescriptor.backendDescriptor, calls, filterJobExecutionsForBackend(calls), workflowOutputs, initializationData.get(backend))
+            _.workflowFinalizationActorProps(workflowDescriptor.backendDescriptor, ioActor, calls, filterJobExecutionsForBackend(calls), workflowOutputs, initializationData.get(backend))
           ).get
           actor = context.actorOf(props, backend)
         } yield actor
       }
 
-      val engineFinalizationActor = Try {
-        context.actorOf(CopyWorkflowOutputsActor.props(workflowIdForLogging, workflowDescriptor, workflowOutputs, initializationData),
-          "CopyWorkflowOutputsActor")
-      }
+      val engineFinalizationActor = Try { copyWorkflowOutputsActorProps.map(context.actorOf(_, "CopyWorkflowOutputsActor")).toList }
 
       val allActors = for {
         backendFinalizationActorsFromTry <- backendFinalizationActors
         engineFinalizationActorFromTry <- engineFinalizationActor
-      } yield backendFinalizationActorsFromTry.toList.+:(engineFinalizationActorFromTry)
+      } yield backendFinalizationActorsFromTry.toList ++ engineFinalizationActorFromTry
 
       allActors match {
         case Failure(ex) =>
@@ -98,11 +108,11 @@ case class WorkflowFinalizationActor(workflowIdForLogging: WorkflowId, workflowD
   }
   
   // Only send to each backend the jobs that it executed
-  private def filterJobExecutionsForBackend(calls: Set[TaskCall]): JobExecutionMap = {
+  private def filterJobExecutionsForBackend(calls: Set[WdlTaskCall]): JobExecutionMap = {
     jobExecutionMap map {
       case (wd, executedKeys) => wd -> (executedKeys filter { jobKey => calls.contains(jobKey.call) })
     } filter {
-      case (wd, keys) => keys.nonEmpty
+      case (_, keys) => keys.nonEmpty
     }
   }
 
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/WorkflowInitializationActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/WorkflowInitializationActor.scala
index 14e8a31a4..b8a0ea503 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/WorkflowInitializationActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/WorkflowInitializationActor.scala
@@ -43,8 +43,10 @@ object WorkflowInitializationActor {
 
   def props(workflowId: WorkflowId,
             workflowDescriptor: EngineWorkflowDescriptor,
-            serviceRegistryActor: ActorRef): Props = {
-    Props(new WorkflowInitializationActor(workflowId, workflowDescriptor, serviceRegistryActor)).withDispatcher(EngineDispatcher)
+            ioActor: ActorRef,
+            serviceRegistryActor: ActorRef,
+            restarting: Boolean): Props = {
+    Props(new WorkflowInitializationActor(workflowId, workflowDescriptor, ioActor, serviceRegistryActor, restarting)).withDispatcher(EngineDispatcher)
   }
 
   case class BackendActorAndBackend(actor: ActorRef, backend: String)
@@ -52,7 +54,9 @@ object WorkflowInitializationActor {
 
 case class WorkflowInitializationActor(workflowIdForLogging: WorkflowId,
                                        workflowDescriptor: EngineWorkflowDescriptor,
-                                       serviceRegistryActor: ActorRef)
+                                       ioActor: ActorRef,
+                                       serviceRegistryActor: ActorRef,
+                                       restarting: Boolean)
   extends AbortableWorkflowLifecycleActor[WorkflowInitializationActorState] {
 
   startWith(InitializationPendingState, WorkflowLifecycleActorData.empty)
@@ -80,7 +84,7 @@ case class WorkflowInitializationActor(workflowIdForLogging: WorkflowId,
         for {
           (backend, calls) <- workflowDescriptor.backendAssignments.groupBy(_._2).mapValues(_.keySet)
           props <- CromwellBackends.backendLifecycleFactoryActorByName(backend).map(factory =>
-            factory.workflowInitializationActorProps(workflowDescriptor.backendDescriptor, calls, serviceRegistryActor)
+            factory.workflowInitializationActorProps(workflowDescriptor.backendDescriptor, ioActor, calls, serviceRegistryActor, restarting)
           ).get
           actor = context.actorOf(props, backend)
         } yield BackendActorAndBackend(actor, backend)
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/WorkflowLifecycleActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/WorkflowLifecycleActor.scala
index 2990a6d2a..1cc4f302b 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/WorkflowLifecycleActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/WorkflowLifecycleActor.scala
@@ -69,7 +69,7 @@ trait WorkflowLifecycleActor[S <: WorkflowLifecycleActorState] extends LoggingFS
   def successResponse(data: WorkflowLifecycleActorData): WorkflowLifecycleSuccessResponse
   def failureResponse(reasons: Seq[Throwable]): WorkflowLifecycleFailureResponse
 
-  override def supervisorStrategy = AllForOneStrategy() {
+  override def supervisorStrategy: SupervisorStrategy = AllForOneStrategy() {
     case ex: ActorInitializationException =>
       context.parent ! failureResponse(Seq(ex))
       context.stop(self)
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/CallMetadataHelper.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/CallMetadataHelper.scala
index bdc6eaf7c..28132a2aa 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/CallMetadataHelper.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/CallMetadataHelper.scala
@@ -8,8 +8,8 @@ import cromwell.core.ExecutionStatus._
 import cromwell.core._
 import cromwell.services.metadata.MetadataService._
 import cromwell.services.metadata._
-import wdl4s._
-import wdl4s.values.WdlValue
+import wdl4s.wdl._
+import wdl4s.wdl.values.WdlValue
 
 import scala.util.Random
 
@@ -27,7 +27,7 @@ trait CallMetadataHelper {
     serviceRegistryActor ! PutMetadataAction(startEvents)
   }
 
-  def pushQueuedCallMetadata(diffs: Seq[WorkflowExecutionDiff]) = {
+  def pushQueuedCallMetadata(diffs: List[WorkflowExecutionDiff]) = {
     val startingEvents = for {
       diff <- diffs
       (jobKey, executionState) <- diff.executionStoreChanges if jobKey.isInstanceOf[BackendJobDescriptorKey] && executionState == ExecutionStatus.QueuedInCromwell
@@ -82,10 +82,10 @@ trait CallMetadataHelper {
   }
 
   def pushFailedCallMetadata(jobKey: JobKey, returnCode: Option[Int], failure: Throwable, retryableFailure: Boolean) = {
-    val failedState = if (retryableFailure) ExecutionStatus.Preempted else ExecutionStatus.Failed
+    val failedState = if (retryableFailure) ExecutionStatus.RetryableFailure else ExecutionStatus.Failed
     val completionEvents = completedCallMetadataEvents(jobKey, failedState, returnCode)
     val retryableFailureEvent = MetadataEvent(metadataKeyForCall(jobKey, CallMetadataKeys.RetryableFailure), MetadataValue(retryableFailure))
-    val failureEvents = throwableToMetadataEvents(metadataKeyForCall(jobKey, s"${CallMetadataKeys.Failures}[$randomNumberString]"), failure).+:(retryableFailureEvent)
+    val failureEvents = throwableToMetadataEvents(metadataKeyForCall(jobKey, s"${CallMetadataKeys.Failures}"), failure).+:(retryableFailureEvent)
 
     serviceRegistryActor ! PutMetadataAction(completionEvents ++ failureEvents)
   }
@@ -134,8 +134,8 @@ trait CallMetadataHelper {
     ) ++ returnCodeEvent.getOrElse(List.empty)
   }
   
-  private def metadataKeyForCall(jobKey: JobKey, myKey: String) = MetadataKey(workflowIdForCallMetadata, Option(MetadataJobKey(jobKey.scope.fullyQualifiedName, jobKey.index, jobKey.attempt)), myKey)
+  def metadataKeyForCall(jobKey: JobKey, myKey: String) = MetadataKey(workflowIdForCallMetadata, Option(MetadataJobKey(jobKey.scope.fullyQualifiedName, jobKey.index, jobKey.attempt)), myKey)
 
   private def randomNumberString: String = Random.nextInt.toString.stripPrefix("-")
-  
+
 }
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala
index a9a111df8..05cc4a115 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/EngineJobExecutionActor.scala
@@ -1,8 +1,7 @@
 package cromwell.engine.workflow.lifecycle.execution
 
-import akka.actor.{ActorRef, ActorRefFactory, LoggingFSM, Props}
-import akka.routing.RoundRobinPool
-import cats.data.NonEmptyList
+import akka.actor.SupervisorStrategy.{Escalate, Stop}
+import akka.actor.{ActorInitializationException, ActorRef, LoggingFSM, OneForOneStrategy, Props}
 import cromwell.backend.BackendCacheHitCopyingActor.CopyOutputsCommand
 import cromwell.backend.BackendJobExecutionActor._
 import cromwell.backend.{BackendInitializationData, BackendJobDescriptor, BackendJobDescriptorKey, BackendLifecycleActorFactory}
@@ -14,17 +13,22 @@ import cromwell.core.logging.WorkflowLogging
 import cromwell.core.simpleton.WdlValueSimpleton
 import cromwell.database.sql.tables.CallCachingEntry
 import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor._
-import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.{CacheHit, CacheMiss, CallCacheHashes, HashError}
+import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCache.CallCacheHashBundle
+import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheReadActor._
+import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheReadingJobActor.NextHit
+import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheWriteActor._
+import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.{CallCacheHashes, _}
 import cromwell.engine.workflow.lifecycle.execution.callcaching.FetchCachedResultsActor.{CachedOutputLookupFailed, CachedOutputLookupSucceeded}
 import cromwell.engine.workflow.lifecycle.execution.callcaching._
 import cromwell.engine.workflow.lifecycle.execution.preparation.CallPreparation.{BackendJobPreparationSucceeded, CallPreparationFailed}
 import cromwell.engine.workflow.lifecycle.execution.preparation.{CallPreparation, JobPreparationActor}
 import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor.{JobExecutionTokenDenied, JobExecutionTokenDispensed, JobExecutionTokenRequest, JobExecutionTokenReturn}
 import cromwell.jobstore.JobStoreActor._
-import cromwell.jobstore.{Pending => _, _}
+import cromwell.jobstore._
 import cromwell.services.SingletonServicesStore
-import cromwell.services.metadata.{CallMetadataKeys, MetadataKey}
-import wdl4s.TaskOutput
+import cromwell.services.metadata.CallMetadataKeys.CallCachingKeys
+import cromwell.services.metadata.{CallMetadataKeys, MetadataJobKey, MetadataKey}
+import wdl4s.wdl.TaskOutput
 
 import scala.concurrent.ExecutionContext
 import scala.util.{Failure, Success, Try}
@@ -36,9 +40,11 @@ class EngineJobExecutionActor(replyTo: ActorRef,
                               initializationData: Option[BackendInitializationData],
                               restarting: Boolean,
                               val serviceRegistryActor: ActorRef,
+                              ioActor: ActorRef,
                               jobStoreActor: ActorRef,
                               callCacheReadActor: ActorRef,
-                              dockerHashActor: ActorRef,
+                              callCacheWriteActor: ActorRef,
+                              workflowDockerLookupActor: ActorRef,
                               jobTokenDispenserActor: ActorRef,
                               backendSingletonActor: Option[ActorRef],
                               backendName: String,
@@ -47,28 +53,43 @@ class EngineJobExecutionActor(replyTo: ActorRef,
   override val workflowIdForLogging = executionData.workflowDescriptor.id
   override val workflowIdForCallMetadata = executionData.workflowDescriptor.id
 
+  override val supervisorStrategy = OneForOneStrategy() {
+    // If an actor fails to initialize, send the exception to self before stopping it so we can fail the job properly
+    case e: ActorInitializationException =>
+      self ! e
+      Stop
+    case t =>
+      super.supervisorStrategy.decider.applyOrElse(t, (_: Any) => Escalate)
+  }
+
   val jobTag = s"${workflowIdForLogging.shortString}:${jobDescriptorKey.call.fullyQualifiedName}:${jobDescriptorKey.index.fromIndex}:${jobDescriptorKey.attempt}"
   val tag = s"EJEA_$jobTag"
 
   // There's no need to check for a cache hit again if we got preempted, or if there's no result copying actor defined
   // NB: this can also change (e.g. if we have a HashError we just force this to CallCachingOff)
-  private var effectiveCallCachingMode = if (factory.cacheHitCopyingActorProps.isEmpty || jobDescriptorKey.attempt > 1) callCachingMode.withoutRead else callCachingMode
+  private[execution] var effectiveCallCachingMode = {
+    if (factory.fileHashingActorProps.isEmpty) CallCachingOff
+    else if (factory.cacheHitCopyingActorProps.isEmpty || jobDescriptorKey.attempt > 1) {
+      callCachingMode.withoutRead
+    } else callCachingMode
+  }
 
   // For tests:
   private[execution] def checkEffectiveCallCachingMode = effectiveCallCachingMode
 
   private[execution] var executionToken: Option[JobExecutionToken] = None
 
-  private val effectiveCallCachingKey = CacheMetadataKeyPrefix + "effectiveCallCachingMode"
-  private val callCachingReadResultMetadataKey = CacheMetadataKeyPrefix + "result"
-  private val callCachingHitResultMetadataKey = CacheMetadataKeyPrefix + "hit"
-  private val callCachingAllowReuseMetadataKey = CacheMetadataKeyPrefix + "allowResultReuse"
+  private val effectiveCallCachingKey = CallCachingKeys.EffectiveModeKey
+  private val callCachingReadResultMetadataKey = CallCachingKeys.ReadResultMetadataKey
+  private val callCachingHitResultMetadataKey = CallCachingKeys.HitResultMetadataKey
+  private val callCachingAllowReuseMetadataKey = CallCachingKeys.AllowReuseMetadataKey
+  private val callCachingHitFailures = CallCachingKeys.HitFailuresKey
+  private val callCachingHashes = CallCachingKeys.HashesKey
 
   implicit val ec: ExecutionContext = context.dispatcher
 
   override def preStart() = {
     log.debug(s"$tag: $effectiveCallCachingKey: $effectiveCallCachingMode")
-    writeCallCachingModeToMetadata()
   }
 
   startWith(Pending, NoData)
@@ -100,20 +121,44 @@ class EngineJobExecutionActor(replyTo: ActorRef,
   // When CheckingJobStore, the FSM always has NoData
   when(CheckingJobStore) {
     case Event(JobNotComplete, NoData) =>
-      prepareJob()
+      checkCacheEntryExistence()
     case Event(JobComplete(jobResult), NoData) =>
       val response = jobResult match {
-        case JobResultSuccess(returnCode, jobOutputs) => JobSucceededResponse(jobDescriptorKey, returnCode, jobOutputs, None, Seq.empty)
+        // Always puts `None` for `dockerImageUsed` for a successfully completed job on restart. This shouldn't be a
+        // problem since `saveJobCompletionToJobStore` will already have sent this to metadata.
+        case JobResultSuccess(returnCode, jobOutputs) => JobSucceededResponse(jobDescriptorKey, returnCode, jobOutputs, None, Seq.empty, None)
         case JobResultFailure(returnCode, reason, false) => JobFailedNonRetryableResponse(jobDescriptorKey, reason, returnCode)
         case JobResultFailure(returnCode, reason, true) => JobFailedRetryableResponse(jobDescriptorKey, reason, returnCode)
       }
       respondAndStop(response)
     case Event(f: JobStoreReadFailure, NoData) =>
+      writeCallCachingModeToMetadata()
       log.error(f.reason, "{}: Error reading from JobStore", tag)
       // Escalate
       throw new RuntimeException(f.reason)
   }
 
+  // When we're restarting but the job store says the job is not complete.
+  // This is to cover for the case where Cromwell was stopped after writing Cache Info to the DB but before
+  // writing to the JobStore. In that case, we don't want to cache to ourselves (turn cache read off), nor do we want to 
+  // try and write the cache info again, which would fail (turn cache write off).
+  // This means call caching should be disabled.
+  // Note that we check that there is not a Cache entry for *this* current job. It's still technically possible
+  // to call cache to another job that finished while this one was running (before the restart).
+  when(CheckingCacheEntryExistence) {
+    // There was already a cache entry for this job
+    case Event(HasCallCacheEntry(_), NoData) =>
+      // Disable call caching
+      effectiveCallCachingMode = CallCachingOff
+      prepareJob()
+    // No cache entry for this job - keep going
+    case Event(NoCallCacheEntry(_), NoData) =>
+      prepareJob()
+    case Event(CacheResultLookupFailure(reason), NoData) =>
+      log.error(reason, "{}: Failure checking for cache entry existence: {}. Attempting to resume job anyway.", jobTag, reason.getMessage)
+      prepareJob()
+  }
+
   // When PreparingJob, the FSM always has NoData
   when(PreparingJob) {
     case Event(BackendJobPreparationSucceeded(jobDescriptor, bjeaProps), NoData) =>
@@ -134,8 +179,10 @@ class EngineJobExecutionActor(replyTo: ActorRef,
         callCachingReadResultMetadataKey -> "Cache Miss"))
       log.debug("Cache miss for job {}", jobTag)
       runJob(data)
+    case Event(hashes: CallCacheHashes, data: ResponsePendingData) =>
+      addHashesAndStay(data, hashes)
     case Event(hit: CacheHit, data: ResponsePendingData) =>
-      fetchCachedResults(jobDescriptorKey.call.task.outputs, hit.cacheResultIds.head, data.copy(cacheHit = Option(hit)))
+      fetchCachedResults(jobDescriptorKey.call.task.outputs, hit.cacheResultId, data.withCacheHit(hit))
     case Event(HashError(t), data: ResponsePendingData) =>
       writeToMetadata(Map(callCachingReadResultMetadataKey -> s"Hashing Error: ${t.getMessage}"))
       disableCallCaching(Option(t))
@@ -148,8 +195,8 @@ class EngineJobExecutionActor(replyTo: ActorRef,
         callCachingHitResultMetadataKey -> true,
         callCachingReadResultMetadataKey -> s"Cache Hit: $cacheHitDetails"))
       log.debug("Cache hit for {}! Fetching cached result {}", jobTag, cacheResultId)
-      makeBackendCopyCacheHit(wdlValueSimpletons, jobDetritus, returnCode, data, cacheResultId)
-    case Event(CachedOutputLookupFailed(callCachingEntryId, error), data: ResponsePendingData) =>
+      makeBackendCopyCacheHit(wdlValueSimpletons, jobDetritus, returnCode, data, cacheResultId) using data.withCacheDetails(cacheHitDetails)
+    case Event(CachedOutputLookupFailed(_, error), data: ResponsePendingData) =>
       log.warning("Can't make a copy of the cached job outputs for {} due to {}. Running job.", jobTag, error)
       runJob(data)
     case Event(hashes: CallCacheHashes, data: ResponsePendingData) =>
@@ -162,16 +209,16 @@ class EngineJobExecutionActor(replyTo: ActorRef,
 
   when(BackendIsCopyingCachedOutputs) {
     // Backend copying response:
-    case Event(response: JobSucceededResponse, data @ ResponsePendingData(_, _, Some(Success(hashes)), _)) =>
+    case Event(response: JobSucceededResponse, data @ ResponsePendingData(_, _, Some(Success(hashes)), _, _)) =>
       saveCacheResults(hashes, data.withSuccessResponse(response))
-    case Event(response: JobSucceededResponse, data @ ResponsePendingData(_, _, None, _)) if effectiveCallCachingMode.writeToCache =>
+    case Event(response: JobSucceededResponse, data @ ResponsePendingData(_, _, None, _, _)) if effectiveCallCachingMode.writeToCache =>
       // Wait for the CallCacheHashes
       stay using data.withSuccessResponse(response)
     case Event(response: JobSucceededResponse, data: ResponsePendingData) => // bad hashes or cache write off
       saveJobCompletionToJobStore(data.withSuccessResponse(response))
-    case Event(response: BackendJobExecutionResponse, data @ ResponsePendingData(_, _, _, Some(cacheHit))) =>
+    case Event(response: BackendJobExecutionResponse, data @ ResponsePendingData(_, _, _, _, Some(cacheHit))) =>
       response match {
-        case f: BackendJobFailedResponse => invalidateCacheHitAndTransition(cacheHit.cacheResultIds.head, data, f.throwable)
+        case f: BackendJobFailedResponse => invalidateCacheHitAndTransition(cacheHit, data, f.throwable)
         case _ => runJob(data)
       }
 
@@ -223,16 +270,22 @@ class EngineJobExecutionActor(replyTo: ActorRef,
       disableCallCaching(Option(t))
       stay using data.copy(hashes = Option(Failure(t)))
 
-    case Event(response: JobSucceededResponse, data @ ResponsePendingData(_, _, Some(Success(hashes)), _)) if effectiveCallCachingMode.writeToCache =>
+    // Success
+    // All hashes already retrieved - save to the cache
+    case Event(response: JobSucceededResponse, data @ ResponsePendingData(_, _, Some(Success(hashes)), _, _)) if effectiveCallCachingMode.writeToCache =>
       eventList ++= response.executionEvents
       saveCacheResults(hashes, data.withSuccessResponse(response))
-    case Event(response: JobSucceededResponse, data @ ResponsePendingData(_, _, None, _)) if effectiveCallCachingMode.writeToCache =>
+    // Some hashes are still missing - waiting for them
+    case Event(response: JobSucceededResponse, data @ ResponsePendingData(_, _, None, _, _)) if effectiveCallCachingMode.writeToCache =>
       eventList ++= response.executionEvents
       log.debug(s"Got job result for {}, awaiting hashes", jobTag)
       stay using data.withSuccessResponse(response)
+    // writeToCache is OFF - complete the job
     case Event(response: JobSucceededResponse, data: ResponsePendingData) =>
       eventList ++= response.executionEvents
       saveJobCompletionToJobStore(data.withSuccessResponse(response))
+
+    // Non-success:
     case Event(response: BackendJobExecutionResponse, data: ResponsePendingData) =>
       saveJobCompletionToJobStore(data.withResponse(response))
   }
@@ -250,7 +303,7 @@ class EngineJobExecutionActor(replyTo: ActorRef,
   when(UpdatingJobStore) {
     case Event(JobStoreWriteSuccess(_), data: ResponseData) =>
       forwardAndStop(data.response)
-    case Event(JobStoreWriteFailure(t), data: ResponseData) =>
+    case Event(JobStoreWriteFailure(t), _: ResponseData) =>
       respondAndStop(JobFailedNonRetryableResponse(jobDescriptorKey, new Exception(s"JobStore write failure: ${t.getMessage}", t), None))
   }
 
@@ -261,31 +314,48 @@ class EngineJobExecutionActor(replyTo: ActorRef,
   }
 
   whenUnhandled {
+    case Event(e: ActorInitializationException, _) =>
+      respondAndStop(JobFailedNonRetryableResponse(jobDescriptorKey, e, None))
     case Event(msg, _) =>
       log.error("Bad message from {} to EngineJobExecutionActor in state {}(with data {}): {}", sender, stateName, stateData, msg)
       stay
   }
-  
+
+  private def publishHashesToMetadata(maybeHashes: Option[Try[CallCacheHashes]]) = maybeHashes match {
+    case Some(Success(hashes)) =>
+      val hashMap = hashes.hashes.collect({
+        case HashResult(HashKey(useInCallCaching, keyComponents), HashValue(value)) if useInCallCaching => 
+          (callCachingHashes + MetadataKey.KeySeparator + keyComponents.mkString(MetadataKey.KeySeparator.toString)) -> value
+      }).toMap
+      writeToMetadata(hashMap)
+    case _ =>
+  }
+
   private def handleReadFromCacheOn(jobDescriptor: BackendJobDescriptor, activity: CallCachingActivity, updatedData: ResponsePendingData) = {
-    jobDescriptor.callCachingEligibility match {
-        // If the job is eligible, initialize job hashing and go to CheckingCallCache state
-      case CallCachingEligible =>
-        initializeJobHashing(jobDescriptor, activity)
-        goto(CheckingCallCache) using updatedData
-      case ineligible: CallCachingIneligible =>
+    jobDescriptor.maybeCallCachingEligible match {
+      // If the job is eligible, initialize job hashing and go to CheckingCallCache state
+      case eligible: CallCachingEligible =>
+        initializeJobHashing(jobDescriptor, activity, eligible) match {
+          case Success(ejha) => goto(CheckingCallCache) using updatedData.withEJHA(ejha)
+          case Failure(failure) => respondAndStop(JobFailedNonRetryableResponse(jobDescriptorKey, failure, None))
+        }
+      case _ =>
         // If the job is ineligible, turn call caching off
-        writeToMetadata(Map(callCachingReadResultMetadataKey -> s"Cache Miss: ${ineligible.message}"))
+        writeToMetadata(Map(callCachingReadResultMetadataKey -> s"Cache Miss"))
         disableCallCaching()
         runJob(updatedData)
     }
   }
 
   private def handleReadFromCacheOff(jobDescriptor: BackendJobDescriptor, activity: CallCachingActivity, updatedData: ResponsePendingData) = {
-    jobDescriptor.callCachingEligibility match {
-        // If the job is eligible, initialize job hashing so it can be written to the cache
-      case CallCachingEligible => initializeJobHashing(jobDescriptor, activity)
+    jobDescriptor.maybeCallCachingEligible match {
+      // If the job is eligible, initialize job hashing so it can be written to the cache
+      case eligible: CallCachingEligible => initializeJobHashing(jobDescriptor, activity, eligible) match {
+        case Failure(failure) => log.error(failure, "Failed to initialize job hashing. The job will not be written to the cache")
+        case _ =>
+      }
       // Don't even initialize hashing to write to the cache if the job is ineligible
-      case ineligible: CallCachingIneligible => disableCallCaching()
+      case _ => disableCallCaching()
     }
     // If read from cache is off, always run the job
     runJob(updatedData)
@@ -316,7 +386,7 @@ class EngineJobExecutionActor(replyTo: ActorRef,
   }
 
   private def disableCallCaching(reason: Option[Throwable] = None) = {
-    reason foreach { r => log.error(r, "{}: Hash error, disabling call caching for this job.", jobTag) }
+    reason foreach { log.error(_, "{}: Hash error, disabling call caching for this job.", jobTag) }
     effectiveCallCachingMode = CallCachingOff
     writeCallCachingModeToMetadata()
     writeToMetadata(Map(callCachingHitResultMetadataKey -> false))
@@ -337,24 +407,39 @@ class EngineJobExecutionActor(replyTo: ActorRef,
 
   def createJobPreparationActor(jobPrepProps: Props, name: String): ActorRef = context.actorOf(jobPrepProps, name)
   def prepareJob() = {
+    writeCallCachingModeToMetadata()
     val jobPreparationActorName = s"BackendPreparationActor_for_$jobTag"
-    val jobPrepProps = JobPreparationActor.props(executionData, jobDescriptorKey, factory, dockerHashActor, initializationData, serviceRegistryActor, backendSingletonActor)
+    val jobPrepProps = JobPreparationActor.props(executionData, jobDescriptorKey, factory, workflowDockerLookupActor = workflowDockerLookupActor,
+      initializationData, serviceRegistryActor = serviceRegistryActor, ioActor = ioActor, backendSingletonActor = backendSingletonActor)
     val jobPreparationActor = createJobPreparationActor(jobPrepProps, jobPreparationActorName)
     jobPreparationActor ! CallPreparation.Start
     goto(PreparingJob)
   }
 
-  def initializeJobHashing(jobDescriptor: BackendJobDescriptor, activity: CallCachingActivity): Unit = {
-    val props = EngineJobHashingActor.props(
-      self,
-      jobDescriptor,
-      initializationData,
-      // Use context.system instead of context as the factory. Otherwise when we die, so will the child actors.
-      factoryFileHashingRouter(backendName, factory, context.system),
-      callCacheReadActor,
-      factory.runtimeAttributeDefinitions(initializationData), backendName, activity)
-    context.actorOf(props, s"ejha_for_$jobDescriptor")
-    ()
+  def initializeJobHashing(jobDescriptor: BackendJobDescriptor, activity: CallCachingActivity, callCachingEligible: CallCachingEligible): Try[ActorRef] = {
+    val maybeFileHashingActorProps = factory.fileHashingActorProps map {
+      _.apply(jobDescriptor, initializationData, serviceRegistryActor, ioActor)
+    }
+
+    maybeFileHashingActorProps match {
+      case Some(fileHashingActorProps) =>
+        val props = EngineJobHashingActor.props(
+          self,
+          serviceRegistryActor,
+          jobDescriptor,
+          initializationData,
+          fileHashingActorProps,
+          CallCacheReadingJobActor.props(callCacheReadActor),
+          factory.runtimeAttributeDefinitions(initializationData),
+          backendName,
+          activity,
+          callCachingEligible
+        )
+        val ejha = context.actorOf(props, s"ejha_for_$jobDescriptor")
+
+        Success(ejha)
+      case None => Failure(new IllegalStateException("Tried to initialize job hashing without a file hashing actor !"))
+    }
   }
 
   def makeFetchCachedResultsActor(callCachingEntryId: CallCachingEntryId, taskOutputs: Seq[TaskOutput]): Unit = {
@@ -370,7 +455,7 @@ class EngineJobExecutionActor(replyTo: ActorRef,
   private def makeBackendCopyCacheHit(wdlValueSimpletons: Seq[WdlValueSimpleton], jobDetritusFiles: Map[String,String], returnCode: Option[Int], data: ResponsePendingData, cacheResultId: CallCachingEntryId) = {
     factory.cacheHitCopyingActorProps match {
       case Some(propsMaker) =>
-        val backendCacheHitCopyingActorProps = propsMaker(data.jobDescriptor, initializationData, serviceRegistryActor)
+        val backendCacheHitCopyingActorProps = propsMaker(data.jobDescriptor, initializationData, serviceRegistryActor, ioActor)
         val cacheHitCopyActor = context.actorOf(backendCacheHitCopyingActorProps, buildCacheHitCopyingActorName(data.jobDescriptor, cacheResultId))
         cacheHitCopyActor ! CopyOutputsCommand(wdlValueSimpletons, jobDetritusFiles, returnCode)
         replyTo ! JobRunning(data.jobDescriptor.key, data.jobDescriptor.inputDeclarations, None)
@@ -396,7 +481,7 @@ class EngineJobExecutionActor(replyTo: ActorRef,
     def updateMetadataForInvalidatedEntry(entry: CallCachingEntry) = {
       import cromwell.core.ExecutionIndex._
       import cromwell.services.metadata.MetadataService.implicits.MetadataAutoPutter
-      
+
       val workflowId = WorkflowId.fromString(entry.workflowExecutionUuid)
       // If the entry doesn't have an attempt, it means that this cache entry was added before this change
       // and we don't know which attempt yielded this cache entry
@@ -404,20 +489,21 @@ class EngineJobExecutionActor(replyTo: ActorRef,
       val key = Option((entry.callFullyQualifiedName, entry.jobIndex.toIndex, entry.jobAttempt.getOrElse(1)))
       serviceRegistryActor.putMetadataWithRawKey(workflowId, key, Map(callCachingAllowReuseMetadataKey -> false))
     }
-    
+
     response match {
       case CallCacheInvalidatedFailure(failure) => log.error(failure, "Failed to invalidate cache entry for job: {}", jobDescriptorKey)
       case CallCacheInvalidatedSuccess(Some(entry)) => updateMetadataForInvalidatedEntry(entry)
       case _ =>
     }
 
-    data.popCacheHitId match {
-      case newData @ ResponsePendingData(_, _, _, Some(cacheHit)) =>
+    data.ejha match {
+      case Some(ejha) =>
         log.info("Trying to use another cache hit for job: {}", jobDescriptorKey)
-        fetchCachedResults(jobDescriptorKey.call.task.outputs, cacheHit.cacheResultIds.head, newData)
-      case newData =>
+        ejha ! NextHit
+        goto(CheckingCallCache)
+      case _ =>
         log.info("Could not find another cache hit, falling back to running job: {}", jobDescriptorKey)
-        runJob(newData)
+        runJob(data)
     }
   }
 
@@ -429,20 +515,31 @@ class EngineJobExecutionActor(replyTo: ActorRef,
     s"$workflowIdForLogging-BackendCacheHitCopyingActor-$jobTag-${cacheResultId.id}"
   }
 
-  protected def createSaveCacheResultsActor(hashes: CallCacheHashes, success: JobSucceededResponse): Unit = {
-    val callCache = new CallCache(SingletonServicesStore.databaseInterface)
-    context.actorOf(CallCacheWriteActor.props(callCache, workflowIdForLogging, hashes, success), s"CallCacheWriteActor-$tag")
-    ()
+  private def publishHitFailure(cache: EJEACacheHit, failure: Throwable) = {
+    import MetadataKey._
+    import cromwell.services.metadata.MetadataService._
+
+    cache.details foreach { details =>
+      val metadataKey = MetadataKey(
+        workflowIdForLogging,
+        Option(MetadataJobKey(jobDescriptorKey.call.fullyQualifiedName, jobDescriptorKey.index, jobDescriptorKey.attempt)),
+        s"$callCachingHitFailures[${cache.hitNumber}]:${details.escapeMeta}"
+      )
+
+      serviceRegistryActor ! PutMetadataAction(throwableToMetadataEvents(metadataKey, failure))
+    }
   }
 
-  private def invalidateCacheHitAndTransition(cacheId: CallCachingEntryId, data: ResponsePendingData, reason: Throwable) = {
+  private def invalidateCacheHitAndTransition(ejeaCacheHit: EJEACacheHit, data: ResponsePendingData, reason: Throwable) = {
+    publishHitFailure(ejeaCacheHit, reason)
+
     val invalidationRequired = effectiveCallCachingMode match {
       case CallCachingOff => throw new RuntimeException("Should not be calling invalidateCacheHit if call caching is off!") // Very unexpected. Fail out of this bad-state EJEA.
       case activity: CallCachingActivity => activity.options.invalidateBadCacheResults
     }
     if (invalidationRequired) {
       log.error(reason, "Failed copying cache results for job {}, invalidating cache entry.", jobDescriptorKey)
-      invalidateCacheHit(cacheId)
+      invalidateCacheHit(ejeaCacheHit.hit.cacheResultId)
       goto(InvalidatingCacheEntry)
     } else {
       handleCacheInvalidatedResponse(CallCacheInvalidationUnnecessary, data)
@@ -455,22 +552,34 @@ class EngineJobExecutionActor(replyTo: ActorRef,
     ()
   }
 
+  private def checkCacheEntryExistence() = {
+    callCacheReadActor ! CallCacheEntryForCall(workflowIdForLogging, jobDescriptorKey)
+    goto(CheckingCacheEntryExistence)
+  }
+
   private def saveCacheResults(hashes: CallCacheHashes, data: SucceededResponseData) = {
-    createSaveCacheResultsActor(hashes, data.successResponse)
+    callCacheWriteActor ! SaveCallCacheHashes(CallCacheHashBundle(workflowIdForLogging, hashes, data.response))
     val updatedData = data.copy(hashes = Option(Success(hashes)))
     goto(UpdatingCallCache) using updatedData
   }
 
   private def saveJobCompletionToJobStore(updatedData: ResponseData) = {
     updatedData.response match {
-      case JobSucceededResponse(jobKey: BackendJobDescriptorKey, returnCode: Option[Int], jobOutputs: CallOutputs, _, executionEvents) =>
+      case JobSucceededResponse(jobKey: BackendJobDescriptorKey, returnCode: Option[Int], jobOutputs: CallOutputs, _, _, _) =>
+        publishHashesToMetadata(updatedData.hashes)
         saveSuccessfulJobResults(jobKey, returnCode, jobOutputs)
-      case AbortedResponse(jobKey: BackendJobDescriptorKey) =>
+      case AbortedResponse(_: BackendJobDescriptorKey) =>
         log.debug("{}: Won't save aborted job response to JobStore", jobTag)
         forwardAndStop(updatedData.response)
-      case JobFailedNonRetryableResponse(jobKey: BackendJobDescriptorKey, throwable: Throwable, returnCode: Option[Int]) => saveUnsuccessfulJobResults(jobKey, returnCode, throwable, retryable = false)
-      case JobFailedRetryableResponse(jobKey: BackendJobDescriptorKey, throwable: Throwable, returnCode: Option[Int]) => saveUnsuccessfulJobResults(jobKey, returnCode, throwable, retryable = true)
+      case JobFailedNonRetryableResponse(jobKey, throwable: Throwable, returnCode: Option[Int]) =>
+        publishHashesToMetadata(updatedData.hashes)
+        writeToMetadata(Map(callCachingAllowReuseMetadataKey -> false))
+        saveUnsuccessfulJobResults(jobKey, returnCode, throwable, retryable = false)
+      case JobFailedRetryableResponse(jobKey: BackendJobDescriptorKey, throwable: Throwable, returnCode: Option[Int]) =>
+        writeToMetadata(Map(callCachingAllowReuseMetadataKey -> false))
+        saveUnsuccessfulJobResults(jobKey, returnCode, throwable, retryable = true)
     }
+    updatedData.dockerImageUsed foreach { image => writeToMetadata(Map("dockerImageUsed" -> image)) }
     goto(UpdatingJobStore) using updatedData
   }
 
@@ -509,6 +618,7 @@ object EngineJobExecutionActor {
   case object PreparingJob extends EngineJobExecutionActorState
   case object RunningJob extends EngineJobExecutionActorState
   case object UpdatingCallCache extends EngineJobExecutionActorState
+  case object CheckingCacheEntryExistence extends EngineJobExecutionActorState
   case object UpdatingJobStore extends EngineJobExecutionActorState
   case object InvalidatingCacheEntry extends EngineJobExecutionActorState
 
@@ -525,9 +635,11 @@ object EngineJobExecutionActor {
             initializationData: Option[BackendInitializationData],
             restarting: Boolean,
             serviceRegistryActor: ActorRef,
+            ioActor: ActorRef,
             jobStoreActor: ActorRef,
             callCacheReadActor: ActorRef,
-            dockerHashActor: ActorRef,
+            callCacheWriteActor: ActorRef,
+            workflowDockerLookupActor: ActorRef,
             jobTokenDispenserActor: ActorRef,
             backendSingletonActor: Option[ActorRef],
             backendName: String,
@@ -540,15 +652,19 @@ object EngineJobExecutionActor {
       initializationData = initializationData,
       restarting = restarting,
       serviceRegistryActor = serviceRegistryActor,
+      ioActor = ioActor,
       jobStoreActor = jobStoreActor,
       callCacheReadActor = callCacheReadActor,
-      dockerHashActor = dockerHashActor,
+      callCacheWriteActor = callCacheWriteActor,
+      workflowDockerLookupActor = workflowDockerLookupActor,
       jobTokenDispenserActor = jobTokenDispenserActor,
       backendSingletonActor = backendSingletonActor,
       backendName = backendName: String,
       callCachingMode = callCachingMode)).withDispatcher(EngineDispatcher)
   }
 
+  case class EJEACacheHit(hit: CacheHit, hitNumber: Int, details: Option[String])
+
   private[execution] sealed trait EJEAData {
     override def toString = getClass.getSimpleName
   }
@@ -558,7 +674,11 @@ object EngineJobExecutionActor {
   private[execution] case class ResponsePendingData(jobDescriptor: BackendJobDescriptor,
                                                     bjeaProps: Props,
                                                     hashes: Option[Try[CallCacheHashes]] = None,
-                                                    cacheHit: Option[CacheHit] = None) extends EJEAData {
+                                                    ejha: Option[ActorRef] = None,
+                                                    ejeaCacheHit: Option[EJEACacheHit] = None) extends EJEAData {
+
+    def withEJHA(ejha: ActorRef): EJEAData = this.copy(ejha = Option(ejha))
+
 
     def withSuccessResponse(success: JobSucceededResponse) = SucceededResponseData(success, hashes)
 
@@ -567,82 +687,31 @@ object EngineJobExecutionActor {
       case failure => NotSucceededResponseData(failure, hashes)
     }
 
-    def withCacheHit(cacheHit: Option[CacheHit]) = this.copy(cacheHit = cacheHit)
+    def withCacheHit(cacheHit: CacheHit) = {
+      val newEjeaCacheHit = ejeaCacheHit map { currentCacheHit =>
+        currentCacheHit.copy(hit = cacheHit, hitNumber = currentCacheHit.hitNumber + 1)
+      } getOrElse EJEACacheHit(cacheHit, 0, None)
 
-    def popCacheHitId: ResponsePendingData = cacheHit match {
-      case Some(hit) => this.copy(cacheHit = NonEmptyList.fromList(hit.cacheResultIds.tail) map CacheHit.apply)
-      case None => this
+      this.copy(ejeaCacheHit = Option(newEjeaCacheHit))
     }
+
+    def withCacheDetails(details: String) = this.copy(ejeaCacheHit = ejeaCacheHit.map(_.copy(details = Option(details))))
   }
 
   private[execution] trait ResponseData extends EJEAData {
     def response: BackendJobExecutionResponse
     def hashes: Option[Try[CallCacheHashes]]
+    def dockerImageUsed: Option[String]
   }
 
   private[execution] case class SucceededResponseData(successResponse: JobSucceededResponse,
                                                       hashes: Option[Try[CallCacheHashes]] = None) extends ResponseData {
     override def response = successResponse
+    override def dockerImageUsed = successResponse.dockerImageUsed
   }
 
   private[execution] case class NotSucceededResponseData(response: BackendJobExecutionResponse,
-                                                         hashes: Option[Try[CallCacheHashes]] = None) extends ResponseData
-
-  /**
-    * Deliberately a singleton (well, a singleton router), so we can globally rate limit hash lookups per backend.
-    *
-    * More refinement may appear via #1377.
-    */
-  private var factoryFileHashingRouters = Map[BackendLifecycleActorFactory, ActorRef]()
-
-  /**
-    * Returns a RoundRobinPool of actors based on the backend factory.
-    *
-    * @param backendName                  Name of the backend.
-    * @param backendLifecycleActorFactory A backend factory.
-    * @param actorRefFactory              An actor factory.
-    * @return a RoundRobinPool of actors based on backend factory.
-    */
-  private def factoryFileHashingRouter(backendName: String,
-                                       backendLifecycleActorFactory: BackendLifecycleActorFactory,
-                                       actorRefFactory: ActorRefFactory): ActorRef = {
-    synchronized {
-      val (originalOrUpdated, result) = getOrElseUpdated(
-        factoryFileHashingRouters, backendLifecycleActorFactory, {
-          val numberOfInstances = backendLifecycleActorFactory.fileHashingActorCount
-          val props = backendLifecycleActorFactory.fileHashingActorProps
-          actorRefFactory.actorOf(RoundRobinPool(numberOfInstances).props(props), s"FileHashingActor-$backendName")
-        }
-      )
-      factoryFileHashingRouters = originalOrUpdated
-      result
-    }
-  }
-
-  /**
-    * Immutable version of mutable.Map.getOrElseUpdate based on:
-    * http://stackoverflow.com/questions/4385976/idiomatic-get-or-else-update-for-immutable-map#answer-5840119
-    *
-    * If given key is already in this map, returns associated value in the copy of the Map.
-    *
-    * Otherwise, computes value from given expression `op`, stores with key
-    * in map and returns that value in a copy of the Map.
-    *
-    * @param map the immutable map
-    * @param key the key to test
-    * @param op  the computation yielding the value to associate with `key`, if
-    *            `key` is previously unbound.
-    * @tparam K type of the key
-    * @tparam V type of the value
-    * @return the value associated with key (either previously or as a result
-    *         of executing the method).
-    */
-  def getOrElseUpdated[K, V](map: Map[K, V], key: K, op: => V): (Map[K, V], V) = {
-    map.get(key) match {
-      case Some(value) => (map, value)
-      case None =>
-        val value = op
-        (map.updated(key, value), value)
-    }
+                                                         hashes: Option[Try[CallCacheHashes]] = None) extends ResponseData {
+    override def dockerImageUsed = None
   }
 }
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/ExecutionStore.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/ExecutionStore.scala
index 64b7c589a..906612963 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/ExecutionStore.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/ExecutionStore.scala
@@ -2,136 +2,136 @@ package cromwell.engine.workflow.lifecycle.execution
 
 import cromwell.backend.BackendJobDescriptorKey
 import cromwell.core.ExecutionStatus._
-import cromwell.core.{CallKey, ExecutionStatus, JobKey}
-import cromwell.engine.workflow.lifecycle.execution.ExecutionStore.ExecutionStoreEntry
+import cromwell.core.{CallKey, JobKey}
+import cromwell.engine.workflow.lifecycle.execution.ExecutionStore.{FqnIndex, RunnableScopes}
 import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor.{apply => _, _}
-import wdl4s._
+import wdl4s.wdl._
 
 
 object ExecutionStore {
-  def empty = ExecutionStore(Map.empty[JobKey, ExecutionStatus])
-  type ExecutionStoreEntry = (JobKey, ExecutionStatus)
-  def apply(workflow: Workflow, workflowCoercedInputs: WorkflowCoercedInputs) = {
+  case class RunnableScopes(scopes: List[JobKey], truncated: Boolean)
+  
+  private type FqnIndex = (String, Option[Int])
+
+  def empty = ExecutionStore(Map.empty[JobKey, ExecutionStatus], hasNewRunnables = false)
+
+  def apply(workflow: WdlWorkflow, workflowCoercedInputs: WorkflowCoercedInputs) = {
     // Only add direct children to the store, the rest is dynamically created when necessary
     val keys = workflow.children map {
-      case call: TaskCall => Option(BackendJobDescriptorKey(call, None, 1))
-      case call: WorkflowCall => Option(SubWorkflowKey(call, None, 1))
+      case call: WdlTaskCall => Option(BackendJobDescriptorKey(call, None, 1))
+      case call: WdlWorkflowCall => Option(SubWorkflowKey(call, None, 1))
       case scatter: Scatter => Option(ScatterKey(scatter))
       case conditional: If => Option(ConditionalKey(conditional, None))
       case declaration: Declaration => Option(DeclarationKey(declaration, None, workflowCoercedInputs))
       case _ => None
     }
 
-    new ExecutionStore(keys.flatten.map(_ -> NotStarted).toMap)
+    new ExecutionStore(keys.flatten.map(_ -> NotStarted).toMap, keys.nonEmpty)
   }
+  
+  val MaxJobsToStartPerTick = 1000
 }
 
-case class ExecutionStore(private val store: Map[JobKey, ExecutionStatus]) {
+final case class ExecutionStore(private val statusStore: Map[JobKey, ExecutionStatus], hasNewRunnables: Boolean) {
+
+  // View of the statusStore more suited for lookup based on status
+  lazy val store: Map[ExecutionStatus, List[JobKey]] = statusStore.groupBy(_._2).mapValues(_.keys.toList)
+  // Takes only keys that are done, and creates a map such that they're indexed by fqn and index
+  // This allows for quicker lookup (by hash) instead of traversing the whole list and yields
+  // significant improvements at large scale (run ExecutionStoreBenchmark)
+  lazy val (doneKeys, terminalKeys) = {
+    def toMapEntry(key: JobKey) = (key.scope.fullyQualifiedName, key.index) -> key
+
+    store.foldLeft((Map.empty[FqnIndex, JobKey], Map.empty[FqnIndex, JobKey]))({
+      case ((done, terminal), (status, keys))  =>
+        lazy val newMapEntries = keys map toMapEntry
+        val newDone = if (status.isDoneOrBypassed) done ++ newMapEntries else done
+        val newTerminal = if (status.isTerminal) terminal ++ newMapEntries else terminal
+
+        newDone -> newTerminal
+    })
+  }
+
+  private def keysWithStatus(status: ExecutionStatus) = store.getOrElse(status, List.empty)
 
-  // This is hugely ripe for optimization, if it becomes a bottleneck
   def isBypassedConditional(jobKey: JobKey, conditional: If): Boolean = {
-    store.exists {
-      case (key: ConditionalKey, status) =>
+    keysWithStatus(Bypassed).exists {
+      case key: ConditionalKey =>
         key.scope.fullyQualifiedName.equals(conditional.fullyQualifiedName) &&
-          key.index.equals(jobKey.index) &&
-          status.equals(ExecutionStatus.Bypassed)
+          key.index.equals(jobKey.index)
       case _ => false
     }
   }
 
   def hasActiveJob: Boolean = {
     def upstreamFailed(scope: Scope): Boolean = scope match {
-      case node: GraphNode => node.upstreamAncestry exists hasFailedScope
+      case node: WdlGraphNode => node.upstreamAncestry exists hasFailedScope
     }
 
-    store exists {
-      case (jobKey, jobStatus) => (jobStatus == NotStarted && !upstreamFailed(jobKey.scope)) || jobStatus == QueuedInCromwell || jobStatus == Starting || jobStatus == Running
-    }
+    keysWithStatus(QueuedInCromwell).nonEmpty ||
+      keysWithStatus(Starting).nonEmpty ||
+      keysWithStatus(Running).nonEmpty ||
+      keysWithStatus(NotStarted).exists(jobKey => !upstreamFailed(jobKey.scope))
   }
 
-  def jobStatus(jobKey: JobKey): Option[ExecutionStatus] = store.get(jobKey)
+  def jobStatus(jobKey: JobKey): Option[ExecutionStatus] = statusStore.get(jobKey)
 
-  def unstartedJobs: List[BackendJobDescriptorKey] = {
-    store.collect({case (k: BackendJobDescriptorKey, status) if status != ExecutionStatus.NotStarted => k }).toList
+  def startedJobs: List[BackendJobDescriptorKey] = {
+    store.filterNot({ case (s, _) => s == NotStarted}).values.toList.flatten collect {
+      case k: BackendJobDescriptorKey => k
+    }
   }
 
-  private def hasFailedScope(s: GraphNode): Boolean = {
-    store.exists({ case (key, status) => status == Failed && key.scope == s })
-  }
+  private def hasFailedScope(s: WdlGraphNode): Boolean = keysWithStatus(Failed).exists(_.scope == s)
 
-  def hasFailedJob: Boolean = {
-    store.values.exists(_ == ExecutionStatus.Failed)
-  }
+  def hasFailedJob: Boolean = keysWithStatus(Failed).nonEmpty
 
   override def toString = store.map { case (j, s) => s"$j -> $s" } mkString System.lineSeparator()
 
-  def add(values: Map[JobKey, ExecutionStatus]) = this.copy(store = store ++ values)
-
-  // Convert the store to a `List` before `collect`ing to sidestep expensive and pointless hashing of `Scope`s when
-  // assembling the result.
-  def runnableScopes = store.toList collect { case entry if isRunnable(entry) => entry._1 }
-
-  private def isRunnable(entry: ExecutionStoreEntry) = {
-    entry match {
-      case (key, ExecutionStatus.NotStarted) => arePrerequisitesDone(key)
-      case _ => false
-    }
+  def add(values: Map[JobKey, ExecutionStatus]) = {
+    this.copy(statusStore = statusStore ++ values, hasNewRunnables = hasNewRunnables || values.values.exists(_.isTerminalOrRetryable))
   }
 
-  def findCompletedShardsForOutput(key: CollectorKey): List[JobKey] = store.toList collect {
-    case (k: CallKey, v) if v.isDoneOrBypassed && k.scope == key.scope && k.isShard => k
-    case (k: DynamicDeclarationKey, v) if v.isDoneOrBypassed && k.scope == key.scope && k.isShard => k
+  /**
+    * Returns the list of jobs ready to be run, along with a Boolean indicating whether or not the list has been truncated.
+    * The size of the list will be MaxJobsToStartPerTick at most. If more jobs where found runnable, the boolean will be true, otherwise false.
+    */
+  def runnableScopes: RunnableScopes = {
+    val readyToStart = keysWithStatus(NotStarted).toStream filter arePrerequisitesDone
+    // Compute the first ExecutionStore.MaxJobsToStartPerTick + 1 runnable scopes
+    val scopesToStartPlusOne = readyToStart.take(ExecutionStore.MaxJobsToStartPerTick + 1).toList
+    // Only take the first ExecutionStore.MaxJobsToStartPerTick from the above list.
+    // Use the fact that we took one more to determine whether or not we truncated the result.
+    RunnableScopes(scopesToStartPlusOne.take(ExecutionStore.MaxJobsToStartPerTick), scopesToStartPlusOne.size > ExecutionStore.MaxJobsToStartPerTick)
   }
 
-  // Just used to decide whether a collector can be run. In case the shard entries haven't been populated into the
-  // execution store yet.
-  case class TempJobKey(scope: Scope with GraphNode, index: Option[Int]) extends JobKey {
-    // If these are ever used, we've done something wrong...
-    override def attempt: Int = throw new NotImplementedError("We've done something wrong.")
-    override def tag: String = throw new NotImplementedError("We've done something wrong.")
+  def findCompletedShardsForOutput(key: CollectorKey): List[JobKey] = doneKeys.values.toList collect {
+    case k @ (_: CallKey | _:DynamicDeclarationKey) if k.scope == key.scope && k.isShard => k
   }
-  def emulateShardEntries(key: CollectorKey): List[JobKey] = {
-    (0 until key.scatterWidth).toList flatMap { i => key.scope match {
-      case c: Call => List(TempJobKey(c, Option(i)))
-      case d: Declaration => List(TempJobKey(d, Option(i)))
+
+  private def emulateShardEntries(key: CollectorKey): Set[FqnIndex] = {
+    (0 until key.scatterWidth).toSet map { i: Int => key.scope match {
+      case c: WdlCall => c.fullyQualifiedName -> Option(i)
+      case d: Declaration => d.fullyQualifiedName -> Option(i)
       case _ => throw new RuntimeException("Don't collect that.")
     }}
   }
 
   private def arePrerequisitesDone(key: JobKey): Boolean = {
-    val upstream = key.scope.upstream collect {
-      case n: Call => upstreamEntry(key, n)
-      case n: Scatter => upstreamEntry(key, n)
-      case n: Declaration => upstreamEntry(key, n)
-    }
-
-    val shardEntriesForCollector: List[JobKey] = key match {
-      case collector: CollectorKey => emulateShardEntries(collector)
-      case _ => Nil
+    lazy val upstreamAreDone = key.scope.upstream forall {
+      case n @ (_: WdlCall | _: Scatter | _: Declaration) => upstreamIsDone(key, n)
+      case _ => true
     }
 
-    /*
-      * We need to use an "exists" in this case because the execution store can contain a job attempt with the same
-      * fqn and index but a preempted status. We wouldn't want that preempted attempt to count against the completion
-      * of the scatter block.
-      */
-    def isDone(e: JobKey): Boolean = store exists {
-      case (k, s) => k.scope.fullyQualifiedName == e.scope.fullyQualifiedName && k.index == e.index && s.isDoneOrBypassed
+    val shardEntriesForCollectorAreTerminal: Boolean = key match {
+      case collector: CollectorKey => emulateShardEntries(collector).diff(terminalKeys.keys.toSet).isEmpty
+      case _ => true
     }
 
-    val dependencies = upstream.flatten.map(_._1) ++ shardEntriesForCollector
-    val dependenciesResolved = dependencies forall { isDone }
-
-    /*
-      * We need to make sure that all prerequisiteScopes have been resolved to some entry before going forward.
-      * If a scope cannot be resolved it may be because it is in a scatter that has not been populated yet,
-      * therefore there is no entry in the executionStore for this scope.
-      * If that's the case this prerequisiteScope has not been run yet, hence the (upstream forall {_.nonEmpty})
-      */
-    (upstream forall { _.nonEmpty }) && dependenciesResolved
+    shardEntriesForCollectorAreTerminal && upstreamAreDone
   }
 
-  private def upstreamEntry(entry: JobKey, prerequisiteScope: Scope): Option[ExecutionStoreEntry] = {
+  private def upstreamIsDone(entry: JobKey, prerequisiteScope: Scope): Boolean = {
     prerequisiteScope.closestCommonAncestor(entry.scope) match {
       /*
         * If this entry refers to a Scope which has a common ancestor with prerequisiteScope
@@ -142,19 +142,13 @@ case class ExecutionStore(private val store: Map[JobKey, ExecutionStatus]) {
         * NOTE: this algorithm was designed for ONE-LEVEL of scattering and probably does not
         * work as-is for nested scatter blocks
         */
-      case Some(ancestor: Scatter) =>
-        store find {
-          case (k, _) => k.scope == prerequisiteScope && k.index == entry.index
-        }
+      case Some(_: Scatter) => doneKeys.contains(prerequisiteScope.fullyQualifiedName -> entry.index)
 
       /*
         * Otherwise, simply refer to the collector entry.  This means that 'entry' depends
         * on every shard of the pre-requisite scope to finish.
         */
-      case _ =>
-        store find {
-          case (k, _) => k.scope == prerequisiteScope && k.index.isEmpty
-        }
+      case _ => doneKeys.contains(prerequisiteScope.fullyQualifiedName -> None)
     }
   }
 }
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/OutputStore.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/OutputStore.scala
index 9223ec1c6..08bd9b821 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/OutputStore.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/OutputStore.scala
@@ -5,16 +5,16 @@ import cromwell.core._
 import cromwell.engine.workflow.lifecycle.execution.OutputStore.{OutputCallKey, OutputEntry}
 import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor.CollectorKey
 import lenthall.util.TryUtil
-import wdl4s.types.{WdlArrayType, WdlType}
-import wdl4s.values.{WdlArray, WdlCallOutputsObject, WdlValue}
-import wdl4s._
+import wdl4s.wdl.types.{WdlArrayType, WdlType}
+import wdl4s.wdl.values.{WdlArray, WdlCallOutputsObject, WdlValue}
+import wdl4s.wdl._
 
 import scala.language.postfixOps
 import scala.util.{Failure, Success, Try}
 
 object OutputStore {
   case class OutputEntry(name: String, wdlType: WdlType, wdlValue: Option[WdlValue])
-  case class OutputCallKey(call: Scope with GraphNode, index: ExecutionIndex)
+  case class OutputCallKey(call: Scope with WdlGraphNode, index: ExecutionIndex)
   def empty = OutputStore(Map.empty)
 }
 
@@ -24,7 +24,7 @@ case class OutputStore(store: Map[OutputCallKey, List[OutputEntry]]) {
 
   def add(values: Map[OutputCallKey, List[OutputEntry]]) = this.copy(store = store ++ values)
 
-  def fetchNodeOutputEntries(node: GraphNode, index: ExecutionIndex): Try[WdlValue] = {
+  def fetchNodeOutputEntries(node: WdlGraphNode, index: ExecutionIndex): Try[WdlValue] = {
     def outputEntriesToMap(outputs: List[OutputEntry]): Map[String, Try[WdlValue]] = {
       outputs map { output =>
         output.wdlValue match {
@@ -34,7 +34,7 @@ case class OutputStore(store: Map[OutputCallKey, List[OutputEntry]]) {
       } toMap
     }
 
-    def callOutputs(call: Call, outputs: List[OutputEntry]) = {
+    def callOutputs(call: WdlCall, outputs: List[OutputEntry]) = {
       TryUtil.sequenceMap(outputEntriesToMap(outputs), s"Output fetching for call ${node.unqualifiedName}") map { outputsMap =>
         WdlCallOutputsObject(call, outputsMap)
       }
@@ -42,7 +42,7 @@ case class OutputStore(store: Map[OutputCallKey, List[OutputEntry]]) {
     
     def declarationOutputs(declaration: Declaration, outputs: List[OutputEntry]) = {
       outputs match {
-        case OutputEntry(name, _, Some(value)) :: Nil => Success(value)
+        case OutputEntry(_, _, Some(value)) :: Nil => Success(value)
         case _ => Failure(new RuntimeException(s"Could not find value for declaration ${declaration.fullyQualifiedName}"))
       }
     }
@@ -50,7 +50,7 @@ case class OutputStore(store: Map[OutputCallKey, List[OutputEntry]]) {
     store.get(OutputCallKey(node, index)) match {
       case Some(outputs) =>
         node match {
-          case call: Call => callOutputs(call, outputs)
+          case call: WdlCall => callOutputs(call, outputs)
           case declaration: Declaration => declarationOutputs(declaration, outputs)
           case other =>  Failure(new RuntimeException(s"Only Calls and Declarations are allowed in the OutputStore, found ${other.getClass.getSimpleName}"))
         }
@@ -58,7 +58,7 @@ case class OutputStore(store: Map[OutputCallKey, List[OutputEntry]]) {
     }
   }
   
-  def collectCall(call: Call, scatter: Scatter, sortedShards: Seq[JobKey]) = Try {
+  def collectCall(call: WdlCall, scatter: Scatter, sortedShards: Seq[JobKey]) = Try {
     val shardsOutputs = sortedShards map { e =>
       fetchNodeOutputEntries(call, e.index) map {
         case callOutputs: WdlCallOutputsObject => callOutputs.outputs
@@ -94,7 +94,7 @@ case class OutputStore(store: Map[OutputCallKey, List[OutputEntry]]) {
     lazy val sortedShards = shards.toSeq sortBy { _.index.fromIndex }
     
     collector.scope match {
-      case call: Call => collectCall(call, collector.scatter, sortedShards)
+      case call: WdlCall => collectCall(call, collector.scatter, sortedShards)
       case declaration: Declaration => collectDeclaration(declaration, collector.scatter, sortedShards)
       case other => Failure(new RuntimeException(s"Cannot retrieve outputs for ${other.fullyQualifiedName}")) 
     }
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/SubWorkflowExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/SubWorkflowExecutionActor.scala
index 007326908..1acd49094 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/SubWorkflowExecutionActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/SubWorkflowExecutionActor.scala
@@ -16,16 +16,18 @@ import cromwell.engine.workflow.lifecycle.execution.preparation.SubWorkflowPrepa
 import cromwell.services.metadata.MetadataService._
 import cromwell.services.metadata._
 import cromwell.subworkflowstore.SubWorkflowStoreActor._
-import wdl4s.EvaluatedTaskInputs
+import wdl4s.wdl.EvaluatedTaskInputs
 
 class SubWorkflowExecutionActor(key: SubWorkflowKey,
                                 data: WorkflowExecutionActorData,
                                 factories: Map[String, BackendLifecycleActorFactory],
+                                ioActor: ActorRef,
                                 override val serviceRegistryActor: ActorRef,
                                 jobStoreActor: ActorRef,
                                 subWorkflowStoreActor: ActorRef,
                                 callCacheReadActor: ActorRef,
-                                dockerHashActor: ActorRef,
+                                callCacheWriteActor: ActorRef,
+                                workflowDockerLookupActor: ActorRef,
                                 jobTokenDispenserActor: ActorRef,
                                 backendSingletonCollection: BackendSingletonCollection,
                                 initializationData: AllBackendInitializationData,
@@ -39,9 +41,9 @@ class SubWorkflowExecutionActor(key: SubWorkflowKey,
   override def jobTag: String = key.tag
 
   startWith(SubWorkflowPendingState, SubWorkflowExecutionActorData.empty)
-  
+
   private var eventList: Seq[ExecutionEvent] = Seq(ExecutionEvent(stateName.toString))
-  
+
   when(SubWorkflowPendingState) {
     case Event(Execute, _) =>
       if (restarting) {
@@ -51,9 +53,9 @@ class SubWorkflowExecutionActor(key: SubWorkflowKey,
         prepareSubWorkflow(createSubWorkflowId())
       }
   }
-  
+
   when(SubWorkflowCheckingStoreState) {
-    case Event(SubWorkflowFound(entry), _) => 
+    case Event(SubWorkflowFound(entry), _) =>
       prepareSubWorkflow(WorkflowId.fromString(entry.subWorkflowExecutionUuid))
     case Event(_: SubWorkflowNotFound, _) =>
       prepareSubWorkflow(createSubWorkflowId())
@@ -61,7 +63,7 @@ class SubWorkflowExecutionActor(key: SubWorkflowKey,
       jobLogger.error(reason, s"SubWorkflowStore failure for command $command, starting sub workflow with fresh ID.")
       prepareSubWorkflow(createSubWorkflowId())
   }
-  
+
   when(SubWorkflowPreparingState) {
     case Event(SubWorkflowPreparationSucceeded(subWorkflowEngineDescriptor, inputs), _) =>
       startSubWorkflow(subWorkflowEngineDescriptor, inputs)
@@ -70,7 +72,7 @@ class SubWorkflowExecutionActor(key: SubWorkflowKey,
       context stop self
       stay()
   }
-  
+
   when(SubWorkflowRunningState) {
     case Event(WorkflowExecutionSucceededResponse(executedJobKeys, outputs), _) =>
       context.parent ! SubWorkflowSucceededResponse(key, executedJobKeys, outputs)
@@ -82,16 +84,13 @@ class SubWorkflowExecutionActor(key: SubWorkflowKey,
       context.parent ! SubWorkflowAbortedResponse(key, executedJobKeys)
       goto(SubWorkflowAbortedState)
   }
-  
+
   when(SubWorkflowSucceededState) { FSM.NullFunction }
   when(SubWorkflowFailedState) { FSM.NullFunction }
   when(SubWorkflowAbortedState) { FSM.NullFunction }
-  
+
   whenUnhandled {
-    case Event(SubWorkflowStoreRegisterSuccess(command), _) =>
-      // Nothing to do here
-      stay()
-    case Event(SubWorkflowStoreCompleteSuccess(command), _) =>
+    case Event(SubWorkflowStoreRegisterSuccess(_), _) =>
       // Nothing to do here
       stay()
     case Event(SubWorkflowStoreFailure(command, reason), _) =>
@@ -100,35 +99,35 @@ class SubWorkflowExecutionActor(key: SubWorkflowKey,
   }
 
   onTransition {
-    case (fromState, toState) =>
-      stateData.subWorkflowId foreach { id => pushCurrentStateToMetadataService(id, toState.workflowState) } 
+    case (_, toState) =>
+      stateData.subWorkflowId foreach { id => pushCurrentStateToMetadataService(id, toState.workflowState) }
   }
-  
+
   onTransition {
-    case (fromState, subWorkflowTerminalState: SubWorkflowTerminalState) =>
+    case (_, _: SubWorkflowTerminalState) =>
       stateData.subWorkflowId match {
-        case Some(id) => 
+        case Some(id) =>
           pushWorkflowEnd(id)
           pushExecutionEventsToMetadataService(key, eventList)
         case None => jobLogger.error("Sub workflow completed without a Sub Workflow UUID.")
-      } 
+      }
       context stop self
   }
-  
+
   onTransition {
-    case fromState -> toState => eventList :+= ExecutionEvent(toState.toString)
+    case _ -> toState => eventList :+= ExecutionEvent(toState.toString)
   }
-  
+
   private def startSubWorkflow(subWorkflowEngineDescriptor: EngineWorkflowDescriptor, inputs: EvaluatedTaskInputs) = {
     val subWorkflowActor = createSubWorkflowActor(subWorkflowEngineDescriptor)
-    
+
     subWorkflowActor ! WorkflowExecutionActor.ExecuteWorkflowCommand
     context.parent ! JobRunning(key, inputs, Option(subWorkflowActor))
     pushWorkflowRunningMetadata(subWorkflowEngineDescriptor.backendDescriptor, inputs)
-    
+
     goto(SubWorkflowRunningState)
   }
-  
+
   private def prepareSubWorkflow(subWorkflowId: WorkflowId) = {
     createSubWorkflowPreparationActor(subWorkflowId) ! Start
     context.parent ! JobStarting(key)
@@ -136,24 +135,26 @@ class SubWorkflowExecutionActor(key: SubWorkflowKey,
     pushWorkflowStart(subWorkflowId)
     goto(SubWorkflowPreparingState) using SubWorkflowExecutionActorData(Option(subWorkflowId))
   }
-  
+
   def createSubWorkflowPreparationActor(subWorkflowId: WorkflowId) = {
     context.actorOf(
       SubWorkflowPreparationActor.props(data, key, subWorkflowId),
       s"$subWorkflowId-SubWorkflowPreparationActor-${key.tag}"
     )
   }
-  
+
   def createSubWorkflowActor(subWorkflowEngineDescriptor: EngineWorkflowDescriptor) = {
     context.actorOf(
       WorkflowExecutionActor.props(
       subWorkflowEngineDescriptor,
-      serviceRegistryActor,
-      jobStoreActor,
-      subWorkflowStoreActor,
-      callCacheReadActor,
-      dockerHashActor,
-      jobTokenDispenserActor,
+      ioActor = ioActor,
+      serviceRegistryActor = serviceRegistryActor,
+      jobStoreActor = jobStoreActor,
+      subWorkflowStoreActor = subWorkflowStoreActor,
+      callCacheReadActor = callCacheReadActor,
+      callCacheWriteActor = callCacheWriteActor,
+      workflowDockerLookupActor = workflowDockerLookupActor,
+      jobTokenDispenserActor = jobTokenDispenserActor,
       backendSingletonCollection,
       initializationData,
       restarting
@@ -251,11 +252,13 @@ object SubWorkflowExecutionActor {
   def props(key: SubWorkflowKey,
             data: WorkflowExecutionActorData,
             factories: Map[String, BackendLifecycleActorFactory],
+            ioActor: ActorRef,
             serviceRegistryActor: ActorRef,
             jobStoreActor: ActorRef,
             subWorkflowStoreActor: ActorRef,
             callCacheReadActor: ActorRef,
-            dockerHashActor: ActorRef,
+            callCacheWriteActor: ActorRef,
+            workflowDockerLookupActor: ActorRef,
             jobTokenDispenserActor: ActorRef,
             backendSingletonCollection: BackendSingletonCollection,
             initializationData: AllBackendInitializationData,
@@ -264,12 +267,14 @@ object SubWorkflowExecutionActor {
       key,
       data,
       factories,
-      serviceRegistryActor,
-      jobStoreActor,
-      subWorkflowStoreActor,
-      callCacheReadActor,
-      dockerHashActor,
-      jobTokenDispenserActor,
+      ioActor = ioActor,
+      serviceRegistryActor = serviceRegistryActor,
+      jobStoreActor = jobStoreActor,
+      subWorkflowStoreActor = subWorkflowStoreActor,
+      callCacheReadActor = callCacheReadActor,
+      callCacheWriteActor = callCacheWriteActor,
+      workflowDockerLookupActor = workflowDockerLookupActor,
+      jobTokenDispenserActor = jobTokenDispenserActor,
       backendSingletonCollection,
       initializationData,
       restarting)
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala
index 9135696cf..7148acdb3 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala
@@ -1,18 +1,17 @@
 package cromwell.engine.workflow.lifecycle.execution
 
-import akka.actor._
+import akka.actor.{Scope => _, _}
 import cats.data.NonEmptyList
-import com.typesafe.config.ConfigFactory
 import cromwell.backend.BackendJobExecutionActor.{AbortedResponse, JobFailedNonRetryableResponse, JobFailedRetryableResponse, JobSucceededResponse}
 import cromwell.backend.BackendLifecycleActor.AbortJobCommand
 import cromwell.backend.{AllBackendInitializationData, BackendJobDescriptorKey, JobExecutionMap}
 import cromwell.core.Dispatcher._
 import cromwell.core.ExecutionIndex._
 import cromwell.core.ExecutionStatus._
-import cromwell.core.WorkflowOptions.WorkflowFailureMode
 import cromwell.core._
 import cromwell.core.logging.WorkflowLogging
 import cromwell.engine.backend.{BackendSingletonCollection, CromwellBackends}
+import cromwell.engine.workflow.lifecycle.execution.ExecutionStore.RunnableScopes
 import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor._
 import cromwell.engine.workflow.lifecycle.{EngineLifecycleActorAbortCommand, EngineLifecycleActorAbortedResponse}
 import cromwell.engine.{ContinueWhilePossible, EngineWorkflowDescriptor}
@@ -20,21 +19,25 @@ import cromwell.util.StopAndLogSupervisor
 import cromwell.webservice.EngineStatsActor
 import lenthall.exception.ThrowableAggregation
 import lenthall.util.TryUtil
-import net.ceedubs.ficus.Ficus._
-import wdl4s.values.{WdlArray, WdlBoolean, WdlOptionalValue, WdlValue, WdlString}
 import org.apache.commons.lang3.StringUtils
-import wdl4s.{Scope, _}
+import wdl4s.wdl.WdlExpression.ScopedLookupFunction
+import wdl4s.wdl._
+import wdl4s.wdl.expression.WdlFunctions
+import wdl4s.wdl.values.WdlArray.WdlArrayLike
+import wdl4s.wdl.values.{WdlBoolean, WdlOptionalValue, WdlString, WdlValue}
 
-import scala.annotation.tailrec
+import scala.concurrent.duration._
 import scala.language.postfixOps
 import scala.util.{Failure, Success, Try}
 
 case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor,
+                                  ioActor: ActorRef,
                                   serviceRegistryActor: ActorRef,
                                   jobStoreActor: ActorRef,
                                   subWorkflowStoreActor: ActorRef,
                                   callCacheReadActor: ActorRef,
-                                  dockerHashActor: ActorRef,
+                                  callCacheWriteActor: ActorRef,
+                                  workflowDockerLookupActor: ActorRef,
                                   jobTokenDispenserActor: ActorRef,
                                   backendSingletonCollection: BackendSingletonCollection,
                                   initializationData: AllBackendInitializationData,
@@ -47,13 +50,9 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor,
   override val workflowIdForCallMetadata = workflowDescriptor.id
 
   private val tag = s"WorkflowExecutionActor [UUID(${workflowDescriptor.id.shortString})]"
-  private val MaxRetries = ConfigFactory.load().as[Option[Int]]("system.max-retries") match {
-    case Some(value) => value
-    case None =>
-      workflowLogger.warn(s"Failed to load the max-retries value from the configuration. Defaulting back to a value of '$DefaultMaxRetriesFallbackValue'.")
-      DefaultMaxRetriesFallbackValue
-  }
   
+  private var checkRunnableCancellable: Option[Cancellable] = None
+
   private val backendFactories = TryUtil.sequenceMap(workflowDescriptor.backendAssignments.values.toSet[String] map { backendName =>
     backendName -> CromwellBackends.backendLifecycleFactoryActorByName(backendName)
   } toMap) recover {
@@ -74,12 +73,14 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor,
   )
 
   when(WorkflowExecutionPendingState) {
-    case Event(ExecuteWorkflowCommand, stateData) =>
-      val data = startRunnableScopes(stateData)
-      goto(WorkflowExecutionInProgressState) using data
+    case Event(ExecuteWorkflowCommand, _) =>
+      scheduleStartRunnableCalls()
+      goto(WorkflowExecutionInProgressState)
   }
 
   when(WorkflowExecutionInProgressState) {
+    case Event(CheckRunnable, data) => handleCheckRunnable(data)
+
     case Event(JobStarting(jobKey), stateData) =>
       pushStartingCallMetadata(jobKey)
       stay() using stateData
@@ -92,9 +93,9 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor,
       
       //Success
         // Job
-    case Event(JobSucceededResponse(jobKey, returnCode, callOutputs, _, _), stateData) =>
-      pushSuccessfulCallMetadata(jobKey, returnCode, callOutputs)
-      handleCallSuccessful(jobKey, callOutputs, stateData, Map.empty)
+    case Event(r: JobSucceededResponse, stateData) =>
+      pushSuccessfulCallMetadata(r.jobKey, r.returnCode, r.jobOutputs)
+      handleCallSuccessful(r.jobKey, r.jobOutputs, stateData, Map.empty)
         // Sub Workflow
     case Event(SubWorkflowSucceededResponse(jobKey, descendantJobKeys, callOutputs), stateData) =>
       pushSuccessfulCallMetadata(jobKey, None, callOutputs)
@@ -111,20 +112,29 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor,
     case Event(BypassedDeclaration(declKey), stateData) =>
       handleDeclarationEvaluationSuccessful(declKey, WdlOptionalValue.none(declKey.scope.wdlType), stateData)
 
-      // Failure
-        // Initialization
+    // Failure
+    // Initialization
     case Event(JobInitializationFailed(jobKey, reason), stateData) =>
       pushFailedCallMetadata(jobKey, None, reason, retryableFailure = false)
       handleNonRetryableFailure(stateData, jobKey, reason, Map.empty)
-      // Job Non Retryable
+    // Job Non Retryable
     case Event(JobFailedNonRetryableResponse(jobKey, reason, returnCode), stateData) =>
       pushFailedCallMetadata(jobKey, returnCode, reason, retryableFailure = false)
       handleNonRetryableFailure(stateData, jobKey, reason, Map.empty)
-      // Job Retryable
+    // Aborted? But we're outside of the AbortingState!?? Could happen if
+      // - The job was aborted by something external to Cromwell
+      // - The job lasted too long (eg JES 6 day timeout)
+      // - We've reconnected to an aborting job (some sort of shutdown race condition?)
+    // Treat it like any other non-retryable failure:
+    case Event(AbortedResponse(jobKey), stateData) =>
+      val cause = new Exception("The job was aborted from outside Cromwell")
+      pushFailedCallMetadata(jobKey, None, cause, retryableFailure = false)
+      handleNonRetryableFailure(stateData, jobKey, cause, Map.empty)
+    // Job Retryable
     case Event(JobFailedRetryableResponse(jobKey, reason, returnCode), _) =>
       pushFailedCallMetadata(jobKey, None, reason, retryableFailure = true)
       handleRetryableFailure(jobKey, reason, returnCode)
-      // Sub Workflow - sub workflow failures are always non retryable
+    // Sub Workflow - sub workflow failures are always non retryable
     case Event(SubWorkflowFailedResponse(jobKey, descendantJobKeys, reason), stateData) =>
       pushFailedCallMetadata(jobKey, None, reason, retryableFailure = false)
       handleNonRetryableFailure(stateData, jobKey, reason, descendantJobKeys)
@@ -142,39 +152,29 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor,
     case Event(SubWorkflowSucceededResponse(subKey, executedKeys, _), stateData) =>
       pushAbortedCallMetadata(subKey)
       handleCallAborted(stateData, subKey, executedKeys)
-    case Event(JobSucceededResponse(jobKey, _, _, _, _), stateData) =>
-      pushAbortedCallMetadata(jobKey)
-      handleCallAborted(stateData, jobKey, Map.empty)
+    case Event(r: JobSucceededResponse, stateData) =>
+      pushAbortedCallMetadata(r.jobKey)
+      handleCallAborted(stateData, r.jobKey, Map.empty)
   }
   
   when(WorkflowExecutionSuccessfulState) {
     FSM.NullFunction
   }
   when(WorkflowExecutionFailedState) {
-    alreadyFailedMopUp
+    FSM.NullFunction
   }
   when(WorkflowExecutionAbortedState) {
-    alreadyFailedMopUp
+    FSM.NullFunction
   }
 
-  /**
-    * Mop up function to handle a set of incoming results if this workflow has already failed:
-    */
-  private def alreadyFailedMopUp: StateFunction = {
-    case Event(JobInitializationFailed(jobKey, reason), _) =>
-      pushFailedCallMetadata(jobKey, None, reason, retryableFailure = false)
-      stay
-    case Event(JobFailedNonRetryableResponse(jobKey, reason, returnCode), _) =>
-      pushFailedCallMetadata(jobKey, returnCode, reason, retryableFailure = false)
-      stay
-    case Event(JobFailedRetryableResponse(jobKey, reason, returnCode), _) =>
-      pushFailedCallMetadata(jobKey, returnCode, reason, retryableFailure = true)
-      stay
-    case Event(JobSucceededResponse(jobKey, returnCode, callOutputs, _, _), _) =>
-      pushSuccessfulCallMetadata(jobKey, returnCode, callOutputs)
-      stay
+  private def scheduleStartRunnableCalls() = {
+    checkRunnableCancellable = Option(context.system.scheduler.scheduleOnce(SweepInterval, self, CheckRunnable))
+  }
+  
+  override def postStop() = {
+    checkRunnableCancellable foreach { _.cancel() }
+    super.postStop()
   }
-
 
   def handleTerminated(actorRef: ActorRef) = {
     // Both of these Should Never Happen (tm), assuming the state data is set correctly on EJEA creation.
@@ -182,10 +182,10 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor,
     val jobKey = stateData.engineCallExecutionActors.getOrElse(actorRef, throw new RuntimeException("Programmer Error: An EJEA has terminated but was not assigned a jobKey"))
     val jobStatus = stateData.executionStore.jobStatus(jobKey).getOrElse(throw new RuntimeException("Programmer Error: An EJEA representing a jobKey which this workflow is not running has sent up a terminated message."))
 
-    if (!jobStatus.isTerminal) {
+    if (!jobStatus.isTerminalOrRetryable) {
       val terminationException = getFailureCause(actorRef) match {
-        case Some(e) => new RuntimeException("Unexpected failure in EJEA.", e)
-        case None => new RuntimeException("Unexpected failure in EJEA (root cause not captured).")
+        case Some(e) => new RuntimeException("Unexpected failure (or early exit) in EJEA.", e)
+        case None => new RuntimeException(s"Unexpected failure (or early exit) in EJEA $actorRef (root cause not captured).")
       }
       self ! JobFailedNonRetryableResponse(jobKey, terminationException, None)
     }
@@ -194,6 +194,7 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor,
   }
 
   whenUnhandled {
+    case Event(CheckRunnable, _) => stay()
     case Event(Terminated(actorRef), stateData) => handleTerminated(actorRef) using stateData.removeEngineJobExecutionActor(actorRef)
     case Event(EngineLifecycleActorAbortCommand, stateData) =>
       if (stateData.hasRunningActors) {
@@ -244,13 +245,13 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor,
   private def handleExecutionFailure(failedJobKey: JobKey, data: WorkflowExecutionActorData, reason: Throwable, jobExecutionMap: JobExecutionMap) = {
     val newData = data.executionFailed(failedJobKey)
     
-    if (workflowDescriptor.getWorkflowOption(WorkflowFailureMode).contains(ContinueWhilePossible.toString)) {
+    if (workflowDescriptor.failureMode == ContinueWhilePossible) {
       newData.workflowCompletionStatus match {
         case Some(completionStatus) if completionStatus == Failed =>
           context.parent ! WorkflowExecutionFailedResponse(newData.jobExecutionMap, reason)
           goto(WorkflowExecutionFailedState) using newData
         case _ =>
-          stay() using startRunnableScopes(newData)
+          stay() using newData
       }
     } else {
       context.parent ! WorkflowExecutionFailedResponse(newData.jobExecutionMap, reason)
@@ -299,45 +300,40 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor,
   }
 
   private def handleRetryableFailure(jobKey: BackendJobDescriptorKey, reason: Throwable, returnCode: Option[Int]) = {
-    // We start with index 1 for #attempts, hence invariant breaks only if jobKey.attempt > MaxRetries
-    if (jobKey.attempt <= MaxRetries) {
-      val newJobKey = jobKey.copy(attempt = jobKey.attempt + 1)
-      workflowLogger.info(s"Retrying job execution for ${newJobKey.tag}")
-      /*  Currently, we update the status of the old key to Preempted, and add a new entry (with the #attempts incremented by 1)
-        * to the execution store with status as NotStarted. This allows startRunnableCalls to re-execute this job */
-      val executionDiff = WorkflowExecutionDiff(Map(jobKey -> ExecutionStatus.Preempted, newJobKey -> ExecutionStatus.NotStarted))
-      val newData = stateData.mergeExecutionDiff(executionDiff).removeCallExecutionActor(jobKey)
-      stay() using startRunnableScopes(newData)
-    } else {
-      workflowLogger.warn(s"Exhausted maximum number of retries for job ${jobKey.tag}. Failing.")
-      goto(WorkflowExecutionFailedState) using stateData.mergeExecutionDiff(WorkflowExecutionDiff(Map(jobKey -> ExecutionStatus.Failed))).removeCallExecutionActor(jobKey)
-    }
+    val newJobKey = jobKey.copy(attempt = jobKey.attempt + 1)
+    workflowLogger.info(s"Retrying job execution for ${newJobKey.tag}")
+    /*  Currently, we update the status of the old key to RetryableFailure, and add a new entry (with the #attempts incremented by 1)
+      * to the execution store with status as NotStarted. This allows startRunnableCalls to re-execute this job */
+    val executionDiff = WorkflowExecutionDiff(Map(jobKey -> ExecutionStatus.RetryableFailure, newJobKey -> ExecutionStatus.NotStarted))
+    val newData = stateData.mergeExecutionDiff(executionDiff).removeCallExecutionActor(jobKey)
+    stay() using newData
   }
 
   private def handleCallSuccessful(jobKey: JobKey, outputs: CallOutputs, data: WorkflowExecutionActorData, jobExecutionMap: JobExecutionMap) = {
-    handleExecutionSuccess(data.callExecutionSuccess(jobKey, outputs).addExecutions(jobExecutionMap))
+    stay() using data.callExecutionSuccess(jobKey, outputs).addExecutions(jobExecutionMap)
   }
   
   private def handleDeclarationEvaluationSuccessful(key: DeclarationKey, value: WdlValue, data: WorkflowExecutionActorData) = {
-    handleExecutionSuccess(data.declarationEvaluationSuccess(key, value))
+    stay() using data.declarationEvaluationSuccess(key, value)
   }
 
   private def handleCallBypassed(callOutputs: Map[CallKey, CallOutputs], data: WorkflowExecutionActorData) = {
     def foldFunc(d: WorkflowExecutionActorData, output: (CallKey, CallOutputs)) = d.callExecutionSuccess(output._1, output._2)
 
     val updatedData = callOutputs.foldLeft(data)(foldFunc)
-    handleExecutionSuccess(updatedData)
+    stay() using updatedData
   }
 
-  private def handleExecutionSuccess(data: WorkflowExecutionActorData) = {
+  private def handleCheckRunnable(data: WorkflowExecutionActorData) = {
     data.workflowCompletionStatus match {
       case Some(ExecutionStatus.Done) =>
         handleWorkflowSuccessful(data)
       case Some(_) =>
-        context.parent ! WorkflowExecutionFailedResponse(data.jobExecutionMap, new Exception("One or more jobs failed in fail-slow mode"))
+        context.parent ! WorkflowExecutionFailedResponse(data.jobExecutionMap, new Exception("One or more jobs failed in ContinueWhilePossible mode"))
         goto(WorkflowExecutionFailedState) using data
       case _ =>
-        stay() using startRunnableScopes(data)
+        scheduleStartRunnableCalls()
+        if (data.hasNewRunnables) stay() using startRunnableScopes(data) else stay()
     }
   }
   
@@ -359,11 +355,11 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor,
     * Attempt to start all runnable jobs and return updated state data.  This will create a new copy
     * of the state data.
     */
-  @tailrec
   private def startRunnableScopes(data: WorkflowExecutionActorData): WorkflowExecutionActorData = {
-    val runnableScopes = data.executionStore.runnableScopes
-    val runnableCalls = runnableScopes.view collect { case k if k.scope.isInstanceOf[Call] => k } sortBy { k =>
+    val RunnableScopes(runnableScopes, truncated) = data.executionStore.runnableScopes
+    val runnableCalls = runnableScopes.view collect { case k if k.scope.isInstanceOf[WdlCall] => k } sortBy { k =>
       (k.scope.fullyQualifiedName, k.index.getOrElse(-1)) } map { _.tag }
+
     if (runnableCalls.nonEmpty) workflowLogger.info("Starting calls: " + runnableCalls.mkString(", "))
 
     // Each process returns a Try[WorkflowExecutionDiff], which, upon success, contains potential changes to be made to the execution store.
@@ -413,13 +409,8 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor,
 
     // Update the metadata for the jobs we just sent to EJEAs (they'll start off queued up waiting for tokens):
     pushQueuedCallMetadata(diffs)
-    if (diffs.exists(_.containsNewEntry)) {
-      val newData = data.mergeExecutionDiffs(diffs)
-      startRunnableScopes(newData)
-    } else {
-      val result = data.mergeExecutionDiffs(diffs)
-      result
-    }
+    val newData = data.mergeExecutionDiffs(diffs)
+    if (truncated || diffs.exists(_.containsNewEntry)) newData else newData.resetCheckRunnable
   }
 
   private def isInBypassedScope(jobKey: JobKey, data: WorkflowExecutionActorData) = {
@@ -460,7 +451,7 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor,
       scatterMap
     )
 
-    declaration.requiredExpression.evaluate(lookup, data.expressionLanguageFunctions) match {
+    declaration.evaluate(lookup, data.expressionLanguageFunctions) match {
       case Success(result) => self ! DeclarationEvaluationSucceededResponse(declaration, result)
       case Failure(ex) => self ! DeclarationEvaluationFailedResponse(declaration, ex)
     }
@@ -481,8 +472,15 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor,
             val ejeaName = s"${workflowDescriptor.id}-EngineJobExecutionActor-${jobKey.tag}"
             val backendSingleton = backendSingletonCollection.backendSingletonActors(backendName)
             val ejeaProps = EngineJobExecutionActor.props(
-              self, jobKey, data, factory, initializationData.get(backendName), restarting, serviceRegistryActor,
-              jobStoreActor, callCacheReadActor, dockerHashActor, jobTokenDispenserActor, backendSingleton, backendName, workflowDescriptor.callCachingMode)
+              self, jobKey, data, factory, initializationData.get(backendName), restarting,
+              serviceRegistryActor = serviceRegistryActor,
+              ioActor = ioActor,
+              jobStoreActor = jobStoreActor,
+              callCacheReadActor = callCacheReadActor,
+              callCacheWriteActor = callCacheWriteActor,
+              workflowDockerLookupActor = workflowDockerLookupActor,
+              jobTokenDispenserActor = jobTokenDispenserActor,
+              backendSingleton, backendName, workflowDescriptor.callCachingMode)
             val ejeaRef = context.actorOf(ejeaProps, ejeaName)
             context watch ejeaRef
             pushNewCallMetadata(jobKey, Option(backendName))
@@ -498,9 +496,16 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor,
   
   private def processRunnableSubWorkflow(key: SubWorkflowKey, data: WorkflowExecutionActorData): Try[WorkflowExecutionDiff] = {
     val sweaRef = context.actorOf(
-      SubWorkflowExecutionActor.props(key, data, backendFactories, serviceRegistryActor, jobStoreActor, subWorkflowStoreActor,
-        callCacheReadActor, dockerHashActor, jobTokenDispenserActor, backendSingletonCollection, initializationData, restarting),
-      s"SubWorkflowExecutionActor-${key.tag}"
+      SubWorkflowExecutionActor.props(key, data, backendFactories,
+        ioActor = ioActor,
+        serviceRegistryActor = serviceRegistryActor,
+        jobStoreActor = jobStoreActor,
+        subWorkflowStoreActor = subWorkflowStoreActor,
+        callCacheReadActor = callCacheReadActor,
+        callCacheWriteActor = callCacheWriteActor,
+        workflowDockerLookupActor = workflowDockerLookupActor,
+        jobTokenDispenserActor = jobTokenDispenserActor,
+        backendSingletonCollection, initializationData, restarting), s"SubWorkflowExecutionActor-${key.tag}"
     )
 
     context watch sweaRef
@@ -545,7 +550,7 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor,
       Success(WorkflowExecutionDiff(scatterKey.populate(0, Map.empty) + (scatterKey -> ExecutionStatus.Bypassed)))
     } else {
       scatterKey.scope.collection.evaluate(lookup, data.expressionLanguageFunctions) map {
-        case a: WdlArray =>
+        case WdlArrayLike(a) =>
           WorkflowExecutionDiff(scatterKey.populate(a.value.size, workflowDescriptor.knownValues) + (scatterKey -> ExecutionStatus.Done))
         case v: WdlValue => throw new RuntimeException(
           s"Scatter collection must evaluate to an array but instead got ${v.wdlType.toWdlString}")
@@ -571,6 +576,8 @@ case class WorkflowExecutionActor(workflowDescriptor: EngineWorkflowDescriptor,
 
 object WorkflowExecutionActor {
 
+  val SweepInterval = 1 second
+
   /**
     * States
     */
@@ -633,6 +640,8 @@ object WorkflowExecutionActor {
   
   private case class DeclarationEvaluationSucceededResponse(declarationKey: DeclarationKey, value: WdlValue)
 
+  private case object CheckRunnable
+
   private[execution] sealed trait BypassedScopeResults
 
   private case class BypassedCallResults(callOutputs: Map[CallKey, CallOutputs]) extends BypassedScopeResults
@@ -673,14 +682,14 @@ object WorkflowExecutionActor {
 
     private def explode(scope: Scope, count: Int, workflowCoercedInputs: WorkflowCoercedInputs): Seq[JobKey] = {
       def makeCollectors(scope: Scope): Seq[CollectorKey] = scope match {
-        case call: Call => List(CollectorKey(call, scatter, count))
+        case call: WdlCall => List(CollectorKey(call, scatter, count))
         case decl: Declaration => List(CollectorKey(decl, scatter, count))
         case i: If => i.children.flatMap(makeCollectors)
       }
 
       (scope match {
-        case call: TaskCall => (0 until count) map { i => BackendJobDescriptorKey(call, Option(i), 1) }
-        case call: WorkflowCall => (0 until count) map { i => SubWorkflowKey(call, Option(i), 1) }
+        case call: WdlTaskCall => (0 until count) map { i => BackendJobDescriptorKey(call, Option(i), 1) }
+        case call: WdlWorkflowCall => (0 until count) map { i => SubWorkflowKey(call, Option(i), 1) }
         case declaration: Declaration => (0 until count) map { i => DeclarationKey(declaration, Option(i), workflowCoercedInputs) }
         case conditional: If => (0 until count) map { i => ConditionalKey(conditional, Option(i)) }
         case _: Scatter =>
@@ -692,13 +701,13 @@ object WorkflowExecutionActor {
   }
 
   // Represents a scatter collection for a call in the execution store
-  case class CollectorKey(scope: Scope with GraphNode, scatter: Scatter, scatterWidth: Int) extends JobKey {
+  case class CollectorKey(scope: Scope with WdlGraphNode, scatter: Scatter, scatterWidth: Int) extends JobKey {
     override val index = None
     override val attempt = 1
     override val tag = s"Collector-${scope.unqualifiedName}"
   }
 
-  case class SubWorkflowKey(scope: WorkflowCall, index: ExecutionIndex, attempt: Int) extends CallKey {
+  case class SubWorkflowKey(scope: WdlWorkflowCall, index: ExecutionIndex, attempt: Int) extends CallKey {
     override val tag = s"SubWorkflow-${scope.unqualifiedName}:${index.fromIndex}:$attempt"
   }
 
@@ -723,8 +732,8 @@ object WorkflowExecutionActor {
       */
     private def keyify(scope: Scope, workflowCoercedInputs: WorkflowCoercedInputs): JobKey = {
       scope match {
-        case call: TaskCall => BackendJobDescriptorKey(call, index, 1)
-        case call: WorkflowCall => SubWorkflowKey(call, index, 1)
+        case call: WdlTaskCall => BackendJobDescriptorKey(call, index, 1)
+        case call: WdlWorkflowCall => SubWorkflowKey(call, index, 1)
         case declaration: Declaration => DeclarationKey(declaration, index, workflowCoercedInputs)
         case i: If => ConditionalKey(i, index)
         case scatter: Scatter if index.isEmpty => ScatterKey(scatter)
@@ -757,27 +766,39 @@ object WorkflowExecutionActor {
   
   case class StaticDeclarationKey(scope: Declaration, index: ExecutionIndex, value: WdlValue) extends DeclarationKey
   
-  case class DynamicDeclarationKey(scope: Declaration, index: ExecutionIndex, requiredExpression: WdlExpression) extends DeclarationKey
+  case class DynamicDeclarationKey(scope: Declaration, index: ExecutionIndex, requiredExpression: WdlExpression) extends DeclarationKey {
+    def evaluate(lookup: ScopedLookupFunction, wdlFunctions: WdlFunctions[WdlValue]) = {
+      requiredExpression.evaluate(lookup, wdlFunctions) flatMap scope.wdlType.coerceRawValue
+    }
+  }
 
   case class WorkflowExecutionException[T <: Throwable](exceptions: NonEmptyList[T]) extends ThrowableAggregation {
     override val throwables = exceptions.toList
     override val exceptionContext = s"WorkflowExecutionActor"
   }
 
-  private lazy val DefaultMaxRetriesFallbackValue = 10
-
   def props(workflowDescriptor: EngineWorkflowDescriptor,
+            ioActor: ActorRef,
             serviceRegistryActor: ActorRef,
             jobStoreActor: ActorRef,
             subWorkflowStoreActor: ActorRef,
             callCacheReadActor: ActorRef,
-            dockerHashActor: ActorRef,
+            callCacheWriteActor: ActorRef,
+            workflowDockerLookupActor: ActorRef,
             jobTokenDispenserActor: ActorRef,
             backendSingletonCollection: BackendSingletonCollection,
             initializationData: AllBackendInitializationData,
             restarting: Boolean): Props = {
-    Props(WorkflowExecutionActor(workflowDescriptor, serviceRegistryActor, jobStoreActor, subWorkflowStoreActor,
-      callCacheReadActor, dockerHashActor, jobTokenDispenserActor, backendSingletonCollection, initializationData, restarting)).withDispatcher(EngineDispatcher)
+    Props(WorkflowExecutionActor(workflowDescriptor,
+      ioActor = ioActor,
+      serviceRegistryActor = serviceRegistryActor,
+      jobStoreActor = jobStoreActor,
+      subWorkflowStoreActor = subWorkflowStoreActor,
+      callCacheReadActor = callCacheReadActor,
+      callCacheWriteActor = callCacheWriteActor,
+      workflowDockerLookupActor = workflowDockerLookupActor,
+      jobTokenDispenserActor = jobTokenDispenserActor,
+      backendSingletonCollection, initializationData, restarting)).withDispatcher(EngineDispatcher)
   }
 
   implicit class EnhancedWorkflowOutputs(val outputs: Map[LocallyQualifiedName, WdlValue]) extends AnyVal {
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala
index 9f4c486ea..7c870517e 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorData.scala
@@ -7,7 +7,7 @@ import cromwell.core._
 import cromwell.engine.workflow.lifecycle.execution.OutputStore.{OutputCallKey, OutputEntry}
 import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor.{DeclarationKey, SubWorkflowKey}
 import cromwell.engine.{EngineWorkflowDescriptor, WdlFunctions}
-import wdl4s.values.WdlValue
+import wdl4s.wdl.values.WdlValue
 
 object WorkflowExecutionDiff {
   def empty = WorkflowExecutionDiff(Map.empty)
@@ -116,15 +116,20 @@ case class WorkflowExecutionActorData(workflowDescriptor: EngineWorkflowDescript
   def mergeExecutionDiff(diff: WorkflowExecutionDiff): WorkflowExecutionActorData = {
     this.copy(
       executionStore = executionStore.add(diff.executionStoreChanges),
-      engineCallExecutionActors = engineCallExecutionActors ++ diff.engineJobExecutionActorAdditions)
+      engineCallExecutionActors = engineCallExecutionActors ++ diff.engineJobExecutionActorAdditions
+    )
   }
 
   def mergeExecutionDiffs(diffs: Traversable[WorkflowExecutionDiff]): WorkflowExecutionActorData = {
     diffs.foldLeft(this)((newData, diff) => newData.mergeExecutionDiff(diff))
   }
   
+  def resetCheckRunnable = this.copy(executionStore = executionStore.copy(hasNewRunnables = false))
+  
+  def hasNewRunnables = executionStore.hasNewRunnables
+  
   def jobExecutionMap: JobExecutionMap = {
-    downstreamExecutionMap updated (workflowDescriptor.backendDescriptor, executionStore.unstartedJobs)
+    downstreamExecutionMap updated (workflowDescriptor.backendDescriptor, executionStore.startedJobs)
   }
   
   def hasRunningActors = backendJobExecutionActors.nonEmpty || subWorkflowExecutionActors.nonEmpty
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowMetadataHelper.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowMetadataHelper.scala
index d569f1fae..563c53006 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowMetadataHelper.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowMetadataHelper.scala
@@ -7,8 +7,6 @@ import cromwell.core.{WorkflowId, WorkflowMetadataKeys, WorkflowState}
 import cromwell.services.metadata.MetadataService._
 import cromwell.services.metadata.{MetadataEvent, MetadataKey, MetadataValue}
 
-import scala.util.Random
-
 trait WorkflowMetadataHelper {
 
   def serviceRegistryActor: ActorRef
@@ -24,7 +22,7 @@ trait WorkflowMetadataHelper {
   }
   
   def pushWorkflowFailures(workflowId: WorkflowId, failures: List[Throwable]) = {
-    val failureEvents = failures flatMap { r => throwableToMetadataEvents(MetadataKey(workflowId, None, s"${WorkflowMetadataKeys.Failures}[${Random.nextInt(Int.MaxValue)}]"), r) }
+    val failureEvents = failures flatMap { r => throwableToMetadataEvents(MetadataKey(workflowId, None, s"${WorkflowMetadataKeys.Failures}"), r) }
     serviceRegistryActor ! PutMetadataAction(failureEvents)
   }
   
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala
index f165fc5ea..0afff7212 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCache.scala
@@ -1,87 +1,146 @@
 package cromwell.engine.workflow.lifecycle.execution.callcaching
 
 import cats.data.NonEmptyList
-import cromwell.backend.BackendJobExecutionActor.JobSucceededResponse
-import cromwell.core.ExecutionIndex.IndexEnhancedIndex
-import cromwell.core.WorkflowId
+import cromwell.backend.BackendJobExecutionActor.{JobFailedNonRetryableResponse, JobSucceededResponse}
+import cromwell.core.ExecutionIndex.{ExecutionIndex, IndexEnhancedIndex}
 import cromwell.core.callcaching.HashResult
 import cromwell.core.path.Path
 import cromwell.core.simpleton.WdlValueSimpleton
+import cromwell.core.{CallOutputs, FullyQualifiedName, JobOutput, LocallyQualifiedName, WorkflowId}
 import cromwell.database.sql.SqlConverters._
 import cromwell.database.sql._
 import cromwell.database.sql.joins.CallCachingJoin
-import cromwell.database.sql.tables.{CallCachingDetritusEntry, CallCachingEntry, CallCachingHashEntry, CallCachingSimpletonEntry}
+import cromwell.database.sql.tables._
+import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCache.CallCacheHashBundle
+import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheReadActor.AggregatedCallHashes
 import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.CallCacheHashes
 
 import scala.concurrent.{ExecutionContext, Future}
 
 final case class CallCachingEntryId(id: Int)
-
 /**
   * Given a database-layer CallCacheStore, this accessor can access the database with engine-friendly data types.
   */
 class CallCache(database: CallCachingSqlDatabase) {
-  def addToCache(workflowId: WorkflowId, callCacheHashes: CallCacheHashes, response: JobSucceededResponse)(implicit ec: ExecutionContext): Future[Unit] = {
-    val metaInfo = CallCachingEntry(
-      workflowExecutionUuid = workflowId.toString,
-      callFullyQualifiedName = response.jobKey.call.fullyQualifiedName,
-      jobIndex = response.jobKey.index.fromIndex,
-      jobAttempt = Option(response.jobKey.attempt),
-      returnCode = response.returnCode,
-      allowResultReuse = true)
-    val hashes = callCacheHashes.hashes
-    import cromwell.core.simpleton.WdlValueSimpleton._
-    val result = response.jobOutputs.mapValues(_.wdlValue).simplify
-    val jobDetritus = response.jobDetritusFiles.getOrElse(Map.empty)
-
-    addToCache(metaInfo, hashes, result, jobDetritus)
+  def addToCache(bundles: Seq[CallCacheHashBundle], batchSize: Int)(implicit ec: ExecutionContext): Future[Unit] = {
+    val joins = bundles map { b =>
+      val metaInfo = CallCachingEntry(
+        workflowExecutionUuid = b.workflowId.toString,
+        callFullyQualifiedName = b.fullyQualifiedName,
+        jobIndex = b.jobIndex.fromIndex,
+        jobAttempt = b.jobAttempt,
+        returnCode = b.returnCode,
+        allowResultReuse = b.allowResultReuse)
+      import cromwell.core.simpleton.WdlValueSimpleton._
+      val result = b.callOutputs.mapValues(_.wdlValue).simplify
+      val jobDetritus = b.jobDetritusFiles.getOrElse(Map.empty)
+      buildCallCachingJoin(metaInfo, b.callCacheHashes, result, jobDetritus)
+    }
+
+    database.addCallCaching(joins, batchSize)
   }
 
-  private def addToCache(callCachingEntry: CallCachingEntry, hashes: Set[HashResult],
-                         result: Iterable[WdlValueSimpleton], jobDetritus: Map[String, Path])
-                        (implicit ec: ExecutionContext): Future[Unit] = {
+  private def buildCallCachingJoin(callCachingEntry: CallCachingEntry, callCacheHashes: CallCacheHashes,
+                                   result: Iterable[WdlValueSimpleton], jobDetritus: Map[String, Path]): CallCachingJoin = {
 
     val hashesToInsert: Iterable[CallCachingHashEntry] = {
-      hashes map { hash => CallCachingHashEntry(hash.hashKey.key, hash.hashValue.value) }
+      callCacheHashes.hashes map { hash => CallCachingHashEntry(hash.hashKey.key, hash.hashValue.value) }
+    }
+
+    val aggregatedHashesToInsert: Option[CallCachingAggregationEntry] = {
+      Option(CallCachingAggregationEntry(
+        baseAggregation = callCacheHashes.aggregatedInitialHash,
+        inputFilesAggregation = callCacheHashes.fileHashes.map(_.aggregatedHash)
+      ))
     }
 
     val resultToInsert: Iterable[CallCachingSimpletonEntry] = {
       result map {
         case WdlValueSimpleton(simpletonKey, wdlPrimitive) =>
-          CallCachingSimpletonEntry(simpletonKey, wdlPrimitive.valueString.toClob, wdlPrimitive.wdlType.toWdlString)
+          CallCachingSimpletonEntry(simpletonKey, wdlPrimitive.valueString.toClobOption, wdlPrimitive.wdlType.toWdlString)
       }
     }
 
     val jobDetritusToInsert: Iterable[CallCachingDetritusEntry] = {
       jobDetritus map {
-        case (fileName, filePath) => CallCachingDetritusEntry(fileName, filePath.pathAsString.toClob)
+        case (fileName, filePath) => CallCachingDetritusEntry(fileName, filePath.pathAsString.toClobOption)
       }
     }
 
-    val callCachingJoin =
-      CallCachingJoin(callCachingEntry, hashesToInsert.toSeq, resultToInsert.toSeq, jobDetritusToInsert.toSeq)
-
-    database.addCallCaching(callCachingJoin)
+    CallCachingJoin(callCachingEntry, hashesToInsert.toSeq, aggregatedHashesToInsert, resultToInsert.toSeq, jobDetritusToInsert.toSeq)
   }
 
-  def callCachingEntryIdsMatchingHashes(callCacheHashes: CallCacheHashes)(implicit ec: ExecutionContext): Future[Set[CallCachingEntryId]] = {
-    callCachingEntryIdsMatchingHashes(NonEmptyList.fromListUnsafe(callCacheHashes.hashes.toList))
+  def hasBaseAggregatedHashMatch(baseAggregatedHash: String)(implicit ec: ExecutionContext): Future[Boolean] = {
+    database.hasMatchingCallCachingEntriesForBaseAggregation(baseAggregatedHash)
   }
 
-  private def callCachingEntryIdsMatchingHashes(hashKeyValuePairs: NonEmptyList[HashResult])
-                                       (implicit ec: ExecutionContext): Future[Set[CallCachingEntryId]] = {
-    val result = database.queryCallCachingEntryIds(hashKeyValuePairs map {
+  def hasKeyValuePairHashMatch(hashes: NonEmptyList[HashResult])(implicit ec: ExecutionContext): Future[Boolean] = {
+    val hashKeyValuePairs = hashes map {
       case HashResult(hashKey, hashValue) => (hashKey.key, hashValue.value)
-    })
+    }
+    database.hasMatchingCallCachingEntriesForHashKeyValues(hashKeyValuePairs)
+  }
 
-    result.map(_.toSet.map(CallCachingEntryId))
+  def callCachingHitForAggregatedHashes(aggregatedCallHashes: AggregatedCallHashes, hitNumber: Int)
+                                       (implicit ec: ExecutionContext): Future[Option[CallCachingEntryId]] = {
+    database.findCacheHitForAggregation(
+      baseAggregationHash = aggregatedCallHashes.baseAggregatedHash,
+      inputFilesAggregationHash = aggregatedCallHashes.inputFilesAggregatedHash,
+      hitNumber).map(_ map CallCachingEntryId.apply)
   }
 
   def fetchCachedResult(callCachingEntryId: CallCachingEntryId)(implicit ec: ExecutionContext): Future[Option[CallCachingJoin]] = {
-    database.queryCallCaching(callCachingEntryId.id)
+    database.queryResultsForCacheId(callCachingEntryId.id)
+  }
+
+  def cacheEntryExistsForCall(workflowUuid: String, callFqn: String, index: Int)(implicit ec: ExecutionContext): Future[Boolean] = {
+    database.cacheEntryExistsForCall(workflowUuid, callFqn, index)
   }
 
   def invalidate(callCachingEntryId: CallCachingEntryId)(implicit ec: ExecutionContext) = {
     database.invalidateCall(callCachingEntryId.id)
   }
 }
+
+object CallCache {
+  object CallCacheHashBundle {
+    def apply(workflowId: WorkflowId, callCacheHashes: CallCacheHashes, jobSucceededResponse: JobSucceededResponse) = {
+      new CallCacheHashBundle(
+        workflowId = workflowId,
+        callCacheHashes = callCacheHashes,
+        fullyQualifiedName = jobSucceededResponse.jobKey.call.fullyQualifiedName,
+        jobIndex = jobSucceededResponse.jobKey.index,
+        jobAttempt = Option(jobSucceededResponse.jobKey.attempt),
+        returnCode = jobSucceededResponse.returnCode,
+        allowResultReuse = true,
+        callOutputs = jobSucceededResponse.jobOutputs,
+        jobDetritusFiles = jobSucceededResponse.jobDetritusFiles
+      )
+    }
+
+    def apply(workflowId: WorkflowId, callCacheHashes: CallCacheHashes, jobFailedNonRetryableResponse: JobFailedNonRetryableResponse) = {
+      new CallCacheHashBundle(
+        workflowId = workflowId,
+        callCacheHashes = callCacheHashes,
+        fullyQualifiedName = jobFailedNonRetryableResponse.jobKey.scope.fullyQualifiedName,
+        jobIndex = jobFailedNonRetryableResponse.jobKey.index,
+        jobAttempt = Option(jobFailedNonRetryableResponse.jobKey.attempt),
+        returnCode = None,
+        allowResultReuse = false,
+        callOutputs = Map.empty[LocallyQualifiedName, JobOutput],
+        jobDetritusFiles = None
+      )
+    }
+  }
+  case class CallCacheHashBundle private (
+                                           workflowId: WorkflowId,
+                                           callCacheHashes: CallCacheHashes,
+                                           fullyQualifiedName: FullyQualifiedName,
+                                           jobIndex: ExecutionIndex,
+                                           jobAttempt: Option[Int],
+                                           returnCode: Option[Int],
+                                           allowResultReuse: Boolean,
+                                           callOutputs: CallOutputs,
+                                           jobDetritusFiles: Option[Map[String, Path]]
+                                         )
+}
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheDiffActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheDiffActor.scala
new file mode 100644
index 000000000..b4d661255
--- /dev/null
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheDiffActor.scala
@@ -0,0 +1,306 @@
+package cromwell.engine.workflow.lifecycle.execution.callcaching
+
+import akka.actor.{ActorRef, LoggingFSM, Props}
+import cats.data.NonEmptyList
+import cats.instances.list._
+import cats.syntax.foldable._
+import cromwell.core.WorkflowId
+import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheDiffActor.{CallCacheDiffActorData, _}
+import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheDiffQueryParameter.CallCacheDiffQueryCall
+import cromwell.services.metadata.CallMetadataKeys.CallCachingKeys
+import cromwell.services.metadata.MetadataService.{GetMetadataQueryAction, MetadataLookupResponse, MetadataServiceKeyLookupFailed}
+import cromwell.services.metadata._
+import cromwell.webservice.metadata.MetadataComponent._
+import cromwell.webservice.metadata._
+import cromwell.core.Dispatcher.EngineDispatcher
+import spray.json.JsObject
+
+import scala.language.postfixOps
+import scala.util.{Failure, Success, Try}
+
+object CallCacheDiffActor {
+  private val PlaceholderMissingHashValue = MetadataPrimitive(MetadataValue("Error: there is a hash entry for this key but the value is null !"))
+
+  final case class CachedCallNotFoundException(message: String) extends Exception {
+    override def getMessage = message
+  }
+
+  // Exceptions when calls exist but have no hashes in their metadata, indicating they were run pre-28
+  private val HashesForCallAAndBNotFoundException = new Exception("callA and callB have not finished yet, or were run on a previous version of Cromwell on which this endpoint was not supported.")
+  private val HashesForCallANotFoundException = new Exception("callA has not finished yet, or was run on a previous version of Cromwell on which this endpoint was not supported.")
+  private val HashesForCallBNotFoundException = new Exception("callB has not finished yet, or was run on a previous version of Cromwell on which this endpoint was not supported.")
+
+  sealed trait CallCacheDiffActorState
+  case object Idle extends CallCacheDiffActorState
+  case object WaitingForMetadata extends CallCacheDiffActorState
+
+  sealed trait CallCacheDiffActorData
+  case object CallCacheDiffNoData extends CallCacheDiffActorData
+  case class CallCacheDiffWithRequest(queryA: MetadataQuery,
+                                      queryB: MetadataQuery,
+                                      responseA: Option[MetadataLookupResponse],
+                                      responseB: Option[MetadataLookupResponse],
+                                      replyTo: ActorRef
+                                     ) extends CallCacheDiffActorData
+
+  sealed abstract class CallCacheDiffActorResponse
+  case class BuiltCallCacheDiffResponse(response: JsObject) extends CallCacheDiffActorResponse
+  case class FailedCallCacheDiffResponse(reason: Throwable) extends CallCacheDiffActorResponse
+
+
+  def props(serviceRegistryActor: ActorRef) = Props(new CallCacheDiffActor(serviceRegistryActor)).withDispatcher(EngineDispatcher)
+}
+
+class CallCacheDiffActor(serviceRegistryActor: ActorRef) extends LoggingFSM[CallCacheDiffActorState, CallCacheDiffActorData] {
+  startWith(Idle, CallCacheDiffNoData)
+
+  when(Idle) {
+    case Event(CallCacheDiffQueryParameter(callA, callB), CallCacheDiffNoData) =>
+      val queryA = makeMetadataQuery(callA)
+      val queryB = makeMetadataQuery(callB)
+      serviceRegistryActor ! GetMetadataQueryAction(queryA)
+      serviceRegistryActor ! GetMetadataQueryAction(queryB)
+      goto(WaitingForMetadata) using CallCacheDiffWithRequest(queryA, queryB, None, None, sender())
+  }
+
+  when(WaitingForMetadata) {
+    // First Response
+    // Response A
+    case Event(response: MetadataLookupResponse, data @ CallCacheDiffWithRequest(queryA, _, None, None, _)) if queryA == response.query =>
+      stay() using data.copy(responseA = Option(response))
+    // Response B
+    case Event(response: MetadataLookupResponse, data @ CallCacheDiffWithRequest(_, queryB, None, None, _)) if queryB == response.query =>
+      stay() using data.copy(responseB = Option(response))
+    // Second Response
+    // Response A
+    case Event(response: MetadataLookupResponse, CallCacheDiffWithRequest(queryA, queryB, None, Some(responseB), replyTo)) if queryA == response.query =>
+      buildDiffAndRespond(queryA, queryB, response, responseB, replyTo)
+    // Response B
+    case Event(response: MetadataLookupResponse, CallCacheDiffWithRequest(queryA, queryB, Some(responseA), None, replyTo)) if queryB == response.query =>
+      buildDiffAndRespond(queryA, queryB, responseA, response, replyTo)
+    case Event(MetadataServiceKeyLookupFailed(_, failure), data: CallCacheDiffWithRequest) =>
+      data.replyTo ! FailedCallCacheDiffResponse(failure)
+      context stop self
+      stay()
+  }
+
+  /**
+    * Builds a response and sends it back as Json.
+    * The response is structured in the following way
+    * {
+    *   "callA": {
+    *     -- information about call A --
+    *   },
+    *   "callB": {
+    *     -- information about call B --
+    *   },
+    *   "hashDifferential": [
+    *     {
+    *       "hash key": {
+    *         "callA": -- hash value for call A, or null --,
+    *         "callB": -- hash value for call B, or null --
+    *       }
+    *     },
+    *     ...
+    *   ]
+    * }
+    */
+  private def buildDiffAndRespond(queryA: MetadataQuery,
+                                  queryB: MetadataQuery,
+                                  responseA: MetadataLookupResponse,
+                                  responseB: MetadataLookupResponse,
+                                  replyTo: ActorRef) = {
+
+    lazy val buildResponse = {
+      diffHashes(responseA.eventList, responseB.eventList) match {
+        case Success(diff) =>
+          val diffObject = MetadataObject(Map(
+            "callA" -> makeCallInfo(queryA, responseA.eventList),
+            "callB" -> makeCallInfo(queryB, responseB.eventList),
+            "hashDifferential" -> diff
+          ))
+
+          BuiltCallCacheDiffResponse(metadataComponentJsonWriter.write(diffObject).asJsObject)
+        case Failure(f) => FailedCallCacheDiffResponse(f)
+      }
+    }
+
+    val response = checkCallsExistence(queryA, queryB, responseA, responseB) match {
+      case Some(msg) => FailedCallCacheDiffResponse(CachedCallNotFoundException(msg))
+      case None => buildResponse
+    }
+
+    replyTo ! response
+
+    context stop self
+    stay()
+  }
+
+  /**
+    * Returns an error message if one or both of the calls are not found, or None if it does
+    */
+  private def checkCallsExistence(queryA: MetadataQuery,
+                                  queryB: MetadataQuery,
+                                  responseA: MetadataLookupResponse,
+                                  responseB: MetadataLookupResponse): Option[String] = {
+    import cromwell.core.ExecutionIndex._
+
+    def makeTag(query: MetadataQuery) = {
+      s"${query.workflowId}:${query.jobKey.get.callFqn}:${query.jobKey.get.index.fromIndex}"
+    }
+
+    def makeNotFoundMessage(queries: NonEmptyList[MetadataQuery]) = {
+      val plural = if (queries.tail.nonEmpty) "s" else ""
+      s"Cannot find call$plural ${queries.map(makeTag).toList.mkString(", ")}"
+    }
+
+    (responseA.eventList, responseB.eventList) match {
+      case (a, b) if a.isEmpty && b.isEmpty => Option(makeNotFoundMessage(NonEmptyList.of(queryA, queryB)))
+      case (a, _) if a.isEmpty => Option(makeNotFoundMessage(NonEmptyList.of(queryA)))
+      case (_, b) if b.isEmpty => Option(makeNotFoundMessage(NonEmptyList.of(queryB)))
+      case _ => None
+    }
+  }
+
+  /**
+    * Generates the "info" section of callA or callB
+    */
+  private def makeCallInfo(query: MetadataQuery, eventList: Seq[MetadataEvent]): MetadataComponent = {
+    val callKey = MetadataObject(Map(
+      "workflowId" -> MetadataPrimitive(MetadataValue(query.workflowId.toString)),
+      "callFqn" -> MetadataPrimitive(MetadataValue(query.jobKey.get.callFqn)),
+      "jobIndex" -> MetadataPrimitive(MetadataValue(query.jobKey.get.index.getOrElse(-1)))
+    ))
+
+    val allowResultReuse = attributeToComponent(eventList, { _ == CallCachingKeys.AllowReuseMetadataKey }, { _ => "allowResultReuse" })
+    val executionStatus = attributeToComponent(eventList, { _ == CallMetadataKeys.ExecutionStatus })
+
+    List(callKey, allowResultReuse, executionStatus) combineAll
+  }
+
+  /**
+    * Collects events from the list for which the keys verify the keyFilter predicate
+    * and apply keyModifier to the event's key
+    */
+  private def collectEvents(events: Seq[MetadataEvent],
+                            keyFilter: (String  => Boolean),
+                            keyModifier: (String => String)) = events collect {
+    case event @ MetadataEvent(metadataKey @ MetadataKey(_, _, key), _, _) if keyFilter(key) =>
+      event.copy(key = metadataKey.copy(key = keyModifier(key)))
+  }
+
+  /**
+    * Given a list of events, a keyFilter and a keyModifier, returns the associated MetadataComponent.
+    * Ensures that events are properly aggregated together (CRDTs and latest timestamp rule)
+    */
+  private def attributeToComponent(events: Seq[MetadataEvent], keyFilter: (String  => Boolean), keyModifier: (String => String) = identity[String]) = {
+    MetadataComponent(collectEvents(events, keyFilter, keyModifier))
+  }
+
+  /**
+    * Makes a diff object out of a key and a pair of values.
+    * Values are Option[Option[MetadataValue]] for the following reason:
+    *
+    * The outer option represents whether or not this key had a corresponding hash metadata entry for the given call
+    * If the above is true, the inner value is the metadata value for this entry, which is nullable, hence an Option.
+    * The first outer option will determine whether the resulting json value will be null (no hash entry for this key),
+    * or the actual value.
+    * If the metadata value (inner option) happens to be None, it's an error, as we don't expect to publish null hash values.
+    * In that case we replace it with the placeholderMissingHashValue.
+    */
+  private def makeHashDiffObject(key: String, valueA: Option[Option[MetadataValue]], valueB: Option[Option[MetadataValue]]) = {
+    def makeFinalValue(value: Option[Option[MetadataValue]]) = value match {
+      case Some(Some(metadataValue)) => MetadataPrimitive(metadataValue)
+      case Some(None) => PlaceholderMissingHashValue
+      case None => MetadataNull
+    }
+
+    MetadataObject(
+      "hashKey" -> MetadataPrimitive(MetadataValue(key.trim, MetadataString)),
+      "callA" -> makeFinalValue(valueA),
+      "callB" -> makeFinalValue(valueB)
+    )
+  }
+
+  /**
+    * Creates the hash differential between 2 list of events
+    */
+  private def diffHashes(eventsA: Seq[MetadataEvent], eventsB: Seq[MetadataEvent]): Try[MetadataComponent] = {
+    val hashesKey = CallCachingKeys.HashesKey + MetadataKey.KeySeparator
+    // Collect hashes events and map their key to only keep the meaningful part of the key
+    // Then map the result to get a Map of hashKey -> Option[MetadataValue]. This will allow for fast lookup when
+    // comparing the 2 hash sets.
+    // Note that it's an Option[MetadataValue] because metadata values can be null, although for this particular
+    // case we don't expect it to be (we should never publish a hash metadata event with a null value)
+    // If that happens we will place a placeholder value in place of the hash to signify of the unexpected absence of it
+    def collectHashes(events: Seq[MetadataEvent]) = {
+      collectEvents(events, { _.startsWith(hashesKey) }, { _.stripPrefix(hashesKey) })  map {
+        case MetadataEvent(MetadataKey(_, _, keyA), valueA, _) => keyA -> valueA
+      } toMap
+    }
+
+    val hashesA: Map[String, Option[MetadataValue]] = collectHashes(eventsA)
+    val hashesB: Map[String, Option[MetadataValue]] = collectHashes(eventsB)
+
+    (hashesA.isEmpty, hashesB.isEmpty) match {
+      case (true, true) => Failure(HashesForCallAAndBNotFoundException)
+      case (true, false) => Failure(HashesForCallANotFoundException)
+      case (false, true) => Failure(HashesForCallBNotFoundException)
+      case (false, false) => Success(diffHashEvents(hashesA, hashesB))
+    }
+
+  }
+
+  private def diffHashEvents(hashesA: Map[String, Option[MetadataValue]], hashesB: Map[String, Option[MetadataValue]]) = {
+    val hashesUniqueToB: Map[String, Option[MetadataValue]] = hashesB.filterNot({ case (k, _) => hashesA.keySet.contains(k) })
+
+    val hashDiff: List[MetadataComponent] = {
+      // Start with all hashes in A
+      hashesA
+        // Try to find the corresponding pair in B.
+        // We end up with a 
+        // List[(Option[String, Option[MetadataValue], Option[String, Option[MetadataValue])]
+        //                ^               ^                     ^               ^
+        //             hashKey        hashValue              hashKey         hashValue
+        //               for             for                   for              for
+        //                A               A                     B                B
+        //      |____________________________________| |___________________________________|
+        //                  hashPair for A                        hashPair for B
+        // 
+        // HashPairs are Some or None depending on whether or not they have a metadata entry for the corresponding hashKey
+        // At this stage we only have Some(hashPair) for A, and either Some(hashPair) or None for B depending on if we found it in hashesB
+        .map({
+        hashPairA => Option(hashPairA) -> hashesB.find(_._1 == hashPairA._1)
+      })
+        // Add the missing hashes that are in B but not in A. The left hashPair is therefore None
+        .++(hashesUniqueToB.map(None -> Option(_)))
+        .collect({
+          // Both have a value but they're different. We can assume the keys are the same (if we did our job right until here)
+          case (Some((keyA, valueA)), Some((_, valueB))) if valueA != valueB =>
+            makeHashDiffObject(keyA, Option(valueA), Option(valueB))
+          // Key is in A but not in B
+          case (Some((keyA, valueA)), None) =>
+            makeHashDiffObject(keyA, Option(valueA), None)
+          // Key is in B but not in A
+          case (None, Some((keyB, valueB))) =>
+            makeHashDiffObject(keyB, None, Option(valueB))
+        })
+        .toList
+    }
+
+    MetadataList(hashDiff)
+  }
+
+  /**
+    * Create a Metadata query from a CallCacheDiffQueryCall
+    */
+  private def makeMetadataQuery(call: CallCacheDiffQueryCall) = MetadataQuery(
+    WorkflowId.fromString(call.workflowId),
+    // jobAttempt None will return keys for all attempts
+    Option(MetadataQueryJobKey(call.callFqn, call.jobIndex, None)),
+    None,
+    Option(NonEmptyList.of("callCaching", "executionStatus")),
+    None,
+    expandSubWorkflows = false
+  )
+}
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheDiffQueryParameter.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheDiffQueryParameter.scala
new file mode 100644
index 000000000..3c2058720
--- /dev/null
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheDiffQueryParameter.scala
@@ -0,0 +1,52 @@
+package cromwell.engine.workflow.lifecycle.execution.callcaching
+
+import cats.data.{NonEmptyList, Validated}
+import cats.implicits._
+import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheDiffQueryParameter.CallCacheDiffQueryCall
+
+import scala.util.{Failure, Success, Try}
+
+object CallCacheDiffQueryParameter {
+  case class CallCacheDiffQueryCall(workflowId: String, callFqn: String, jobIndex: Option[Int])
+
+  private def missingWorkflowError(attribute: String) = s"missing $attribute query parameter".invalidNel
+
+  def fromParameters(parameters: Seq[(String, String)]): Validated[NonEmptyList[String], CallCacheDiffQueryParameter] = {
+    def extractIndex(parameter: String): Validated[NonEmptyList[String], Option[Int]] = {
+      parameters.find(_._1 == parameter) match {
+        case Some((_, value)) => Try(value.toInt) match {
+          case Success(index) => Option(index).validNel
+          case Failure(f) => f.getMessage.invalidNel
+        }
+        case None => None.validNel
+      }
+    }
+
+    def extractAttribute(parameter: String): Validated[NonEmptyList[String], String] = {
+      parameters.find(_._1 == parameter) match {
+        case Some((_, value)) => value.validNel
+        case None => missingWorkflowError(parameter)
+      }
+    }
+    
+    val workflowAValidation = extractAttribute("workflowA")
+    val workflowBValidation = extractAttribute("workflowB")
+
+    val callAValidation = extractAttribute("callA")
+    val callBValidation = extractAttribute("callB")
+
+    val indexAValidation = extractIndex("indexA")
+    val indexBValidation = extractIndex("indexB")
+
+    (workflowAValidation |@| callAValidation |@| indexAValidation |@|
+      workflowBValidation |@| callBValidation |@| indexBValidation) map {
+      case ((workflowA, callA, indexA, workflowB, callB, indexB)) =>
+        CallCacheDiffQueryParameter(
+          CallCacheDiffQueryCall(workflowA, callA, indexA),
+          CallCacheDiffQueryCall(workflowB, callB, indexB)
+        )
+    }
+  }
+}
+
+case class CallCacheDiffQueryParameter(callA: CallCacheDiffQueryCall, callB: CallCacheDiffQueryCall)
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheHashingJobActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheHashingJobActor.scala
new file mode 100644
index 000000000..05591c555
--- /dev/null
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheHashingJobActor.scala
@@ -0,0 +1,294 @@
+package cromwell.engine.workflow.lifecycle.execution.callcaching
+
+import java.security.MessageDigest
+import javax.xml.bind.DatatypeConverter
+
+import akka.actor.{ActorRef, LoggingFSM, Props, Terminated}
+import cats.data.NonEmptyList
+import cromwell.backend.standard.callcaching.StandardFileHashingActor.{FileHashResponse, SingleFileHashRequest}
+import cromwell.backend.validation.RuntimeAttributesKeys
+import cromwell.backend.{BackendInitializationData, BackendJobDescriptor, RuntimeAttributeDefinition}
+import cromwell.core.callcaching._
+import cromwell.core.simpleton.WdlValueSimpleton
+import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheHashingJobActor.CallCacheHashingJobActorData._
+import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheHashingJobActor._
+import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.CacheMiss
+import cromwell.core.Dispatcher.EngineDispatcher
+
+import wdl4s.wdl.values.WdlFile
+
+/**
+  * Actor responsible for calculating individual as well as aggregated hashes for a job.
+  * First calculate the initial hashes (individual and aggregated), and send them to its parent
+  * as well as the callCacheReadingJobActor if one was provided.
+  * From there, either wait for a NextBatchOfFileHashesRequest message from the callCacheReadingJobActor to trigger the next
+  * batch of file hash requests, or send it itself if there is no callCacheReadingJobActor.
+  * When we get all the file hashes for one batch, send those hashes to the CCRead actor and wait for the next NextBatchOfFileHashesRequest.
+  * If it was the last batch and all the files have been hashed, send all the hashes along with the aggregated file hash.
+  * If at any point the callCacheReadingJobActor dies (either because it decided this was a CacheMiss or it found a successful
+  * CacheHit), either keep hashing the files if writeToCache is true, or die if it's not.
+  * Both this actor and the CCRJA are children of the EJHA. 
+  * The CCRJA reports its findings (cache hit / miss) directly to the EJHA that forwards them to the EJEA.
+  * This actor does not need nor care about cache hit/miss.
+  */
+class CallCacheHashingJobActor(jobDescriptor: BackendJobDescriptor,
+                               callCacheReadingJobActor: Option[ActorRef],
+                               initializationData: Option[BackendInitializationData],
+                               runtimeAttributeDefinitions: Set[RuntimeAttributeDefinition],
+                               backendName: String,
+                               fileHashingActorProps: Props,
+                               writeToCache: Boolean,
+                               callCachingEligible: CallCachingEligible
+                              ) extends LoggingFSM[CallCacheHashingJobActorState, CallCacheHashingJobActorData] {
+
+  val fileHashingActor = makeFileHashingActor()
+
+  // Watch the read actor, as it will die when it's done (cache miss or successful cache hit)
+  // When that happens we want to either stop if writeToCache is false, or keep going
+  callCacheReadingJobActor map context.watch
+
+  initializeCCHJA()
+
+  override def preStart(): Unit = {
+    if (callCacheReadingJobActor.isEmpty && !writeToCache) {
+      log.error("Programmer error ! There is no reason to have a hashing actor if both read and write to cache are off")
+      context.parent ! CacheMiss
+      context stop self
+    }
+    super.preStart()
+  }
+
+  when(WaitingForHashFileRequest) {
+    case Event(NextBatchOfFileHashesRequest, data) =>
+      data.fileHashRequestsRemaining.headOption match {
+        case Some(batch) if batch.nonEmpty =>
+          batch foreach { fileHashingActor ! _ }
+          goto(HashingFiles)
+        case _ =>
+          sendToCCReadActor(NoFileHashesResult, data)
+          stopAndStay(Option(NoFileHashesResult))
+      }
+    case Event(Terminated(_), data) if writeToCache =>
+      self ! NextBatchOfFileHashesRequest
+      stay() using data.copy(callCacheReadingJobActor = None)
+  }
+
+  when(HashingFiles) {
+    case Event(FileHashResponse(result), data) =>
+      addFileHash(result, data) match {
+        case (newData, Some(partialHashes: PartialFileHashingResult)) =>
+          sendToCCReadActor(partialHashes, data)
+          // If there is no CCReader, send a message to itself to trigger the next batch
+          if (newData.callCacheReadingJobActor.isEmpty) self ! NextBatchOfFileHashesRequest
+          goto(WaitingForHashFileRequest) using newData
+        case (newData, Some(finalResult: FinalFileHashingResult)) =>
+          sendToCCReadActor(finalResult, newData)
+          stopAndStay(Option(finalResult))
+        case (newData, None) =>
+          stay() using newData
+      }
+    case Event(Terminated(_), data) if writeToCache =>
+      stay() using data.copy(callCacheReadingJobActor = None)
+  }
+
+  whenUnhandled {
+    case Event(Terminated(_), _) =>
+      stopAndStay(None)
+    case Event(error: HashingFailedMessage, data) =>
+      log.error(error.reason, s"Failed to hash ${error.file}")
+      sendToCCReadActor(error, data)
+      context.parent ! error
+      stopAndStay(None)
+  }
+  
+  // Is its own function so it can be overriden in the test
+  private [callcaching] def addFileHash(hashResult: HashResult, data: CallCacheHashingJobActorData) = {
+    data.withFileHash(hashResult)
+  }
+
+  private def stopAndStay(fileHashResult: Option[FinalFileHashingResult]) = {
+    fileHashResult foreach { context.parent ! _ }
+    context stop fileHashingActor
+    context stop self
+    stay()
+  }
+
+  private def sendToCCReadActor(message: Any, data: CallCacheHashingJobActorData) = {
+    data.callCacheReadingJobActor foreach { _ ! message }
+  }
+
+  private def initializeCCHJA() = {
+    import cromwell.core.simpleton.WdlValueSimpleton._
+
+    val unqualifiedInputs = jobDescriptor.inputDeclarations map { case (declaration, value) => declaration.unqualifiedName -> value }
+
+    val inputSimpletons = unqualifiedInputs.simplify
+    val (fileInputSimpletons, nonFileInputSimpletons) = inputSimpletons partition {
+      case WdlValueSimpleton(_, _: WdlFile) => true
+      case _ => false
+    }
+
+    val initialHashes = calculateInitialHashes(nonFileInputSimpletons, fileInputSimpletons)
+
+    val fileHashRequests = fileInputSimpletons collect {
+      case WdlValueSimpleton(name, x: WdlFile) => SingleFileHashRequest(jobDescriptor.key, HashKey(true, "input", s"File $name"), x, initializationData)
+    }
+
+    val hashingJobActorData = CallCacheHashingJobActorData(fileHashRequests.toList, callCacheReadingJobActor)
+    startWith(WaitingForHashFileRequest, hashingJobActorData)
+
+    val initialHashingResult = InitialHashingResult(initialHashes, calculateHashAggregation(initialHashes, MessageDigest.getInstance("MD5")))
+
+    sendToCCReadActor(initialHashingResult, hashingJobActorData)
+    context.parent ! initialHashingResult
+    
+    // If there is no CCRead actor, we need to send ourselves the next NextBatchOfFileHashesRequest
+    if (hashingJobActorData.callCacheReadingJobActor.isEmpty) self ! NextBatchOfFileHashesRequest
+  }
+
+  private def calculateInitialHashes(nonFileInputs: Iterable[WdlValueSimpleton], fileInputs: Iterable[WdlValueSimpleton]): Set[HashResult] = {
+
+    val commandTemplateHash = HashResult(HashKey("command template"), jobDescriptor.call.task.commandTemplateString.md5HashValue)
+    val backendNameHash = HashResult(HashKey("backend name"), backendName.md5HashValue)
+    val inputCountHash = HashResult(HashKey("input count"), (nonFileInputs.size + fileInputs.size).toString.md5HashValue)
+    val outputCountHash = HashResult(HashKey("output count"), jobDescriptor.call.task.outputs.size.toString.md5HashValue)
+
+    val runtimeAttributeHashes = runtimeAttributeDefinitions map { definition => jobDescriptor.runtimeAttributes.get(definition.name) match {
+      case Some(_) if definition.name == RuntimeAttributesKeys.DockerKey && callCachingEligible.dockerHash.isDefined =>
+        HashResult(HashKey(definition.usedInCallCaching, "runtime attribute", definition.name), callCachingEligible.dockerHash.get.md5HashValue)
+      case Some(wdlValue) => HashResult(HashKey(definition.usedInCallCaching, "runtime attribute", definition.name), wdlValue.valueString.md5HashValue)
+      case None => HashResult(HashKey(definition.usedInCallCaching, "runtime attribute", definition.name), UnspecifiedRuntimeAttributeHashValue)
+    }}
+
+    val inputHashResults = nonFileInputs map {
+      case WdlValueSimpleton(name, value) => HashResult(HashKey("input", s"${value.wdlType.toWdlString} $name"),  value.toWdlString.md5HashValue)
+    }
+
+    val outputExpressionHashResults = jobDescriptor.call.task.outputs map { output =>
+      HashResult(HashKey("output expression", s"${output.wdlType.toWdlString} ${output.unqualifiedName}"), output.requiredExpression.valueString.md5HashValue)
+    }
+
+    // Build these all together for the final set of initial hashes:
+    Set(commandTemplateHash, backendNameHash, inputCountHash, outputCountHash) ++ runtimeAttributeHashes ++ inputHashResults ++ outputExpressionHashResults
+  }
+
+  private [callcaching] def makeFileHashingActor() = {
+    val fileHashingActorName = s"FileHashingActor_for_${jobDescriptor.key.tag}"
+    context.actorOf(fileHashingActorProps, fileHashingActorName)
+  }
+}
+
+object CallCacheHashingJobActor {
+
+  def props(jobDescriptor: BackendJobDescriptor,
+            callCacheReadingJobActor: Option[ActorRef],
+            initializationData: Option[BackendInitializationData],
+            runtimeAttributeDefinitions: Set[RuntimeAttributeDefinition],
+            backendName: String,
+            fileHashingActorProps: Props,
+            writeToCache: Boolean,
+            callCachingEligible: CallCachingEligible
+           ) = Props(new CallCacheHashingJobActor(
+    jobDescriptor,
+    callCacheReadingJobActor,
+    initializationData,
+    runtimeAttributeDefinitions,
+    backendName,
+    fileHashingActorProps,
+    writeToCache,
+    callCachingEligible
+  )).withDispatcher(EngineDispatcher)
+
+  sealed trait CallCacheHashingJobActorState
+  case object WaitingForHashFileRequest extends CallCacheHashingJobActorState
+  case object HashingFiles extends CallCacheHashingJobActorState
+
+  /**
+    * Aggregates hash results together in a predictable and reproducible manner.
+    * Purposefully takes an Iterable, because it will be sorted appropriately in this method
+    * to ensure deterministic result, so the type of collection doesn't matter.
+    * Only aggregates hash values for which checkForHitOrMiss is true.
+    * If several hash keys are identical, the result of this method is undefined.
+    */
+  private def calculateHashAggregation(hashes: Iterable[HashResult], messageDigest: MessageDigest) = {
+    val sortedHashes = hashes.toList
+      .filter(_.hashKey.checkForHitOrMiss)
+      .sortBy(_.hashKey.key)
+      .map({ case HashResult(hashKey, HashValue(hashValue)) => hashKey.key + hashValue })
+      .map(_.getBytes)
+    sortedHashes foreach messageDigest.update
+    DatatypeConverter.printHexBinary(messageDigest.digest())
+  }
+
+  object CallCacheHashingJobActorData {
+    // Slick will eventually build a prepared statement with that many parameters. Don't set this too high or it will stackoverflow.
+    val BatchSize = 100
+
+    def apply(fileHashRequestsRemaining: List[SingleFileHashRequest], callCacheReadingJobActor: Option[ActorRef]) = {
+      new CallCacheHashingJobActorData(fileHashRequestsRemaining.grouped(BatchSize).toList, List.empty, callCacheReadingJobActor)
+    }
+  }
+
+  final case class CallCacheHashingJobActorData(
+                                           fileHashRequestsRemaining: List[List[SingleFileHashRequest]],
+                                           fileHashResults: List[HashResult],
+                                           callCacheReadingJobActor: Option[ActorRef]
+                                         ) {
+    private val md5Digest = MessageDigest.getInstance("MD5")
+
+    /**
+      * Returns the updated state data along with an optional message to be sent back to CCRJA and parent.
+      */
+    def withFileHash(hashResult: HashResult): (CallCacheHashingJobActorData, Option[CCHJAFileHashResponse]) = {
+      // Add the hash result to the list of known hash results
+      val newFileHashResults = hashResult +: fileHashResults
+
+      // Returns a pair of the updated fileHashRequestsRemaining and optionally either a PartialFileHashingResult or a CompleteFileHashingResult
+      val (updatedRequestsList, responseMessage) = fileHashRequestsRemaining match {
+        case Nil => (List.empty, Option(NoFileHashesResult))
+        case lastBatch :: Nil =>
+          val updatedBatch = lastBatch.filterNot(_.hashKey == hashResult.hashKey)
+          // If we're processing the last batch, and it's now empty, then we're done
+          // In that case compute the aggregated hash and send that
+          if (updatedBatch.isEmpty) (List.empty, Option(CompleteFileHashingResult(newFileHashResults.toSet, calculateHashAggregation(newFileHashResults, md5Digest))))
+          // Otherwise just return the updated batch and no message  
+          else (List(updatedBatch), None)
+        case currentBatch :: otherBatches =>
+          val updatedBatch = currentBatch.filterNot(_.hashKey == hashResult.hashKey)
+          // If the current batch is empty, we got a partial result, take the first BatchSize of the list 
+          if (updatedBatch.isEmpty) {
+            // hashResult + fileHashResults.take(BatchSize - 1) -> BatchSize elements
+            val partialHashes = NonEmptyList.of[HashResult](hashResult, fileHashResults.take(BatchSize - 1): _*)
+            (otherBatches, Option(PartialFileHashingResult(partialHashes)))
+          }
+          // Otherwise just return the updated request list and no message
+          else (updatedBatch :: otherBatches, None)
+      }
+
+      (this.copy(fileHashRequestsRemaining = updatedRequestsList, fileHashResults = newFileHashResults), responseMessage)
+    }
+  }
+
+  object UnspecifiedRuntimeAttributeHashValue extends HashValue("N/A")
+
+  sealed trait CCHJARequest
+  case object NextBatchOfFileHashesRequest extends CCHJARequest
+
+  sealed trait CCHJAResponse
+  case class InitialHashingResult(initialHashes: Set[HashResult], aggregatedBaseHash: String) extends CCHJAResponse
+
+  // File Hashing responses
+  sealed trait CCHJAFileHashResponse extends CCHJAResponse
+  case class PartialFileHashingResult(initialHashes: NonEmptyList[HashResult]) extends CCHJAFileHashResponse
+
+  sealed trait FinalFileHashingResult extends CCHJAFileHashResponse
+  case class CompleteFileHashingResult(fileHashes: Set[HashResult], aggregatedFileHash: String) extends FinalFileHashingResult
+  case object NoFileHashesResult extends FinalFileHashingResult
+
+  implicit class StringMd5er(val unhashedString: String) extends AnyVal {
+    def md5HashValue: HashValue = {
+      val hashBytes = java.security.MessageDigest.getInstance("MD5").digest(unhashedString.getBytes)
+      HashValue(javax.xml.bind.DatatypeConverter.printHexBinary(hashBytes))
+    }
+  }
+}
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheReadActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheReadActor.scala
index 739030c5f..7ee8eb693 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheReadActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheReadActor.scala
@@ -1,11 +1,13 @@
 package cromwell.engine.workflow.lifecycle.execution.callcaching
 
-import akka.actor.{Actor, ActorLogging, ActorRef, Props}
+import akka.actor.{Actor, ActorLogging, ActorRef, Props, Status}
 import akka.pattern.pipe
+import cats.data.NonEmptyList
+import cromwell.backend.BackendJobDescriptorKey
 import cromwell.core.Dispatcher.EngineDispatcher
+import cromwell.core.WorkflowId
 import cromwell.core.callcaching.HashResult
 import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheReadActor._
-import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.CallCacheHashes
 
 import scala.concurrent.ExecutionContext
 
@@ -22,23 +24,47 @@ class CallCacheReadActor(cache: CallCache) extends Actor with ActorLogging {
   private var currentRequester: Option[ActorRef] = None
 
   override def receive: Receive = {
-    case CacheLookupRequest(callCacheHashes) =>
-      receiveNewRequest(callCacheHashes)
-    case r: CallCacheReadActorResponse =>
-      currentRequester foreach { _ ! r }
+    case request: CallCacheReadActorRequest => receiveNewRequest(request)
+    case response: CallCacheReadActorResponse =>
+      currentRequester foreach { _ ! response }
+      cycleRequestQueue()
+    case Status.Failure(f) =>
+      currentRequester foreach { _ ! CacheResultLookupFailure(new Exception(s"Call Cache query failure: ${f.getMessage}")) }
       cycleRequestQueue()
     case other =>
       log.error("Unexpected message type to CallCacheReadActor: " + other.getClass.getSimpleName)
   }
 
-  private def runRequest(callCacheHashes: CallCacheHashes): Unit = {
-    val response = cache.callCachingEntryIdsMatchingHashes(callCacheHashes) map {
-      CacheResultMatchesForHashes(callCacheHashes.hashes, _)
-    } recover {
+  private def runRequest(request: CallCacheReadActorRequest): Unit = {
+    val response = request match {
+      case HasMatchingInitialHashLookup(initialHash) =>
+        cache.hasBaseAggregatedHashMatch(initialHash) map {
+          case true => HasMatchingEntries
+          case false => NoMatchingEntries
+        }
+      case HasMatchingInputFilesHashLookup(fileHashes) =>
+        cache.hasKeyValuePairHashMatch(fileHashes) map {
+          case true => HasMatchingEntries
+          case false => NoMatchingEntries
+        }
+      case CacheLookupRequest(aggregatedCallHashes, cacheHitNumber) =>
+        cache.callCachingHitForAggregatedHashes(aggregatedCallHashes, cacheHitNumber) map {
+          case Some(nextHit) => CacheLookupNextHit(nextHit)
+          case None => CacheLookupNoHit
+        }
+      case call @ CallCacheEntryForCall(workflowId, jobKey) =>
+        import cromwell.core.ExecutionIndex._
+        cache.cacheEntryExistsForCall(workflowId.toString, jobKey.call.fullyQualifiedName, jobKey.index.fromIndex) map {
+          case true => HasCallCacheEntry(call)
+          case false => NoCallCacheEntry(call)
+        }
+    }
+
+    val recovered = response recover {
       case t => CacheResultLookupFailure(t)
     }
 
-    response.pipeTo(self)
+    recovered.pipeTo(self)
     ()
   }
 
@@ -51,22 +77,44 @@ class CallCacheReadActor(cache: CallCache) extends Actor with ActorLogging {
       currentRequester = None
   }
 
-  private def receiveNewRequest(callCacheHashes: CallCacheHashes): Unit = currentRequester match {
-    case Some(x) => requestQueue :+= RequestTuple(sender, callCacheHashes)
+  private def receiveNewRequest(request: CallCacheReadActorRequest): Unit = currentRequester match {
+    case Some(_) => requestQueue :+= RequestTuple(sender, request)
     case None =>
       currentRequester = Option(sender)
-      runRequest(callCacheHashes)
+      runRequest(request)
   }
 }
 
 object CallCacheReadActor {
   def props(callCache: CallCache): Props = Props(new CallCacheReadActor(callCache)).withDispatcher(EngineDispatcher)
 
-  private[CallCacheReadActor] case class RequestTuple(requester: ActorRef, hashes: CallCacheHashes)
+  private[CallCacheReadActor] case class RequestTuple(requester: ActorRef, request: CallCacheReadActorRequest)
 
-  case class CacheLookupRequest(callCacheHashes: CallCacheHashes)
+  object AggregatedCallHashes {
+    def apply(baseAggregatedHash: String, inputFilesAggregatedHash: String) = {
+      new AggregatedCallHashes(baseAggregatedHash, Option(inputFilesAggregatedHash))
+    }
+  }
+  case class AggregatedCallHashes(baseAggregatedHash: String, inputFilesAggregatedHash: Option[String])
+
+  sealed trait CallCacheReadActorRequest
+  final case class CacheLookupRequest(aggregatedCallHashes: AggregatedCallHashes, cacheHitNumber: Int) extends CallCacheReadActorRequest
+  final case class HasMatchingInitialHashLookup(aggregatedTaskHash: String) extends CallCacheReadActorRequest
+  final case class HasMatchingInputFilesHashLookup(fileHashes: NonEmptyList[HashResult]) extends CallCacheReadActorRequest
+  final case class CallCacheEntryForCall(workflowId: WorkflowId, jobKey: BackendJobDescriptorKey) extends CallCacheReadActorRequest
 
   sealed trait CallCacheReadActorResponse
-  case class CacheResultMatchesForHashes(hashResults: Set[HashResult], cacheResultIds: Set[CallCachingEntryId]) extends CallCacheReadActorResponse
+  // Responses on whether or not there is at least one matching entry (can for initial matches of file matches)
+  case object HasMatchingEntries extends CallCacheReadActorResponse
+  case object NoMatchingEntries extends CallCacheReadActorResponse
+
+  // Responses when asking for the next cache hit
+  final case class CacheLookupNextHit(hit: CallCachingEntryId) extends CallCacheReadActorResponse
+  case object CacheLookupNoHit extends CallCacheReadActorResponse
+  
+  final case class HasCallCacheEntry(call: CallCacheEntryForCall) extends CallCacheReadActorResponse
+  final case class NoCallCacheEntry(call: CallCacheEntryForCall) extends CallCacheReadActorResponse
+  
+  // Failure Response
   case class CacheResultLookupFailure(reason: Throwable) extends CallCacheReadActorResponse
 }
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheReadingJobActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheReadingJobActor.scala
new file mode 100644
index 000000000..85f607952
--- /dev/null
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheReadingJobActor.scala
@@ -0,0 +1,102 @@
+package cromwell.engine.workflow.lifecycle.execution.callcaching
+
+import akka.actor.{ActorRef, LoggingFSM, Props}
+import cromwell.core.callcaching.HashingFailedMessage
+import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheHashingJobActor.{CompleteFileHashingResult, InitialHashingResult, NextBatchOfFileHashesRequest, NoFileHashesResult, PartialFileHashingResult}
+import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheReadActor._
+import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheReadingJobActor._
+import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.{CacheHit, CacheMiss, HashError}
+import cromwell.core.Dispatcher.EngineDispatcher
+
+/**
+  * Receives hashes from the CallCacheHashingJobActor and makes requests to the database to determine whether or not there might be a hit
+  * for this job.
+  * 
+  * First receives the initial hashes, and asks the database if there is at least one entry with the same aggregated initial hash.
+  * If not, it's a CacheMiss.
+  * If yes, ask the CallCacheHashingJobActor for the next batch of file hashes.
+  * Every time a new batch of hashes is received, check against the database if at least one entry matches all those hashes.
+  * Keep asking for new batches until either one returns no matching entry, in which case it's a CacheMiss, or until it receives
+  * the last batch along with the aggregated file hash.
+  * In the latter case, asks the database for the first entry matching both the initial and aggregated file hash (if any).
+  * Sends the response to its parent.
+  * In case of a CacheHit, stays alive in case using the hit fails and it needs to fetch the next one. Otherwise just dies.
+  */
+class CallCacheReadingJobActor(callCacheReadActor: ActorRef) extends LoggingFSM[CallCacheReadingJobActorState, CCRJAData] {
+  
+  startWith(WaitingForInitialHash, CCRJANoData)
+  
+  when(WaitingForInitialHash) {
+    case Event(InitialHashingResult(_, aggregatedBaseHash), CCRJANoData) =>
+      callCacheReadActor ! HasMatchingInitialHashLookup(aggregatedBaseHash)
+      goto(WaitingForHashCheck) using CCRJAWithData(sender(), aggregatedBaseHash, None, 1)
+  }
+  
+  when(WaitingForHashCheck) {
+    case Event(HasMatchingEntries, CCRJAWithData(hashingActor, _, _, _)) =>
+      hashingActor ! NextBatchOfFileHashesRequest
+      goto(WaitingForFileHashes)
+    case Event(NoMatchingEntries, _) =>
+      cacheMiss
+  }
+  
+  when(WaitingForFileHashes) {
+    case Event(PartialFileHashingResult(hashes), _) =>
+      callCacheReadActor ! HasMatchingInputFilesHashLookup(hashes)
+      goto(WaitingForHashCheck)
+    case Event(CompleteFileHashingResult(_, aggregatedFileHash), data: CCRJAWithData) =>
+      callCacheReadActor ! CacheLookupRequest(AggregatedCallHashes(data.initialHash, aggregatedFileHash), data.currentHitNumber)
+      goto(WaitingForCacheHitOrMiss) using data.withFileHash(aggregatedFileHash)
+    case Event(NoFileHashesResult, data: CCRJAWithData) =>
+      callCacheReadActor ! CacheLookupRequest(AggregatedCallHashes(data.initialHash, None), data.currentHitNumber)
+      goto(WaitingForCacheHitOrMiss)
+  }
+  
+  when(WaitingForCacheHitOrMiss) {
+    case Event(CacheLookupNextHit(hit), data: CCRJAWithData) =>
+      context.parent ! CacheHit(hit)
+      stay() using data.increment
+    case Event(CacheLookupNoHit, _) =>
+      cacheMiss
+    case Event(NextHit, CCRJAWithData(_, aggregatedInitialHash, aggregatedFileHash, currentHitNumber)) =>
+      callCacheReadActor ! CacheLookupRequest(AggregatedCallHashes(aggregatedInitialHash, aggregatedFileHash), currentHitNumber)
+      stay()
+  }
+  
+  whenUnhandled {
+    case Event(_: HashingFailedMessage, _) =>
+      // No need to send to the parent since it also receives file hash updates
+      cacheMiss
+    case Event(CacheResultLookupFailure(failure), _) =>
+      context.parent ! HashError(failure)
+      cacheMiss
+  }
+  
+  def cacheMiss = {
+    context.parent ! CacheMiss
+    context stop self
+    stay()
+  }
+}
+
+object CallCacheReadingJobActor {
+  
+  def props(callCacheReadActor: ActorRef) = {
+    Props(new CallCacheReadingJobActor(callCacheReadActor)).withDispatcher(EngineDispatcher)
+  }
+  
+  sealed trait CallCacheReadingJobActorState
+  case object WaitingForInitialHash extends CallCacheReadingJobActorState
+  case object WaitingForHashCheck extends CallCacheReadingJobActorState
+  case object WaitingForFileHashes extends CallCacheReadingJobActorState
+  case object WaitingForCacheHitOrMiss extends CallCacheReadingJobActorState
+
+  sealed trait CCRJAData
+  case object CCRJANoData extends CCRJAData
+  case class CCRJAWithData(hashingActor: ActorRef, initialHash: String, fileHash: Option[String], currentHitNumber: Int) extends CCRJAData {
+    def increment = this.copy(currentHitNumber = currentHitNumber + 1)
+    def withFileHash(aggregatedFileHash: String) = this.copy(fileHash = Option(aggregatedFileHash))
+  }
+
+  case object NextHit
+}
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheWriteActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheWriteActor.scala
index b77bda043..8e7aad74e 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheWriteActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheWriteActor.scala
@@ -1,39 +1,73 @@
 package cromwell.engine.workflow.lifecycle.execution.callcaching
 
-import akka.actor.{Actor, ActorLogging, Props}
-import cromwell.backend.BackendJobExecutionActor
-import cromwell.backend.BackendJobExecutionActor.JobSucceededResponse
+import akka.actor.{ActorRef, LoggingFSM, Props}
+import cats.instances.list._
+import cats.instances.tuple._
+import cats.syntax.foldable._
 import cromwell.core.Dispatcher.EngineDispatcher
-import cromwell.core.WorkflowId
-import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.CallCacheHashes
+import cromwell.core.actor.BatchingDbWriter._
+import cromwell.core.actor.{BatchingDbWriter, BatchingDbWriterActor}
+import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCache.CallCacheHashBundle
+import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheWriteActor.{SaveCallCacheHashes, _}
 
 import scala.concurrent.ExecutionContext
-import scala.util.{Failure, Success}
+import scala.concurrent.duration._
+import scala.language.postfixOps
 
-case class CallCacheWriteActor(callCache: CallCache, workflowId: WorkflowId, callCacheHashes: CallCacheHashes, succeededResponse: BackendJobExecutionActor.JobSucceededResponse) extends Actor with ActorLogging {
+case class CallCacheWriteActor(callCache: CallCache) extends LoggingFSM[BatchingDbWriterState, BatchingDbWriter.BatchingDbWriterData] with BatchingDbWriterActor {
 
   implicit val ec: ExecutionContext = context.dispatcher
+  
+  override val dbFlushRate = CallCacheWriteActor.dbFlushRate
 
-  def receiver = context.parent
+  startWith(WaitingToWrite, NoData)
 
-  callCache.addToCache(workflowId, callCacheHashes, succeededResponse) onComplete {
-    case Success(_) =>
-      receiver ! CallCacheWriteSuccess
-      context.stop(self)
-    case Failure(t) =>
-      receiver ! CallCacheWriteFailure(t)
-      context.stop(self)
+  when(WaitingToWrite) {
+    case Event(command: SaveCallCacheHashes, curData) =>
+      curData.addData(CommandAndReplyTo(command, sender)) match {
+        case newData: HasData[_] if newData.length >= dbBatchSize => goto(WritingToDb) using newData
+        case newData => stay() using newData
+      }
+    case Event(ScheduledFlushToDb, _) =>
+      log.debug("Initiating periodic call cache flush to DB")
+      goto(WritingToDb)
   }
 
-  override def receive = {
-    case any => log.error("Unexpected message to CallCacheWriteActor: " + any)
+  when(WritingToDb) {
+    case Event(ScheduledFlushToDb, _) => stay
+    case Event(command: SaveCallCacheHashes, curData) => stay using curData.addData(CommandAndReplyTo(command, sender))
+    case Event(FlushBatchToDb, NoData) =>
+      log.debug("Attempted call cache hash set flush to DB but had nothing to write")
+      goto(WaitingToWrite)
+    case Event(FlushBatchToDb, HasData(data)) =>
+      log.debug("Flushing {} call cache hashes sets to the DB", data.length)
+
+      // Collect all the bundles of hashes that should be written and all the senders which should be informed of
+      // success or failure.
+      val (bundles, replyTos) = data.foldMap { case CommandAndReplyTo(s: SaveCallCacheHashes, r: ActorRef) => (List(s.bundle), List(r)) }
+      if (bundles.nonEmpty) {
+        val futureMessage = callCache.addToCache(bundles, dbBatchSize) map { _ => CallCacheWriteSuccess } recover { case t => CallCacheWriteFailure(t) }
+        futureMessage map { message =>
+          replyTos foreach { _ ! message }
+          self ! DbWriteComplete
+        }
+      }
+      stay using NoData
+    case Event(DbWriteComplete, _) =>
+      log.debug("Flush of cache data complete")
+      goto(WaitingToWrite)
   }
 }
 
 object CallCacheWriteActor {
-  def props(callCache: CallCache, workflowId: WorkflowId, callCacheHashes: CallCacheHashes, succeededResponse: JobSucceededResponse): Props =
-    Props(CallCacheWriteActor(callCache, workflowId, callCacheHashes, succeededResponse)).withDispatcher(EngineDispatcher)
+  def props(callCache: CallCache): Props = Props(CallCacheWriteActor(callCache)).withDispatcher(EngineDispatcher)
+
+  case class SaveCallCacheHashes(bundle: CallCacheHashBundle)
+
+  val dbBatchSize = 100
+  val dbFlushRate = 3 seconds
 }
 
-case object CallCacheWriteSuccess
-case class CallCacheWriteFailure(t: Throwable)
+sealed trait CallCacheWriteResponse
+case object CallCacheWriteSuccess extends CallCacheWriteResponse
+case class CallCacheWriteFailure(t: Throwable) extends CallCacheWriteResponse
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActor.scala
index 4830951c0..94933ba73 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActor.scala
@@ -1,269 +1,144 @@
 package cromwell.engine.workflow.lifecycle.execution.callcaching
 
-import akka.actor.{ActorLogging, ActorRef, LoggingFSM, Props}
-import cats.data.NonEmptyList
-import cromwell.backend.callcaching.FileHashingActor.SingleFileHashRequest
+import akka.actor.{Actor, ActorLogging, ActorRef, Props}
 import cromwell.backend.{BackendInitializationData, BackendJobDescriptor, RuntimeAttributeDefinition}
-import cromwell.core.callcaching._
 import cromwell.core.Dispatcher.EngineDispatcher
-import cromwell.core.simpleton.WdlValueSimpleton
-import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheReadActor.{CacheLookupRequest, CacheResultLookupFailure, CacheResultMatchesForHashes}
+import cromwell.core.WorkflowId
+import cromwell.core.callcaching._
+import cromwell.core.logging.JobLogging
+import cromwell.engine.workflow.lifecycle.execution.CallMetadataHelper
+import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheHashingJobActor.{CompleteFileHashingResult, FinalFileHashingResult, InitialHashingResult, NoFileHashesResult}
+import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheReadingJobActor.NextHit
 import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor._
-import wdl4s.values.WdlFile
+import cromwell.services.metadata.CallMetadataKeys
 
 /**
+  * Coordinates the CallCacheHashingJobActor and the CallCacheReadingJobActor.
   * Over time will emit up to two messages back to its parent:
   *  * (if read enabled): Either a CacheHit(id) or CacheMiss message
   *  * (if write enabled): A CallCacheHashes(hashes) message
   */
-case class EngineJobHashingActor(receiver: ActorRef,
-                                 jobDescriptor: BackendJobDescriptor,
-                                 initializationData: Option[BackendInitializationData],
-                                 fileHashingActor: ActorRef,
-                                 callCacheReadActor: ActorRef,
-                                 runtimeAttributeDefinitions: Set[RuntimeAttributeDefinition],
-                                 backendName: String,
-                                 activity: CallCachingActivity) extends LoggingFSM[EJHAState, EJHAData] with ActorLogging {
-
-  initializeEJHA()
-
-  when(DeterminingHitOrMiss) {
-    case Event(hashResultMessage: SuccessfulHashResultMessage, _) =>
-      // This is in DeterminingHitOrMiss, so use the new hash results to search for cache results.
-      // Also update the state data with these new hash results.
-      val newHashResults = hashResultMessage.hashes
-      findCacheResults(newHashResults)
-      updateStateDataWithNewHashResultsAndTransition(newHashResults)
-    case Event(newCacheResults: CacheResultMatchesForHashes, _) =>
-      checkWhetherHitOrMissIsKnownThenTransition(stateData.intersectCacheResults(newCacheResults))
+class EngineJobHashingActor(receiver: ActorRef,
+                            override val serviceRegistryActor: ActorRef,
+                            jobDescriptor: BackendJobDescriptor,
+                            initializationData: Option[BackendInitializationData],
+                            fileHashingActorProps: Props,
+                            callCacheReadingJobActorProps: Props,
+                            runtimeAttributeDefinitions: Set[RuntimeAttributeDefinition],
+                            backendName: String,
+                            activity: CallCachingActivity,
+                            callCachingEligible: CallCachingEligible) extends Actor with ActorLogging with JobLogging with CallMetadataHelper {
+
+  override val jobTag = jobDescriptor.key.tag
+  override val workflowId = jobDescriptor.workflowDescriptor.id
+  override val workflowIdForCallMetadata: WorkflowId = workflowId
+
+  private [callcaching] var initialHash: Option[InitialHashingResult] = None
+
+  private [callcaching] val callCacheReadingJobActor = if (activity.readFromCache) {
+    Option(context.actorOf(callCacheReadingJobActorProps))
+  } else None
+
+  override def preStart(): Unit = {
+    context.actorOf(CallCacheHashingJobActor.props(
+      jobDescriptor,
+      callCacheReadingJobActor,
+      initializationData,
+      runtimeAttributeDefinitions,
+      backendName,
+      fileHashingActorProps,
+      activity.writeToCache,
+      callCachingEligible
+    ))
+    super.preStart()
   }
 
-  when(GeneratingAllHashes) {
-    case Event(hashResultMessage: SuccessfulHashResultMessage, _) => checkWhetherAllHashesAreKnownAndTransition(stateData.withNewKnownHashes(hashResultMessage.hashes))
-    case Event(CacheResultMatchesForHashes(_, _), _) => stay // Don't care; we already know the hit/miss status. Ignore this message
-  }
-
-  whenUnhandled {
-    case Event(CacheResultLookupFailure(reason), _) =>
-      receiver ! HashError(new Exception(s"Failure looking up call cache results: ${reason.getMessage}"))
-      context.stop(self)
-      stay
-    case Event(HashingFailedMessage(hashKey, reason), _) =>
-      receiver ! HashError(new Exception(s"Unable to generate ${hashKey.key} hash. Caused by ${reason.getMessage}", reason))
-      context.stop(self)
-      stay
-    case Event(other, _) =>
-      log.error(s"Bad message in $stateName with $stateData: $other")
-      stay
+  override def receive = {
+    case initialHashResult: InitialHashingResult => initialHash = Option(initialHashResult)
+    case finalFileHashResult: FinalFileHashingResult => sendHashes(finalFileHashResult)
+    case CacheMiss => receiver ! CacheMiss
+    case hit: CacheHit => receiver ! hit
+    case NextHit =>
+      callCacheReadingJobActor match {
+        case Some(readActor) =>
+          readActor ! NextHit
+        case None =>
+          failAndStop(new IllegalStateException("Requested cache hit but there is no cache read actor"))
+      }
+    case hashingFailed: HashingFailedMessage =>
+      failAndStop(hashingFailed.reason)
+    case unexpected =>
+      jobLogger.error(s"Received unexpected event $unexpected")
   }
 
-  onTransition {
-    case fromState -> toState =>
-      log.debug("Transitioning from {}({}) to {}({})", fromState, stateData, toState, nextStateData)
+  private def publishHashFailure(failure: Throwable) = {
+    import cromwell.services.metadata.MetadataService._
+    val failureAsEvents = throwableToMetadataEvents(metadataKeyForCall(jobDescriptor.key, CallMetadataKeys.CallCachingKeys.HashFailuresKey), failure)
+    serviceRegistryActor ! PutMetadataAction(failureAsEvents)
   }
 
-  private def initializeEJHA() = {
-
-    import cromwell.core.simpleton.WdlValueSimpleton._
-
-    val unqualifiedInputs = jobDescriptor.inputDeclarations map { case (declaration, value) => declaration.unqualifiedName -> value }
-
-    val inputSimpletons = unqualifiedInputs.simplify
-    val (fileInputSimpletons, nonFileInputSimpletons) = inputSimpletons partition {
-      case WdlValueSimpleton(_, f: WdlFile) => true
-      case _ => false
-    }
-
-    val initialHashes = calculateInitialHashes(nonFileInputSimpletons, fileInputSimpletons)
-
-    val fileContentHashesNeeded = fileInputSimpletons collect {
-      case WdlValueSimpleton(name, x: WdlFile) => SingleFileHashRequest(jobDescriptor.key, HashKey(s"input: File $name"), x, initializationData)
-    }
-
-    val hashesNeeded: Set[HashKey] = initialHashes.map(_.hashKey) ++ fileContentHashesNeeded.map(_.hashKey)
-
-    val initialState = if (activity.readFromCache) DeterminingHitOrMiss else GeneratingAllHashes
-    val initialData = EJHAData(hashesNeeded, activity)
-
-    startWith(initialState, initialData)
-
-    // Submit the set of initial hashes for checking against the DB:
-    self ! EJHAInitialHashingResults(initialHashes)
-    // Find the hashes for all input files:
-    fileContentHashesNeeded.foreach(fileHashingActor ! _)
+  private def failAndStop(reason: Throwable) = {
+    publishHashFailure(reason)
+    receiver ! HashError(reason)
+    context stop self
   }
 
-  private def calculateInitialHashes(nonFileInputs: Iterable[WdlValueSimpleton], fileInputs: Iterable[WdlValueSimpleton]): Set[HashResult] = {
-
-    val commandTemplateHash = HashResult(HashKey("command template"), jobDescriptor.call.task.commandTemplateString.md5HashValue)
-    val backendNameHash = HashResult(HashKey("backend name"), backendName.md5HashValue)
-    val inputCountHash = HashResult(HashKey("input count"), (nonFileInputs.size + fileInputs.size).toString.md5HashValue)
-    val outputCountHash = HashResult(HashKey("output count"), jobDescriptor.call.task.outputs.size.toString.md5HashValue)
-
-    val runtimeAttributeHashes = runtimeAttributeDefinitions map { definition => jobDescriptor.runtimeAttributes.get(definition.name) match {
-      case Some(wdlValue) => HashResult(HashKey("runtime attribute: " + definition.name, definition.usedInCallCaching), wdlValue.valueString.md5HashValue)
-      case None => HashResult(HashKey("runtime attribute: " + definition.name, definition.usedInCallCaching), UnspecifiedRuntimeAttributeHashValue)
-    }}
-
-    val inputHashResults = nonFileInputs map {
-      case WdlValueSimpleton(name, value) => HashResult(HashKey(s"input: ${value.wdlType.toWdlString} $name"),  value.toWdlString.md5HashValue)
-    }
-
-    val outputExpressionHashResults = jobDescriptor.call.task.outputs map { output =>
-      HashResult(HashKey(s"output expression: ${output.wdlType.toWdlString} ${output.unqualifiedName}"), output.requiredExpression.valueString.md5HashValue)
+  private def sendHashes(finalFileHashingResult: FinalFileHashingResult) = {
+    val fileHashes = finalFileHashingResult match {
+      case CompleteFileHashingResult(fileHashResults, aggregatedFileHash) =>
+        Option(FileHashes(fileHashResults, aggregatedFileHash))
+      case NoFileHashesResult => None
     }
 
-    // Build these all together for the final set of initial hashes:
-    Set(commandTemplateHash, backendNameHash, inputCountHash, outputCountHash) ++ runtimeAttributeHashes ++ inputHashResults ++ outputExpressionHashResults
-  }
-
-  private def checkWhetherHitOrMissIsKnownThenTransition(newData: EJHAData) = {
-    if (newData.isDefinitelyCacheHitOrMiss) {
-      respondWithHitOrMissThenTransition(newData)
-    }
-    else {
-      stay() using newData // Stay in DeterminingHitOrMiss
+    initialHash match {
+      case Some(initData) =>
+        receiver ! CallCacheHashes(initData.initialHashes, initData.aggregatedBaseHash, fileHashes)
+      case None =>
+        failAndStop(new IllegalStateException("Received file hashes without initial hash."))
     }
   }
+}
 
-  private def respondWithHitOrMissThenTransition(newData: EJHAData) = {
-    import cats.data.NonEmptyList
-    val hitOrMissResponse: EJHAResponse = newData.cacheHits map { _.toList } flatMap NonEmptyList.fromList map CacheHit.apply getOrElse CacheMiss
+object EngineJobHashingActor {
+  sealed trait EJHAState
+  case object Running extends EJHAState
+  case object WaitingForHashes extends EJHAState
+  case object WaitingForJobSuccess extends EJHAState
+  case object Done extends EJHAState
 
-    receiver ! hitOrMissResponse
-    if (!activity.writeToCache) {
-      context.stop(self)
-      stay
-    } else {
-      checkWhetherAllHashesAreKnownAndTransition(newData)
-    }
+  object EJHAData {
+    def empty = EJHAData(None)
   }
 
-  private def checkWhetherAllHashesAreKnownAndTransition(newData: EJHAData) = {
-    if (newData.allHashesKnown) {
-      receiver ! CallCacheHashes(newData.hashesKnown)
-      context.stop(self)
-    }
-    goto(GeneratingAllHashes) using newData
-  }
+  case class EJHAData(initialHash: Option[InitialHashingResult])
 
-  /**
-    * Needs to convert a hash result into the set of CachedResults which are consistent with it
-    */
-  private def findCacheResults(hashResults: Set[HashResult]) = {
-    val filtered = hashResults.filter(_.hashKey.checkForHitOrMiss)
-
-    if (filtered.nonEmpty) {
-      val hashes = CallCacheHashes(filtered)
-      val subsets = hashes.hashes.grouped(100)
-      subsets foreach { subset =>
-        callCacheReadActor ! CacheLookupRequest(CallCacheHashes(subset))
-      }
-    } else ()
-  }
-
-  def updateStateDataWithNewHashResultsAndTransition(hashResults: Set[HashResult]) = {
-    if (activity.writeToCache) {
-      val newData = stateData.withNewKnownHashes(hashResults)
-      if (newData.isDefinitelyCacheHitOrMiss) {
-        log.info("New hash results, hit or miss already known (none are cache-checked. Checking if we're done...)")
-        checkWhetherAllHashesAreKnownAndTransition(newData)
-      } else {
-        stay using newData
-      }
-    } else {
-      stay using stateData
-    }
+  sealed trait EJHAResponse
+  case object CacheMiss extends EJHAResponse
+  case class CacheHit(cacheResultId: CallCachingEntryId) extends EJHAResponse
+  case class HashError(reason: Throwable) extends EJHAResponse
+  case class FileHashes(hashes: Set[HashResult], aggregatedHash: String)
+  case class CallCacheHashes(initialHashes: Set[HashResult], aggregatedInitialHash: String, fileHashes: Option[FileHashes]) extends EJHAResponse {
+    val hashes = initialHashes ++ fileHashes.map(_.hashes).getOrElse(Set.empty)
   }
-}
-
-object EngineJobHashingActor {
 
   def props(receiver: ActorRef,
+            serviceRegistryActor: ActorRef,
             jobDescriptor: BackendJobDescriptor,
             initializationData: Option[BackendInitializationData],
-            fileHashingActor: ActorRef,
-            callCacheReadActor: ActorRef,
+            fileHashingActorProps: Props,
+            callCacheReadingJobActorProps: Props,
             runtimeAttributeDefinitions: Set[RuntimeAttributeDefinition],
             backendName: String,
-            activity: CallCachingActivity): Props = Props(new EngineJobHashingActor(
-      receiver = receiver,
-      jobDescriptor = jobDescriptor,
-      initializationData = initializationData,
-      fileHashingActor = fileHashingActor,
-      callCacheReadActor = callCacheReadActor,
-      runtimeAttributeDefinitions = runtimeAttributeDefinitions,
-      backendName = backendName,
-      activity = activity)).withDispatcher(EngineDispatcher)
-
-  private[callcaching] case class EJHAInitialHashingResults(hashes: Set[HashResult]) extends SuccessfulHashResultMessage
-  private[callcaching] case object CheckWhetherAllHashesAreKnown
-
-  sealed trait EJHAState
-  case object DeterminingHitOrMiss extends EJHAState
-  case object GeneratingAllHashes extends EJHAState
-
-  sealed trait EJHAResponse
-  case class CacheHit(cacheResultIds: NonEmptyList[CallCachingEntryId]) extends EJHAResponse
-  case object CacheMiss extends EJHAResponse
-  case class HashError(t: Throwable) extends EJHAResponse {
-    override def toString = s"HashError(${t.getMessage})"
-  }
-  case class CallCacheHashes(hashes: Set[HashResult]) extends EJHAResponse
-  object UnspecifiedRuntimeAttributeHashValue extends HashValue("N/A")
-
-  implicit class StringMd5er(unhashedString: String) {
-    def md5HashValue: HashValue = {
-      val hashBytes = java.security.MessageDigest.getInstance("MD5").digest(unhashedString.getBytes)
-      HashValue(javax.xml.bind.DatatypeConverter.printHexBinary(hashBytes))
-    }
-  }
-}
-
-/**
-  * Transient data for the EJHA.
-  *
-  * @param possibleCacheResults The set of cache results which have matched all currently tried hashes
-  * @param remainingCacheChecks The set of hash keys which have not yet had their cache results fetched
-  * @param hashesKnown The set of all hashes calculated so far (including initial hashes)
-  * @param remainingHashesNeeded The set of hashes which are still needed for writing to the database
-  */
-private[callcaching] case class EJHAData(possibleCacheResults: Option[Set[CallCachingEntryId]],
-                                         remainingCacheChecks: Set[HashKey],
-                                         hashesKnown: Set[HashResult],
-                                         remainingHashesNeeded: Set[HashKey]) {
-  // Manipulators
-  def intersectCacheResults(newCacheResults: CacheResultMatchesForHashes) = {
-    val newIds = newCacheResults.cacheResultIds
-    val intersectedIds = possibleCacheResults match {
-      case None => newIds
-      case Some(currentCacheResults) => currentCacheResults.intersect(newIds)
-    }
-    this.copy(
-      possibleCacheResults = Option(intersectedIds),
-      remainingCacheChecks = remainingCacheChecks.diff(newCacheResults.hashResults.map(_.hashKey))
-    )
-  }
-  def withNewKnownHashes(hashResults: Set[HashResult]) = {
-    this.copy(
-      hashesKnown = hashesKnown ++ hashResults,
-      remainingHashesNeeded = remainingHashesNeeded.diff(hashResults.map(_.hashKey)))
-  }
-
-  // Queries
-  def allHashesKnown = remainingHashesNeeded.isEmpty
-  def allCacheResultsIntersected = remainingCacheChecks.isEmpty
-  def cacheHit = if (allCacheResultsIntersected) possibleCacheResults flatMap { _.headOption } else None
-  def cacheHits = if (allCacheResultsIntersected) possibleCacheResults else None
-  def isDefinitelyCacheHit = cacheHits.isDefined
-  def isDefinitelyCacheMiss = possibleCacheResults.exists(_.isEmpty)
-  def isDefinitelyCacheHitOrMiss = isDefinitelyCacheHit || isDefinitelyCacheMiss
-}
-
-private[callcaching] object EJHAData {
-  def apply(hashesNeeded: Set[HashKey], activity: CallCachingActivity): EJHAData = EJHAData(
-    None,
-    if (activity.readFromCache) hashesNeeded.filter(_.checkForHitOrMiss) else Set.empty,
-    Set.empty,
-    if (activity.writeToCache) hashesNeeded else Set.empty)
+            activity: CallCachingActivity,
+            callCachingEligible: CallCachingEligible): Props = Props(new EngineJobHashingActor(
+    receiver = receiver,
+    serviceRegistryActor = serviceRegistryActor,
+    jobDescriptor = jobDescriptor,
+    initializationData = initializationData,
+    fileHashingActorProps = fileHashingActorProps,
+    callCacheReadingJobActorProps = callCacheReadingJobActorProps,
+    runtimeAttributeDefinitions = runtimeAttributeDefinitions,
+    backendName = backendName,
+    activity = activity,
+    callCachingEligible = callCachingEligible)).withDispatcher(EngineDispatcher)
 }
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/package.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/package.scala
index d0350e662..d44006f7c 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/package.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/package.scala
@@ -1,7 +1,7 @@
 package cromwell.engine.workflow.lifecycle
 
 import akka.actor.ActorRef
-import wdl4s._
+import wdl4s.wdl._
 
 package execution {
 
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/preparation/CallPreparation.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/preparation/CallPreparation.scala
index c39806191..3ec044208 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/preparation/CallPreparation.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/preparation/CallPreparation.scala
@@ -5,10 +5,10 @@ import cromwell.backend.BackendJobDescriptor
 import cromwell.core.{CallKey, JobKey}
 import cromwell.engine.EngineWorkflowDescriptor
 import cromwell.engine.workflow.lifecycle.execution.OutputStore
-import wdl4s.exception.VariableLookupException
-import wdl4s.expression.WdlStandardLibraryFunctions
-import wdl4s.values.WdlValue
-import wdl4s.{Declaration, Scatter}
+import wdl4s.wdl.exception.VariableLookupException
+import wdl4s.wdl.expression.WdlStandardLibraryFunctions
+import wdl4s.wdl.values.WdlValue
+import wdl4s.wdl.{Declaration, Scatter}
 
 import scala.util.{Failure, Try}
 
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/preparation/JobPreparationActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/preparation/JobPreparationActor.scala
index 5524ff640..7b3a57335 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/preparation/JobPreparationActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/preparation/JobPreparationActor.scala
@@ -1,98 +1,120 @@
 package cromwell.engine.workflow.lifecycle.execution.preparation
 
-import akka.actor.{ActorRef, Cancellable, FSM, Props}
+import akka.actor.{ActorRef, FSM, Props}
 import cromwell.backend._
-import cromwell.backend.validation.RuntimeAttributesKeys
+import cromwell.backend.validation.{DockerValidation, RuntimeAttributesKeys}
 import cromwell.core.Dispatcher.EngineDispatcher
-import cromwell.core.callcaching.docker.DockerHashActor.{DockerHashBackPressure, DockerHashFailureResponse, DockerHashResponseSuccess}
-import cromwell.core.callcaching.docker._
 import cromwell.core.callcaching._
 import cromwell.core.logging.WorkflowLogging
+import cromwell.docker.DockerHashActor.DockerHashSuccessResponse
+import cromwell.docker._
+import cromwell.engine.workflow.WorkflowDockerLookupActor.{WorkflowDockerLookupFailure, WorkflowDockerTerminalFailure}
 import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActorData
 import cromwell.engine.workflow.lifecycle.execution.preparation.CallPreparation._
 import cromwell.engine.workflow.lifecycle.execution.preparation.JobPreparationActor._
-import wdl4s._
-import wdl4s.values.{WdlString, WdlValue}
+import cromwell.services.keyvalue.KeyValueServiceActor.{KvGet, KvJobKey, KvResponse, ScopedKey}
+import wdl4s.wdl._
+import wdl4s.wdl.values.WdlValue
 
 import scala.concurrent.duration._
 import scala.language.postfixOps
 import scala.util.{Failure, Success, Try}
 
-case class JobPreparationActor(executionData: WorkflowExecutionActorData,
-                                     jobKey: BackendJobDescriptorKey,
-                                     factory: BackendLifecycleActorFactory,
-                                     dockerHashingActor: ActorRef,
-                                     initializationData: Option[BackendInitializationData],
-                                     serviceRegistryActor: ActorRef,
-                                     backendSingletonActor: Option[ActorRef])
-  extends FSM[JobPreparationActorState, Option[JobPreparationActorData]] with WorkflowLogging {
+class JobPreparationActor(executionData: WorkflowExecutionActorData,
+                          jobKey: BackendJobDescriptorKey,
+                          factory: BackendLifecycleActorFactory,
+                          val workflowDockerLookupActor: ActorRef,
+                          initializationData: Option[BackendInitializationData],
+                          serviceRegistryActor: ActorRef,
+                          ioActor: ActorRef,
+                          backendSingletonActor: Option[ActorRef])
+  extends FSM[JobPreparationActorState, JobPreparationActorData] with WorkflowLogging {
 
-  lazy val workflowIdForLogging = workflowDescriptor.id
-  
-  // Number of attempts to make contact with the docker actor before giving up if no response
-  private [preparation] lazy val maxDockerAttempts = 3
-  // Amount of time after which the docker request should be considered lost and sent agagin
-  private [preparation] lazy val dockerNoResponseTimeout = 30 seconds
-  // Amount of time to wait when we get a Backpressure response before sending the request again
-  private [preparation] lazy val backpressureWaitTime = 15 seconds
+  override lazy val workflowIdForLogging = workflowDescriptor.id
+
+  private[preparation] lazy val noResponseTimeout: FiniteDuration = 3 minutes
   
   private lazy val workflowDescriptor = executionData.workflowDescriptor
-  private val ec = context.system.dispatcher
-  private lazy val expressionLanguageFunctions = factory.expressionLanguageFunctions(workflowDescriptor.backendDescriptor, jobKey, initializationData)
-  private lazy val dockerHashCredentials = factory.dockerHashCredentials(initializationData)
-  
-  startWith(Idle, None)
+  private[preparation] lazy val expressionLanguageFunctions = factory.expressionLanguageFunctions(workflowDescriptor.backendDescriptor, jobKey, initializationData)
+  private[preparation] lazy val dockerHashCredentials = factory.dockerHashCredentials(initializationData)
+  private[preparation] lazy val runtimeAttributeDefinitions = factory.runtimeAttributeDefinitions(initializationData)
+  private[preparation] lazy val hasDockerDefinition = runtimeAttributeDefinitions.exists(_.name == DockerValidation.instance.key)
+
+  startWith(Idle, JobPreparationActorNoData)
 
   when(Idle) {
-    case Event(Start, _) =>
-      evaluateInputsAndAttributes match {
-        case Success((inputs, attributes)) => startPreparation(inputs, attributes)
-        case Failure(failure) => sendFailureAndStop(failure)
+    case Event(Start, JobPreparationActorNoData) =>
+      val keyLookupRequests = kvStoreKeysToPrefetch
+      if (keyLookupRequests.nonEmpty) {
+        lookupKeyValueEntries(keyLookupRequests)
+      } else {
+        evaluateInputsAndFetchDockerHashes(KeyValueLookupResults(Map.empty))
+      }
+  }
+
+  when(FetchingKeyValueStoreEntries) {
+    case Event(kvResponse: KvResponse, JobPreparationKeyLookupData(keyLookups)) =>
+      keyLookups.withResponse(kvResponse.key, kvResponse) match {
+        case newPartialLookup: PartialKeyValueLookups => stay using JobPreparationKeyLookupData(newPartialLookup)
+        case finished: KeyValueLookupResults => evaluateInputsAndFetchDockerHashes(finished)
       }
   }
 
   when(WaitingForDockerHash) {
-    case Event(DockerHashResponseSuccess(dockerHash), Some(data)) =>
+    case Event(DockerHashSuccessResponse(dockerHash, _), data: JobPreparationHashLookupData) =>
       handleDockerHashSuccess(dockerHash, data)
-    case Event(failureResponse: DockerHashFailureResponse, Some(data)) =>
-      log.warning(failureResponse.reason)
-      handleDockerHashFailed(failureResponse.reason, data)
-    case Event(DockerHashBackPressure(request), Some(data)) =>
-      handleDockHashBackPressure(request, data)
-    case Event(DockerNoResponseTimeout, Some(data)) =>
-      if (data.maxAttemptsReached) handleDockerHashFailed("Docker Hash service is unreachable", data)
-      else {
-        dockerHashingActor ! data.dockerHashRequest
-        stay() using data.newAttempt(newRequestLostTimer)
-      }
+    case Event(WorkflowDockerLookupFailure(reason, _), data: JobPreparationHashLookupData) =>
+      workflowLogger.warn("Docker lookup failed", reason)
+      handleDockerHashFailed(data)
+    case Event(WorkflowDockerTerminalFailure(reason, _), _: JobPreparationHashLookupData) =>
+      sendFailureAndStop(reason)
   }
-  
+
   whenUnhandled {
     case Event(unexpectedMessage, _) =>
       workflowLogger.warn(s"JobPreparation actor received an unexpected message in state $stateName: $unexpectedMessage")
       stay()
   }
-  
+
+  private[preparation] lazy val kvStoreKeysToPrefetch = factory.requestedKeyValueStoreKeys
+  private[preparation] def scopedKey(key: String) = ScopedKey(workflowDescriptor.id, KvJobKey(jobKey), key)
+  private[preparation] def lookupKeyValueEntries(lookups: Seq[String]) = {
+    val keysToLookup: Seq[ScopedKey] = lookups map scopedKey
+    keysToLookup foreach { serviceRegistryActor ! KvGet(_) }
+    goto(FetchingKeyValueStoreEntries) using JobPreparationKeyLookupData(PartialKeyValueLookups(Map.empty, keysToLookup))
+  }
+
   private [preparation] def evaluateInputsAndAttributes = {
     for {
       evaluatedInputs <- resolveAndEvaluateInputs(jobKey, workflowDescriptor, expressionLanguageFunctions, executionData.outputStore)
       runtimeAttributes <- prepareRuntimeAttributes(evaluatedInputs)
     } yield (evaluatedInputs, runtimeAttributes)
   }
-  
-  private def startPreparation(inputs: Map[Declaration, WdlValue], attributes: Map[LocallyQualifiedName, WdlValue]) = {
+
+  private def evaluateInputsAndFetchDockerHashes(kvStoreLookupResults: KeyValueLookupResults) = {
+    evaluateInputsAndAttributes match {
+      case Success((inputs, attributes)) => fetchDockerHashes(kvStoreLookupResults, inputs, attributes)
+      case Failure(failure) => sendFailureAndStop(failure)
+    }
+  }
+
+  private def fetchDockerHashes(kvStoreLookupResults: KeyValueLookupResults, inputs: Map[Declaration, WdlValue], attributes: Map[LocallyQualifiedName, WdlValue]) = {
     def sendDockerRequest(dockerImageId: DockerImageIdentifierWithoutHash) = {
       val dockerHashRequest = DockerHashRequest(dockerImageId, dockerHashCredentials)
-      val newData = JobPreparationActorData(dockerHashRequest, inputs, attributes, newRequestLostTimer, maxDockerAttempts)
-      dockerHashingActor ! dockerHashRequest
-      goto(WaitingForDockerHash) using Option(newData)
+      val newData = JobPreparationHashLookupData(kvStoreLookupResults, dockerHashRequest, inputs, attributes)
+      workflowDockerLookupActor ! dockerHashRequest
+      goto(WaitingForDockerHash) using newData
     }
-    
+
     def handleDockerValue(value: String) = DockerImageIdentifier.fromString(value) match {
-      case Success(dockerImageId: DockerImageIdentifierWithoutHash) => sendDockerRequest(dockerImageId)
-      case Success(dockerImageIdWithHash: DockerImageIdentifierWithHash) =>
-        // If the docker value already has a hash - we're ok for call caching
-        val response = prepareBackendDescriptor(inputs, attributes, CallCachingEligible)
+      case Success(dockerImageId: DockerImageIdentifierWithoutHash) if hasDockerDefinition => sendDockerRequest(dockerImageId)
+      case Success(_: DockerImageIdentifierWithoutHash) if !hasDockerDefinition =>
+        // If the backend doesn't support docker - no need to lookup and we're ok for call caching
+        val response = prepareBackendDescriptor(inputs, attributes, NoDocker, kvStoreLookupResults.unscoped)
+        sendResponseAndStop(response)
+      case Success(dockerImageId: DockerImageIdentifierWithHash) =>
+        // If the docker value already has a hash - no need to lookup and we're ok for call caching
+        val response = prepareBackendDescriptor(inputs, attributes, DockerWithHash(dockerImageId.fullName), kvStoreLookupResults.unscoped)
         sendResponseAndStop(response)
       case Failure(failure) => sendFailureAndStop(failure)
     }
@@ -101,38 +123,20 @@ case class JobPreparationActor(executionData: WorkflowExecutionActorData,
       case Some(dockerValue) => handleDockerValue(dockerValue.valueString)
       case None =>
         // If there is no docker attribute at all - we're ok for call caching
-        val response = prepareBackendDescriptor(inputs, attributes, CallCachingEligible)
+        val response = prepareBackendDescriptor(inputs, attributes, NoDocker, kvStoreLookupResults.unscoped)
         sendResponseAndStop(response)
     }
   }
   
-  // Schedule a DockerNoResponseTimeout message to be sent to self
-  private final def newRequestLostTimer = context.system.scheduler.scheduleOnce(dockerNoResponseTimeout, self, DockerNoResponseTimeout)(ec, self)
-  // Schedule a DockerHashRequest to be sent to the docker hashing actor
-  private final def newBackPressureTimer(dockerHashRequest: DockerHashRequest) = context.system.scheduler.scheduleOnce(backpressureWaitTime, dockerHashingActor, dockerHashRequest)(ec, self)
-
-  /*
-   * If we get backpressured, recreate both timers
-   */
-  private def handleDockHashBackPressure(request: DockerHashRequest, data: JobPreparationActorData) = {
-    data.cancelTimers()
-    val requestLostTimer = newRequestLostTimer
-    val backpressureTimer = newBackPressureTimer(data.dockerHashRequest)
-    stay() using data.backpressure(requestLostTimer, backpressureTimer)
-  }
-  
-  private def handleDockerHashSuccess(dockerHashResult: DockerHashResult, data: JobPreparationActorData) = {
-    data.cancelTimers()
-    val dockerValueWithHash = data.dockerHashRequest.dockerImageID.withHash(dockerHashResult).fullName
-    val newRuntimeAttributes = data.attributes updated (RuntimeAttributesKeys.DockerKey, WdlString(dockerValueWithHash))
-    // We had to ask for the docker hash, which means the attribute had a floating tag, we're NOT ok for call caching
-    val response = prepareBackendDescriptor(data.inputs, newRuntimeAttributes, FloatingDockerTagWithHash(dockerValueWithHash))
+  private def handleDockerHashSuccess(dockerHashResult: DockerHashResult, data: JobPreparationHashLookupData) = {
+    val hashValue = data.dockerHashRequest.dockerImageID.withHash(dockerHashResult)
+    val response = prepareBackendDescriptor(data.inputs, data.attributes, DockerWithHash(hashValue.fullName), data.keyLookupResults.unscoped)
     sendResponseAndStop(response)
   }
 
-  private def handleDockerHashFailed(failure: String, data: JobPreparationActorData) = {
-    data.cancelTimers()
-    val response = prepareBackendDescriptor(data.inputs, data.attributes, FloatingDockerTagWithoutHash)
+  private def handleDockerHashFailed(data: JobPreparationHashLookupData) = {
+    val floatingDockerTag = data.dockerHashRequest.dockerImageID.fullName
+    val response = prepareBackendDescriptor(data.inputs, data.attributes, FloatingDockerTagWithoutHash(floatingDockerTag), data.keyLookupResults.unscoped)
     sendResponseAndStop(response)
   }
 
@@ -140,21 +144,29 @@ case class JobPreparationActor(executionData: WorkflowExecutionActorData,
     context.parent ! response
     stay()
   }
-  
+
   private def sendFailureAndStop(failure: Throwable) = {
     sendResponseAndStop(CallPreparationFailed(jobKey, failure))
   }
-  
-  private [preparation] def prepareBackendDescriptor(inputEvaluation: Map[Declaration, WdlValue],
-                               runtimeAttributes: Map[LocallyQualifiedName, WdlValue],
-                               callCachingEligibility: CallCachingEligibility) = {
-    val jobDescriptor = BackendJobDescriptor(workflowDescriptor.backendDescriptor, jobKey, runtimeAttributes, inputEvaluation, callCachingEligibility)
-    BackendJobPreparationSucceeded(jobDescriptor, factory.jobExecutionActorProps(jobDescriptor, initializationData, serviceRegistryActor, backendSingletonActor))
+
+  // 'jobExecutionProps' is broken into a separate function for TestJobPreparationActor to override:
+  private[preparation] def jobExecutionProps(jobDescriptor: BackendJobDescriptor,
+                                             initializationData: Option[BackendInitializationData],
+                                             serviceRegistryActor: ActorRef,
+                                             ioActor: ActorRef,
+                                             backendSingletonActor: Option[ActorRef]) = factory.jobExecutionActorProps(jobDescriptor, initializationData, serviceRegistryActor, ioActor, backendSingletonActor)
+
+  private[preparation] def prepareBackendDescriptor(inputEvaluation: Map[Declaration, WdlValue],
+                                                    runtimeAttributes: Map[LocallyQualifiedName, WdlValue],
+                                                    maybeCallCachingEligible: MaybeCallCachingEligible,
+                                                    prefetchedJobStoreEntries: Map[String, KvResponse]): BackendJobPreparationSucceeded = {
+    val jobDescriptor = BackendJobDescriptor(workflowDescriptor.backendDescriptor, jobKey, runtimeAttributes, inputEvaluation, maybeCallCachingEligible, prefetchedJobStoreEntries)
+    BackendJobPreparationSucceeded(jobDescriptor, jobExecutionProps(jobDescriptor, initializationData, serviceRegistryActor, ioActor, backendSingletonActor))
   }
-  
+
   private [preparation] def prepareRuntimeAttributes(inputEvaluation: Map[Declaration, WdlValue]): Try[Map[LocallyQualifiedName, WdlValue]] = {
     import RuntimeAttributeDefinition.{addDefaultsToAttributes, evaluateRuntimeAttributes}
-    val curriedAddDefaultsToAttributes = addDefaultsToAttributes(factory.runtimeAttributeDefinitions(initializationData), workflowDescriptor.backendDescriptor.workflowOptions) _
+    val curriedAddDefaultsToAttributes = addDefaultsToAttributes(runtimeAttributeDefinitions, workflowDescriptor.backendDescriptor.workflowOptions) _
 
     for {
       unevaluatedRuntimeAttributes <- Try(jobKey.call.task.runtimeAttributes)
@@ -164,52 +176,37 @@ case class JobPreparationActor(executionData: WorkflowExecutionActorData,
 }
 
 object JobPreparationActor {
-  case class JobPreparationActorData(dockerHashRequest: DockerHashRequest,
-                                     inputs: Map[Declaration, WdlValue],
-                                     attributes: Map[LocallyQualifiedName, WdlValue],
-                                     requestLostTimer: Cancellable,
-                                     maxAttempts: Int,
-                                     currentAttempts: Int = 0,
-                                     backpressureTimer: Option[Cancellable] = None
-                                    ) {
-    def cancelTimers(): Unit = {
-      requestLostTimer.cancel()
-      backpressureTimer foreach { _.cancel() }
-      ()
-    }
-    
-    def backpressure(newBackpressureTimer: Cancellable, newRequestLostTimer: Cancellable) = {
-      Option(copy(requestLostTimer = newRequestLostTimer, backpressureTimer = Option(newBackpressureTimer)))
-    }
-    
-    def newAttempt(newBackpressureTimer: Cancellable) = {
-      Option(copy(requestLostTimer = newBackpressureTimer, currentAttempts = currentAttempts + 1))
-    }
-    
-    def maxAttemptsReached = currentAttempts == maxAttempts
-  }
+
+  sealed trait JobPreparationActorData
+  case object JobPreparationActorNoData extends JobPreparationActorData
+  case class JobPreparationKeyLookupData(keyLookups: PartialKeyValueLookups) extends JobPreparationActorData
+  private final case class JobPreparationHashLookupData(keyLookupResults: KeyValueLookupResults,
+                                                        dockerHashRequest: DockerHashRequest,
+                                                        inputs: Map[Declaration, WdlValue],
+                                                        attributes: Map[LocallyQualifiedName, WdlValue]) extends JobPreparationActorData
 
   sealed trait JobPreparationActorState
   case object Idle extends JobPreparationActorState
   case object WaitingForDockerHash extends JobPreparationActorState
-  
-  private case object DockerNoResponseTimeout
+  case object FetchingKeyValueStoreEntries extends JobPreparationActorState
   
   def props(executionData: WorkflowExecutionActorData,
             jobKey: BackendJobDescriptorKey,
             factory: BackendLifecycleActorFactory,
-            dockerHashingActor: ActorRef,
+            workflowDockerLookupActor: ActorRef,
             initializationData: Option[BackendInitializationData],
             serviceRegistryActor: ActorRef,
+            ioActor: ActorRef,
             backendSingletonActor: Option[ActorRef]) = {
     // Note that JobPreparationActor doesn't run on the engine dispatcher as it mostly executes backend-side code
     // (WDL expression evaluation using Backend's expressionLanguageFunctions)
     Props(new JobPreparationActor(executionData,
       jobKey,
       factory,
-      dockerHashingActor,
+      workflowDockerLookupActor = workflowDockerLookupActor,
       initializationData,
-      serviceRegistryActor,
-      backendSingletonActor)).withDispatcher(EngineDispatcher)
+      serviceRegistryActor = serviceRegistryActor,
+      ioActor = ioActor,
+      backendSingletonActor = backendSingletonActor)).withDispatcher(EngineDispatcher)
   }
 }
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/preparation/KeyValueLookups.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/preparation/KeyValueLookups.scala
new file mode 100644
index 000000000..2f3dec446
--- /dev/null
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/preparation/KeyValueLookups.scala
@@ -0,0 +1,24 @@
+package cromwell.engine.workflow.lifecycle.execution.preparation
+
+import cromwell.services.keyvalue.KeyValueServiceActor.{KvResponse, ScopedKey}
+
+/**
+  * Handles the determination of when we know key lookups are successful.
+  */
+private sealed trait KeyValueLookups
+
+private[preparation] final case class PartialKeyValueLookups(responses: Map[ScopedKey, KvResponse], awaiting: Seq[ScopedKey]) {
+  def withResponse(key: ScopedKey, response: KvResponse) = {
+    val newResponses = responses + (key -> response)
+    val newAwaiting = awaiting diff List(key)
+    if (newAwaiting.isEmpty) {
+      KeyValueLookupResults(newResponses)
+    } else {
+      PartialKeyValueLookups(newResponses, newAwaiting)
+    }
+  }
+}
+
+private final case class KeyValueLookupResults(values: Map[ScopedKey, KvResponse]) {
+  def unscoped: Map[String, KvResponse] = values map { case (k, v) => k.key -> v }
+}
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/preparation/SubWorkflowPreparationActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/preparation/SubWorkflowPreparationActor.scala
index f4cbec0f5..11dddea5e 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/preparation/SubWorkflowPreparationActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/preparation/SubWorkflowPreparationActor.scala
@@ -10,8 +10,8 @@ import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor.SubWo
 import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActorData
 import cromwell.engine.workflow.lifecycle.execution.preparation.CallPreparation.{CallPreparationFailed, Start, _}
 import cromwell.engine.workflow.lifecycle.execution.preparation.SubWorkflowPreparationActor.SubWorkflowPreparationSucceeded
-import wdl4s._
-import wdl4s.values.WdlValue
+import wdl4s.wdl._
+import wdl4s.wdl.values.WdlValue
 
 class SubWorkflowPreparationActor(executionData: WorkflowExecutionActorData,
                                    callKey: SubWorkflowKey,
diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStore.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStore.scala
index be17de643..d8ca17ab6 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStore.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStore.scala
@@ -9,6 +9,8 @@ import cromwell.database.sql.SqlConverters._
 import cromwell.database.sql.WorkflowStoreSqlDatabase
 import cromwell.database.sql.tables.WorkflowStoreEntry
 import cromwell.engine.workflow.workflowstore.WorkflowStoreState.StartableState
+import eu.timepit.refined.api.Refined
+import eu.timepit.refined.collection._
 import net.ceedubs.ficus.Ficus._
 
 import scala.concurrent.{ExecutionContext, Future}
@@ -53,11 +55,13 @@ case class SqlWorkflowStore(sqlDatabase: WorkflowStoreSqlDatabase) extends Workf
 
   private def fromWorkflowStoreEntry(workflowStoreEntry: WorkflowStoreEntry): WorkflowToStart = {
     val sources = WorkflowSourceFilesCollection(
-      workflowStoreEntry.workflowDefinition.toRawString,
-      workflowStoreEntry.workflowInputs.toRawString,
-      workflowStoreEntry.workflowOptions.toRawString,
-      workflowStoreEntry.customLabels.toRawString,
-      workflowStoreEntry.importsZip.toBytesOption
+      workflowSource = workflowStoreEntry.workflowDefinition.toRawString,
+      workflowType = workflowStoreEntry.workflowType,
+      workflowTypeVersion = workflowStoreEntry.workflowTypeVersion,
+      inputsJson = workflowStoreEntry.workflowInputs.toRawString,
+      workflowOptionsJson = workflowStoreEntry.workflowOptions.toRawString,
+      labelsJson = workflowStoreEntry.customLabels.toRawString,
+      importsFile = workflowStoreEntry.importsZip.toBytesOption
     )
     WorkflowToStart(
       WorkflowId.fromString(workflowStoreEntry.workflowExecutionUuid),
@@ -66,15 +70,20 @@ case class SqlWorkflowStore(sqlDatabase: WorkflowStoreSqlDatabase) extends Workf
   }
 
   private def toWorkflowStoreEntry(workflowSourceFiles: WorkflowSourceFilesCollection): WorkflowStoreEntry = {
+    import eu.timepit.refined._
+    val nonEmptyJsonString: String Refined NonEmpty  = refineMV[NonEmpty]("{}")
+
     WorkflowStoreEntry(
       workflowExecutionUuid = WorkflowId.randomId().toString,
-      workflowDefinition = workflowSourceFiles.wdlSource.toClob,
-      workflowInputs = workflowSourceFiles.inputsJson.toClob,
-      workflowOptions = workflowSourceFiles.workflowOptionsJson.toClob,
-      customLabels = workflowSourceFiles.labelsJson.toClob,
+      workflowDefinition = workflowSourceFiles.workflowSource.toClobOption,
+      workflowType = workflowSourceFiles.workflowType,
+      workflowTypeVersion = workflowSourceFiles.workflowTypeVersion,
+      workflowInputs = workflowSourceFiles.inputsJson.toClobOption,
+      workflowOptions = workflowSourceFiles.workflowOptionsJson.toClobOption,
+      customLabels = workflowSourceFiles.labelsJson.toClob(default = nonEmptyJsonString),
       workflowState = WorkflowStoreState.Submitted.toString,
       submissionTime = OffsetDateTime.now.toSystemTimestamp,
-      importsZip = workflowSourceFiles.importsZipFileOption.toBlob
+      importsZip = workflowSourceFiles.importsZipFileOption.toBlobOption
     )
   }
 
diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreActor.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreActor.scala
index bebd40823..dad70fb96 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreActor.scala
@@ -4,14 +4,18 @@ import akka.actor.{Actor, ActorLogging, ActorRef, Props}
 import cats.data.NonEmptyList
 import cromwell.core._
 import cromwell.core.Dispatcher.EngineDispatcher
+import cromwell.database.sql.SqlDatabase
+import cromwell.util.GracefulShutdownHelper
+import cromwell.util.GracefulShutdownHelper.ShutdownCommand
 
-final case class WorkflowStoreActor(store: WorkflowStore, serviceRegistryActor: ActorRef) extends Actor with ActorLogging {
+final case class WorkflowStoreActor private(store: WorkflowStore, serviceRegistryActor: ActorRef, database: SqlDatabase) extends Actor with ActorLogging with GracefulShutdownHelper {
   import WorkflowStoreActor._
 
-  lazy val workflowStoreSubmitActor: ActorRef = context.actorOf(WorkflowStoreSubmitActor.props(store, serviceRegistryActor))
-  lazy val workflowStoreEngineActor: ActorRef = context.actorOf(WorkflowStoreEngineActor.props(store, serviceRegistryActor))
+  lazy val workflowStoreSubmitActor: ActorRef = context.actorOf(WorkflowStoreSubmitActor.props(store, serviceRegistryActor), "WorkflowStoreSubmitActor")
+  lazy val workflowStoreEngineActor: ActorRef = context.actorOf(WorkflowStoreEngineActor.props(store, serviceRegistryActor, database), "WorkflowStoreEngineActor")
 
   override def receive = {
+    case ShutdownCommand => waitForActorsAndShutdown(NonEmptyList.of(workflowStoreSubmitActor))
     case cmd: WorkflowStoreActorSubmitCommand => workflowStoreSubmitActor forward cmd
     case cmd: WorkflowStoreActorEngineCommand => workflowStoreEngineActor forward cmd
   }
@@ -20,7 +24,6 @@ final case class WorkflowStoreActor(store: WorkflowStore, serviceRegistryActor:
 object WorkflowStoreActor {
   sealed trait WorkflowStoreActorEngineCommand
   final case class FetchRunnableWorkflows(n: Int) extends WorkflowStoreActorEngineCommand
-  final case class RemoveWorkflow(id: WorkflowId) extends WorkflowStoreActorEngineCommand
   final case class AbortWorkflow(id: WorkflowId, manager: ActorRef) extends WorkflowStoreActorEngineCommand
   case object InitializerCommand extends WorkflowStoreActorEngineCommand
   case object WorkDone extends WorkflowStoreActorEngineCommand
@@ -29,7 +32,7 @@ object WorkflowStoreActor {
   final case class SubmitWorkflow(source: WorkflowSourceFilesCollection) extends WorkflowStoreActorSubmitCommand
   final case class BatchSubmitWorkflows(sources: NonEmptyList[WorkflowSourceFilesCollection]) extends WorkflowStoreActorSubmitCommand
 
-  def props(workflowStoreDatabase: WorkflowStore, serviceRegistryActor: ActorRef) = {
-    Props(WorkflowStoreActor(workflowStoreDatabase, serviceRegistryActor)).withDispatcher(EngineDispatcher)
+  def props(workflowStoreDatabase: WorkflowStore, serviceRegistryActor: ActorRef, database: SqlDatabase) = {
+    Props(WorkflowStoreActor(workflowStoreDatabase, serviceRegistryActor, database)).withDispatcher(EngineDispatcher)
   }
 }
diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreEngineActor.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreEngineActor.scala
index b35346581..e2a7ac9a1 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreEngineActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreEngineActor.scala
@@ -3,19 +3,19 @@ package cromwell.engine.workflow.workflowstore
 import akka.actor.{ActorLogging, ActorRef, LoggingFSM, Props}
 import cats.data.NonEmptyList
 import cromwell.core.Dispatcher._
+import cromwell.core.WorkflowId
+import cromwell.database.sql.SqlDatabase
 import cromwell.engine.workflow.WorkflowManagerActor
 import cromwell.engine.workflow.WorkflowManagerActor.WorkflowNotFoundException
-import cromwell.engine.workflow.workflowstore.WorkflowStoreEngineActor.WorkflowStoreActorState
-import cromwell.engine.workflow.workflowstore.WorkflowStoreState.StartableState
-import WorkflowStoreEngineActor._
-import cromwell.core.WorkflowId
 import cromwell.engine.workflow.workflowstore.WorkflowStoreActor._
+import cromwell.engine.workflow.workflowstore.WorkflowStoreEngineActor.{WorkflowStoreActorState, _}
+import cromwell.engine.workflow.workflowstore.WorkflowStoreState.StartableState
 import org.apache.commons.lang3.exception.ExceptionUtils
 
 import scala.concurrent.{ExecutionContext, Future}
 import scala.util.{Failure, Success}
 
-final case class WorkflowStoreEngineActor(store: WorkflowStore, serviceRegistryActor: ActorRef)
+final case class WorkflowStoreEngineActor private(store: WorkflowStore, serviceRegistryActor: ActorRef, database: SqlDatabase)
   extends LoggingFSM[WorkflowStoreActorState, WorkflowStoreActorData] with ActorLogging {
 
   implicit val ec: ExecutionContext = context.dispatcher
@@ -65,7 +65,7 @@ final case class WorkflowStoreEngineActor(store: WorkflowStore, serviceRegistryA
 
   private def startNewWork(command: WorkflowStoreActorEngineCommand, sndr: ActorRef, nextData: WorkflowStoreActorData) = {
     val work: Future[Any] = command match {
-      case cmd @ FetchRunnableWorkflows(n) =>
+      case FetchRunnableWorkflows(n) =>
         newWorkflowMessage(n) map { nwm =>
           nwm match {
             case NewWorkflowsToStart(workflows) => log.info("{} new workflows fetched", workflows.toList.size)
@@ -74,22 +74,22 @@ final case class WorkflowStoreEngineActor(store: WorkflowStore, serviceRegistryA
           }
           sndr ! nwm
         }
-      case cmd @ AbortWorkflow(id, manager) =>
+      case AbortWorkflow(id, manager) =>
         store.remove(id) map { removed =>
           if (removed) {
-            log.debug(s"Workflow $id aborted and removed from the workflow store.")
             manager ! WorkflowManagerActor.AbortWorkflowCommand(id, sndr)
+            log.debug(s"Workflow $id removed from the workflow store, abort requested.")
           } else {
             sndr ! WorkflowAbortFailed(id, new WorkflowNotFoundException(s"Couldn't abort $id because no workflow with that ID is in progress"))
           }
-        }
-      case cmd @ RemoveWorkflow(id) =>
-        store.remove(id) map { removed =>
-          if (removed) {
-            log.debug("Workflow {} removed from store successfully.", id)
-          } else {
-            log.warning(s"Attempted to remove ID {} from the WorkflowStore but it didn't exist", id)
-          }
+        } recover {
+          case t =>
+            val message = s"Error aborting workflow $id: could not remove from workflow store"
+            log.error(t, message)
+            // A generic exception type like RuntimeException will produce a 500 at the API layer, which seems appropriate
+            // given we don't know much about what went wrong here.  `t.getMessage` so the cause propagates to the client.
+            val e = new RuntimeException(s"$message: ${t.getMessage}", t)
+            sndr ! WorkflowAbortFailed(id, e)
         }
       case oops =>
         log.error("Unexpected type of start work command: {}", oops.getClass.getSimpleName)
@@ -129,20 +129,26 @@ final case class WorkflowStoreEngineActor(store: WorkflowStore, serviceRegistryA
     runnableWorkflows map {
       case x :: xs => NewWorkflowsToStart(NonEmptyList.of(x, xs: _*))
       case _ => NoNewWorkflowsToStart
+    } recover {
+      case e =>
+        // Log the error but return a successful Future so as not to hang future workflow store polls.
+        log.error(e, "Error trying to fetch new workflows")
+        NoNewWorkflowsToStart
     }
   }
 }
 
 object WorkflowStoreEngineActor {
-  def props(workflowStoreDatabase: WorkflowStore, serviceRegistryActor: ActorRef) = {
-    Props(WorkflowStoreEngineActor(workflowStoreDatabase, serviceRegistryActor)).withDispatcher(EngineDispatcher)
+  def props(workflowStoreDatabase: WorkflowStore, serviceRegistryActor: ActorRef, database: SqlDatabase) = {
+    Props(WorkflowStoreEngineActor(workflowStoreDatabase, serviceRegistryActor, database)).withDispatcher(EngineDispatcher)
   }
 
   sealed trait WorkflowStoreEngineActorResponse
   case object NoNewWorkflowsToStart extends WorkflowStoreEngineActorResponse
   final case class NewWorkflowsToStart(workflows: NonEmptyList[WorkflowToStart]) extends WorkflowStoreEngineActorResponse
-  final case class WorkflowAborted(workflowId: WorkflowId) extends WorkflowStoreEngineActorResponse
-  final case class WorkflowAbortFailed(workflowId: WorkflowId, reason: Throwable) extends WorkflowStoreEngineActorResponse
+  sealed abstract class WorkflowStoreEngineAbortResponse extends WorkflowStoreEngineActorResponse
+  final case class WorkflowAborted(workflowId: WorkflowId) extends WorkflowStoreEngineAbortResponse
+  final case class WorkflowAbortFailed(workflowId: WorkflowId, reason: Throwable) extends WorkflowStoreEngineAbortResponse
 
 
   final case class WorkflowStoreActorCommandWithSender(command: WorkflowStoreActorEngineCommand, sender: ActorRef)
diff --git a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreSubmitActor.scala b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreSubmitActor.scala
index b9381863e..67e06b4df 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreSubmitActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/workflowstore/WorkflowStoreSubmitActor.scala
@@ -4,37 +4,67 @@ import java.time.OffsetDateTime
 
 import akka.actor.{Actor, ActorLogging, ActorRef, Props}
 import cats.data.NonEmptyList
+import cats.instances.future._
+import cats.instances.list._
+import cats.syntax.traverse._
 import cromwell.core.Dispatcher._
 import cromwell.core._
+import cromwell.engine.workflow.lifecycle.execution.WorkflowMetadataHelper
 import cromwell.engine.workflow.workflowstore.WorkflowStoreActor._
-import cromwell.engine.workflow.workflowstore.WorkflowStoreSubmitActor.{WorkflowSubmittedToStore, WorkflowsBatchSubmittedToStore}
-import cromwell.services.metadata.{MetadataEvent, MetadataKey, MetadataValue}
+import cromwell.engine.workflow.workflowstore.WorkflowStoreSubmitActor.{WorkflowSubmitFailed, WorkflowSubmittedToStore, WorkflowsBatchSubmittedToStore}
 import cromwell.services.metadata.MetadataService.PutMetadataAction
+import cromwell.services.metadata.{MetadataEvent, MetadataKey, MetadataValue}
 
 import scala.concurrent.{ExecutionContext, Future}
 import scala.util.{Failure, Success}
 
-final case class WorkflowStoreSubmitActor(store: WorkflowStore, serviceRegistryActor: ActorRef) extends Actor with ActorLogging {
+final case class WorkflowStoreSubmitActor(store: WorkflowStore, serviceRegistryActor: ActorRef) extends Actor with ActorLogging with WorkflowMetadataHelper with MonitoringCompanionHelper {
   implicit val ec: ExecutionContext = context.dispatcher
 
-  override def receive = {
+  val workflowStoreReceive: Receive = {
     case cmd: SubmitWorkflow =>
+      addWork()
       val sndr = sender()
-      storeWorkflowSources(NonEmptyList.of(cmd.source)) foreach { ids =>
-        val id = ids.head
-        registerSubmissionWithMetadataService(id, cmd.source)
-        sndr ! WorkflowSubmittedToStore(id)
-        log.info("Workflow {} submitted.", id)
+
+      val futureId = for {
+        ids <- storeWorkflowSources(NonEmptyList.of(cmd.source))
+        id = ids.head
+        _ <- registerSubmissionWithMetadataService(id, cmd.source)
+      } yield id
+
+      futureId onComplete {
+        case Success(id) =>
+          log.info("Workflow {} submitted.", id)
+          sndr ! WorkflowSubmittedToStore(id)
+          removeWork()
+        case Failure(throwable) =>
+          log.error("Workflow {} submit failed.", throwable)
+          sndr ! WorkflowSubmitFailed(throwable)
+          removeWork()
       }
+
     case cmd: BatchSubmitWorkflows =>
+      addWork()
       val sndr = sender()
-      storeWorkflowSources(cmd.sources) foreach { ids =>
-        val assignedSources = ids.toList.zip(cmd.sources.toList)
-        assignedSources foreach { case (id, sourceFiles) => registerSubmissionWithMetadataService(id, sourceFiles) }
-        sndr ! WorkflowsBatchSubmittedToStore(ids)
-        log.info("Workflows {} submitted.", ids.toList.mkString(", "))
+
+      val futureIds = for {
+        ids <- storeWorkflowSources(cmd.sources)
+        _ <- (ids.toList zip cmd.sources.toList) traverse (registerSubmissionWithMetadataService _).tupled
+      } yield ids
+
+      futureIds onComplete {
+        case Success(ids) =>
+          log.info("Workflows {} submitted.", ids.toList.mkString(", "))
+          sndr ! WorkflowsBatchSubmittedToStore(ids)
+          removeWork()
+        case Failure(throwable) =>
+          log.error("Workflow {} submit failed.", throwable)
+          sndr ! WorkflowSubmitFailed(throwable)
+          removeWork()
       }
   }
+  
+  override def receive = workflowStoreReceive.orElse(monitoringReceive)
 
   private def storeWorkflowSources(sources: NonEmptyList[WorkflowSourceFilesCollection]): Future[NonEmptyList[WorkflowId]] = {
     for {
@@ -73,19 +103,30 @@ final case class WorkflowStoreSubmitActor(store: WorkflowStore, serviceRegistryA
   /**
     * Takes the workflow id and sends it over to the metadata service w/ default empty values for inputs/outputs
     */
-  private def registerSubmissionWithMetadataService(id: WorkflowId, originalSourceFiles: WorkflowSourceFilesCollection): Unit = {
-    processSource(_.clearEncryptedValues)(originalSourceFiles) foreach { sourceFiles =>
-      val submissionEvents = List(
+  private def registerSubmissionWithMetadataService(
+      id: WorkflowId,
+      originalSourceFiles: WorkflowSourceFilesCollection): Future[Unit] = {
+    processSource(_.clearEncryptedValues)(originalSourceFiles) map { sourceFiles =>
+      val submissionEvents: List[MetadataEvent] = List(
         MetadataEvent(MetadataKey(id, None, WorkflowMetadataKeys.SubmissionTime), MetadataValue(OffsetDateTime.now.toString)),
         MetadataEvent.empty(MetadataKey(id, None, WorkflowMetadataKeys.Inputs)),
         MetadataEvent.empty(MetadataKey(id, None, WorkflowMetadataKeys.Outputs)),
+        MetadataEvent(MetadataKey(id, None, WorkflowMetadataKeys.Status), MetadataValue(WorkflowSubmitted)),
 
-        MetadataEvent(MetadataKey(id, None, WorkflowMetadataKeys.SubmissionSection, WorkflowMetadataKeys.SubmissionSection_Workflow), MetadataValue(sourceFiles.wdlSource)),
+        MetadataEvent(MetadataKey(id, None, WorkflowMetadataKeys.SubmissionSection, WorkflowMetadataKeys.SubmissionSection_Workflow), MetadataValue(sourceFiles.workflowSource)),
         MetadataEvent(MetadataKey(id, None, WorkflowMetadataKeys.SubmissionSection, WorkflowMetadataKeys.SubmissionSection_Inputs), MetadataValue(sourceFiles.inputsJson)),
-        MetadataEvent(MetadataKey(id, None, WorkflowMetadataKeys.SubmissionSection, WorkflowMetadataKeys.SubmissionSection_Options), MetadataValue(sourceFiles.workflowOptionsJson))
+        MetadataEvent(MetadataKey(id, None, WorkflowMetadataKeys.SubmissionSection, WorkflowMetadataKeys.SubmissionSection_Options), MetadataValue(sourceFiles.workflowOptionsJson)),
+        MetadataEvent(MetadataKey(id, None, WorkflowMetadataKeys.SubmissionSection, WorkflowMetadataKeys.SubmissionSection_Labels), MetadataValue(sourceFiles.labelsJson))
+      )
+
+      // Don't publish metadata for either workflow type or workflow type version if not defined.
+      val workflowTypeAndVersionEvents: List[Option[MetadataEvent]] = List(
+        sourceFiles.workflowType map { wt => MetadataEvent(MetadataKey(id, None, WorkflowMetadataKeys.SubmissionSection, WorkflowMetadataKeys.SubmissionSection_WorkflowType), MetadataValue(wt)) },
+        sourceFiles.workflowTypeVersion map { wtv => MetadataEvent(MetadataKey(id, None, WorkflowMetadataKeys.SubmissionSection, WorkflowMetadataKeys.SubmissionSection_WorkflowTypeVersion), MetadataValue(wtv)) }
       )
 
-      serviceRegistryActor ! PutMetadataAction(submissionEvents)
+      serviceRegistryActor ! PutMetadataAction(submissionEvents ++ workflowTypeAndVersionEvents.flatten)
+      ()
     }
   }
 }
@@ -98,5 +139,6 @@ object WorkflowStoreSubmitActor {
   sealed trait WorkflowStoreSubmitActorResponse
   final case class WorkflowSubmittedToStore(workflowId: WorkflowId) extends WorkflowStoreSubmitActorResponse
   final case class WorkflowsBatchSubmittedToStore(workflowIds: NonEmptyList[WorkflowId]) extends WorkflowStoreSubmitActorResponse
-}
 
+  final case class WorkflowSubmitFailed(throwable: Throwable) extends WorkflowStoreSubmitActorResponse
+}
diff --git a/engine/src/main/scala/cromwell/jobstore/EmptyJobStoreActor.scala b/engine/src/main/scala/cromwell/jobstore/EmptyJobStoreActor.scala
index a2eb585a3..598f65085 100644
--- a/engine/src/main/scala/cromwell/jobstore/EmptyJobStoreActor.scala
+++ b/engine/src/main/scala/cromwell/jobstore/EmptyJobStoreActor.scala
@@ -2,6 +2,7 @@ package cromwell.jobstore
 
 import akka.actor.{Actor, Props}
 import cromwell.jobstore.JobStoreActor._
+import cromwell.core.Dispatcher.EngineDispatcher
 
 class EmptyJobStoreActor extends Actor {
   override def receive: Receive = {
@@ -11,5 +12,5 @@ class EmptyJobStoreActor extends Actor {
 }
 
 object EmptyJobStoreActor {
-  def props: Props = Props(new EmptyJobStoreActor())
+  def props: Props = Props(new EmptyJobStoreActor()).withDispatcher(EngineDispatcher)
 }
diff --git a/engine/src/main/scala/cromwell/jobstore/JobStore.scala b/engine/src/main/scala/cromwell/jobstore/JobStore.scala
index eba177251..ad57bcaaf 100644
--- a/engine/src/main/scala/cromwell/jobstore/JobStore.scala
+++ b/engine/src/main/scala/cromwell/jobstore/JobStore.scala
@@ -1,11 +1,18 @@
 package cromwell.jobstore
 
 import cromwell.core.WorkflowId
-import wdl4s.TaskOutput
+import cromwell.jobstore.JobStore.{JobCompletion, WorkflowCompletion}
+import wdl4s.wdl.TaskOutput
 
 import scala.concurrent.{ExecutionContext, Future}
 
 trait JobStore {
-  def writeToDatabase(jobCompletions: Map[JobStoreKey, JobResult], workflowCompletions: List[WorkflowId])(implicit ec: ExecutionContext): Future[Unit]
+  def writeToDatabase(workflowCompletions: Seq[WorkflowCompletion], jobCompletions: Seq[JobCompletion], batchSize: Int)(implicit ec: ExecutionContext): Future[Unit]
   def readJobResult(jobStoreKey: JobStoreKey, taskOutputs: Seq[TaskOutput])(implicit ec: ExecutionContext): Future[Option[JobResult]]
 }
+
+object JobStore {
+  sealed trait Completion
+  case class WorkflowCompletion(workflowId: WorkflowId) extends Completion
+  case class JobCompletion(key: JobStoreKey, result: JobResult) extends Completion
+}
diff --git a/engine/src/main/scala/cromwell/jobstore/JobStoreActor.scala b/engine/src/main/scala/cromwell/jobstore/JobStoreActor.scala
index e1e55ac19..73b031dfd 100644
--- a/engine/src/main/scala/cromwell/jobstore/JobStoreActor.scala
+++ b/engine/src/main/scala/cromwell/jobstore/JobStoreActor.scala
@@ -1,21 +1,28 @@
 package cromwell.jobstore
 
-import akka.actor.{Actor, Props}
+import akka.actor.{Actor, ActorLogging, Props}
+import cats.data.NonEmptyList
 import cromwell.core.Dispatcher.EngineDispatcher
 import cromwell.core.WorkflowId
-import cromwell.jobstore.JobStoreActor.{JobStoreReaderCommand, JobStoreWriterCommand}
-import wdl4s.TaskOutput
+import cromwell.jobstore.JobStore.{Completion, JobCompletion, WorkflowCompletion}
+import cromwell.util.GracefulShutdownHelper
+import cromwell.util.GracefulShutdownHelper.ShutdownCommand
+import wdl4s.wdl.TaskOutput
 
+import scala.concurrent.duration._
+import scala.language.postfixOps
 /**
   * Joins the service registry API to the JobStoreReaderActor and JobStoreWriterActor.
   *
   * This level of indirection is a tiny bit awkward but allows the database to be injected.
   */
-class JobStoreActor(database: JobStore) extends Actor {
-  val jobStoreWriterActor = context.actorOf(JobStoreWriterActor.props(database))
-  val jobStoreReaderActor = context.actorOf(JobStoreReaderActor.props(database))
+class JobStoreActor(jobStore: JobStore, dbBatchSize: Int, dbFlushRate: FiniteDuration) extends Actor with ActorLogging with GracefulShutdownHelper {
+  import JobStoreActor._
+  val jobStoreWriterActor = context.actorOf(JobStoreWriterActor.props(jobStore, dbBatchSize, dbFlushRate), "JobStoreWriterActor")
+  val jobStoreReaderActor = context.actorOf(JobStoreReaderActor.props(jobStore), "JobStoreReaderActor")
 
   override def receive: Receive = {
+    case ShutdownCommand => waitForActorsAndShutdown(NonEmptyList.of(jobStoreWriterActor))
     case command: JobStoreWriterCommand => jobStoreWriterActor.tell(command, sender())
     case command: JobStoreReaderCommand => jobStoreReaderActor.tell(command, sender())
   }
@@ -24,9 +31,15 @@ class JobStoreActor(database: JobStore) extends Actor {
 object JobStoreActor {
   sealed trait JobStoreCommand
 
-  sealed trait JobStoreWriterCommand extends JobStoreCommand
-  case class RegisterJobCompleted(jobKey: JobStoreKey, jobResult: JobResult) extends JobStoreWriterCommand
-  case class RegisterWorkflowCompleted(workflowId: WorkflowId) extends JobStoreWriterCommand
+  sealed trait JobStoreWriterCommand extends JobStoreCommand {
+    def completion: Completion
+  }
+  case class RegisterJobCompleted(jobKey: JobStoreKey, jobResult: JobResult) extends JobStoreWriterCommand {
+    override def completion = JobCompletion(jobKey, jobResult)
+  }
+  case class RegisterWorkflowCompleted(workflowId: WorkflowId) extends JobStoreWriterCommand {
+    override def completion = WorkflowCompletion(workflowId)
+  }
 
   sealed trait JobStoreWriterResponse
   case class JobStoreWriteSuccess(originalCommand: JobStoreWriterCommand) extends JobStoreWriterResponse
@@ -51,5 +64,11 @@ object JobStoreActor {
 
   case class JobStoreReadFailure(reason: Throwable) extends JobStoreReaderResponse
 
-  def props(database: JobStore) = Props(new JobStoreActor(database)).withDispatcher(EngineDispatcher)
+  def props(database: JobStore) = Props(new JobStoreActor(database, dbBatchSize, dbFlushRate)).withDispatcher(EngineDispatcher)
+
+  val dbFlushRate = 1 second
+
+  // `dbBatchSize` applies to simpletons only.  Batching job store entry writes while returning the inserted IDs works at the
+  // Slick API level, but didn't actually batch the SQL.  Unfortunately these IDs are required to assign into the simpletons.
+  val dbBatchSize = 1000
 }
diff --git a/engine/src/main/scala/cromwell/jobstore/JobStoreWriterActor.scala b/engine/src/main/scala/cromwell/jobstore/JobStoreWriterActor.scala
index 70d0329d3..bb91ae81e 100644
--- a/engine/src/main/scala/cromwell/jobstore/JobStoreWriterActor.scala
+++ b/engine/src/main/scala/cromwell/jobstore/JobStoreWriterActor.scala
@@ -1,96 +1,69 @@
 package cromwell.jobstore
 
-import akka.actor.{ActorRef, LoggingFSM, Props}
-import cromwell.jobstore.JobStoreActor._
+import akka.actor.{LoggingFSM, Props}
 import cromwell.core.Dispatcher.EngineDispatcher
+import cromwell.core.actor.BatchingDbWriter._
+import cromwell.core.actor.{BatchingDbWriter, BatchingDbWriterActor}
+import cromwell.jobstore.JobStore.{JobCompletion, WorkflowCompletion}
+import cromwell.jobstore.JobStoreActor._
 
+import scala.concurrent.duration._
+import scala.language.postfixOps
 import scala.util.{Failure, Success}
 
-/**
-  * Singleton actor to coordinate writing job statuses to the database.
-  *
-  * State: Represents an actor either doing nothing, or currently writing to the database
-  * Data: If currently writing, the actor stores pending updates in the data. When one write completes, any further writes are written
-  */
-case class JobStoreWriterActor(jsd: JobStore) extends LoggingFSM[JobStoreWriterState, JobStoreWriterData] {
-
-  implicit val ec = context.dispatcher
-
-  startWith(Pending, JobStoreWriterData.empty)
 
-  when(Pending) {
-    case Event(command: JobStoreWriterCommand, stateData) =>
-      val newData = writeNextOperationToDatabase(stateData.withNewOperation(sender, command))
-      goto(WritingToDatabase) using newData
-  }
+case class JobStoreWriterActor(jsd: JobStore, dbBatchSize: Int, override val dbFlushRate: FiniteDuration) extends LoggingFSM[BatchingDbWriterState, BatchingDbWriter.BatchingDbWriterData] with BatchingDbWriterActor {
 
-  when(WritingToDatabase) {
-    case Event(command: JobStoreWriterCommand, stateData) =>
-      stay using stateData.withNewOperation(sender, command)
-    case Event(WriteComplete, stateData) =>
-      val newData = writeNextOperationToDatabase(stateData)
-      goto(if (newData.isEmpty) Pending else WritingToDatabase) using newData
-  }
+  implicit val ec = context.dispatcher
 
-  whenUnhandled {
-    case Event(someMessage, stateData) =>
-      log.error(s"JobStoreWriter: Unexpected message received in state $stateName: $someMessage")
-      stay()
-  }
+  startWith(WaitingToWrite, NoData)
 
-  onTransition {
-    case (oldState, newState) =>
-      log.debug(s"Transitioning from $oldState to $newState")
+  when(WaitingToWrite) {
+    case Event(command: JobStoreWriterCommand, curData) =>
+      curData.addData(CommandAndReplyTo(command, sender)) match {
+        case newData: HasData[_] if newData.length >= dbBatchSize => goto(WritingToDb) using newData
+        case newData => stay() using newData
+      }
+    case Event(ScheduledFlushToDb, curData) =>
+      log.debug("Initiating periodic job store flush to DB")
+      goto(WritingToDb) using curData
   }
 
-  def writeNextOperationToDatabase(data: JobStoreWriterData): JobStoreWriterData = {
-
-    val newData = data.rolledOver
-
-    val workflowCompletions = newData.currentOperation collect {
-      case (_, RegisterWorkflowCompleted(wfid)) =>  wfid
-    }
-
-    val jobCompletions = newData.currentOperation collect {
-      case (_, RegisterJobCompleted(jobStoreKey, jobResult)) if !workflowCompletions.contains(jobStoreKey.workflowId) => (jobStoreKey, jobResult)
-    }
-
-    if (!(workflowCompletions.isEmpty && jobCompletions.isEmpty)) {
-      jsd.writeToDatabase(jobCompletions.toMap, workflowCompletions) onComplete {
-        case Success(_) =>
-          newData.currentOperation foreach { case (actor, message) =>
-            val msg = JobStoreWriteSuccess(message)
-            actor ! msg
-          }
-          self ! WriteComplete
-        case Failure(reason) =>
-          log.error(s"Failed to write to database: $reason")
-          newData.currentOperation foreach { case (actor, message) => actor ! JobStoreWriteFailure(reason) }
-          self ! WriteComplete
+  when(WritingToDb) {
+    case Event(ScheduledFlushToDb, _) => stay
+    case Event(command: JobStoreWriterCommand, curData) => stay using curData.addData(CommandAndReplyTo(command, sender))
+    case Event(FlushBatchToDb, NoData) =>
+      log.debug("Attempted job store flush to DB but had nothing to write")
+      goto(WaitingToWrite)
+    case Event(FlushBatchToDb, HasData(data)) =>
+      log.debug("Flushing {} job store commands to the DB", data.length)
+      val completions = data.toVector.collect({ case CommandAndReplyTo(c: JobStoreWriterCommand, _) => c.completion })
+
+      if (completions.nonEmpty) {
+        val workflowCompletions = completions collect { case w: WorkflowCompletion => w }
+        val completedWorkflowIds = workflowCompletions map { _.workflowId } toSet
+        // Filter job completions that also have a corresponding workflow completion; these would just be
+        // immediately deleted anyway.
+        val jobCompletions = completions.toList collect { case j: JobCompletion if !completedWorkflowIds.contains(j.key.workflowId) => j }
+
+        jsd.writeToDatabase(workflowCompletions, jobCompletions, dbBatchSize) onComplete {
+          case Success(_) =>
+            data map { case CommandAndReplyTo(c: JobStoreWriterCommand, r) => r ! JobStoreWriteSuccess(c) }
+            self ! DbWriteComplete
+          case Failure(regerts) =>
+            log.error("Failed to properly job store entries to database", regerts)
+            data map { case CommandAndReplyTo(_, r) => r ! JobStoreWriteFailure(regerts) }
+            self ! DbWriteComplete
+        }
       }
-    }
-
-    newData
+      stay using NoData
+    case Event(DbWriteComplete, _) =>
+      log.debug("Flush of job store commands complete")
+      goto(WaitingToWrite)
   }
 }
 
 object JobStoreWriterActor {
-  def props(jobStoreDatabase: JobStore): Props = Props(new JobStoreWriterActor(jobStoreDatabase)).withDispatcher(EngineDispatcher)
-}
-
-object JobStoreWriterData {
-  def empty = JobStoreWriterData(List.empty, List.empty)
-}
 
-case class JobStoreWriterData(currentOperation: List[(ActorRef, JobStoreWriterCommand)], nextOperation: List[(ActorRef, JobStoreWriterCommand)]) {
-  def isEmpty = nextOperation.isEmpty && currentOperation.isEmpty
-  def withNewOperation(sender: ActorRef, command: JobStoreWriterCommand) = this.copy(nextOperation = this.nextOperation :+ ((sender, command)))
-  def rolledOver = JobStoreWriterData(this.nextOperation, List.empty)
+  def props(jobStoreDatabase: JobStore, dbBatchSize: Int, dbFlushRate: FiniteDuration): Props = Props(new JobStoreWriterActor(jobStoreDatabase, dbBatchSize, dbFlushRate)).withDispatcher(EngineDispatcher)
 }
-
-sealed trait JobStoreWriterState
-case object Pending extends JobStoreWriterState
-case object WritingToDatabase extends JobStoreWriterState
-
-sealed trait JobStoreWriterInternalMessage
-case object WriteComplete extends JobStoreWriterInternalMessage
diff --git a/engine/src/main/scala/cromwell/jobstore/SqlJobStore.scala b/engine/src/main/scala/cromwell/jobstore/SqlJobStore.scala
index de7eef88a..37504beb0 100644
--- a/engine/src/main/scala/cromwell/jobstore/SqlJobStore.scala
+++ b/engine/src/main/scala/cromwell/jobstore/SqlJobStore.scala
@@ -1,30 +1,40 @@
 package cromwell.jobstore
 
+import cats.instances.future._
+import cats.instances.list._
+import cats.syntax.traverse._
+
 import cromwell.Simpletons._
 import cromwell.backend.async.JobAlreadyFailedInJobStore
 import cromwell.core.ExecutionIndex._
-import cromwell.core.WorkflowId
 import cromwell.core.simpleton.WdlValueBuilder
 import cromwell.core.simpleton.WdlValueSimpleton._
-import cromwell.database.sql.JobStoreSqlDatabase
+import cromwell.database.sql.SqlConverters._
+import cromwell.database.sql.SqlDatabase
 import cromwell.database.sql.joins.JobStoreJoin
 import cromwell.database.sql.tables.{JobStoreEntry, JobStoreSimpletonEntry}
-import wdl4s.TaskOutput
+import cromwell.jobstore.JobStore.{JobCompletion, WorkflowCompletion}
+import org.slf4j.LoggerFactory
+import wdl4s.wdl.TaskOutput
 
 import scala.concurrent.{ExecutionContext, Future}
-import cromwell.database.sql.SqlConverters._
 
-class SqlJobStore(sqlDatabase: JobStoreSqlDatabase) extends JobStore {
-  override def writeToDatabase(jobCompletions: Map[JobStoreKey, JobResult], workflowCompletions: List[WorkflowId])(implicit ec: ExecutionContext): Future[Unit] = {
+class SqlJobStore(sqlDatabase: SqlDatabase) extends JobStore {
+  val log = LoggerFactory.getLogger(classOf[SqlJobStore])
+
+  override def writeToDatabase(workflowCompletions: Seq[WorkflowCompletion], jobCompletions: Seq[JobCompletion], batchSize: Int)(implicit ec: ExecutionContext): Future[Unit] = {
+    val completedWorkflowIds = workflowCompletions.toList.map(_.workflowId.toString)
     for {
-      _ <- sqlDatabase.addJobStores(jobCompletions.toSeq map toDatabase)
-      _ <- sqlDatabase.removeJobStores(workflowCompletions.map(_.toString))
+      _ <- sqlDatabase.addJobStores(jobCompletions map toDatabase, batchSize)
+      _ <- completedWorkflowIds traverse sqlDatabase.removeWorkflowStoreEntry
+      _ <- completedWorkflowIds traverse sqlDatabase.removeDockerHashStoreEntries
+      _ <- sqlDatabase.removeJobStores(completedWorkflowIds)
     } yield ()
   }
 
-  private def toDatabase(jobCompletion: (JobStoreKey, JobResult)): JobStoreJoin = {
+  private def toDatabase(jobCompletion: JobCompletion): JobStoreJoin = {
     jobCompletion match {
-      case (key, JobResultSuccess(returnCode, jobOutputs)) =>
+      case JobCompletion(key, JobResultSuccess(returnCode, jobOutputs)) =>
         val entry = JobStoreEntry(
           key.workflowId.toString,
           key.callFqn,
@@ -37,11 +47,11 @@ class SqlJobStore(sqlDatabase: JobStoreSqlDatabase) extends JobStore {
         val jobStoreResultSimpletons =
           jobOutputs.mapValues(_.wdlValue).simplify.map {
             wdlValueSimpleton => JobStoreSimpletonEntry(
-              wdlValueSimpleton.simpletonKey, wdlValueSimpleton.simpletonValue.valueString.toClob,
+              wdlValueSimpleton.simpletonKey, wdlValueSimpleton.simpletonValue.valueString.toClobOption,
               wdlValueSimpleton.simpletonValue.wdlType.toWdlString)
           }
         JobStoreJoin(entry, jobStoreResultSimpletons.toSeq)
-      case (key, JobResultFailure(returnCode, throwable, retryable)) =>
+      case JobCompletion(key, JobResultFailure(returnCode, throwable, retryable)) =>
         val entry = JobStoreEntry(
           key.workflowId.toString,
           key.callFqn,
@@ -49,7 +59,7 @@ class SqlJobStore(sqlDatabase: JobStoreSqlDatabase) extends JobStore {
           key.attempt,
           jobSuccessful = false,
           returnCode,
-          throwable.getMessage.toClob,
+          throwable.getMessage.toClobOption,
           Option(retryable))
         JobStoreJoin(entry, Seq.empty)
     }
diff --git a/engine/src/main/scala/cromwell/server/CromwellRootActor.scala b/engine/src/main/scala/cromwell/server/CromwellRootActor.scala
index 0a6b708b2..6b0e7d765 100644
--- a/engine/src/main/scala/cromwell/server/CromwellRootActor.scala
+++ b/engine/src/main/scala/cromwell/server/CromwellRootActor.scala
@@ -1,31 +1,41 @@
 package cromwell.server
 
 import akka.actor.SupervisorStrategy.{Escalate, Restart}
-import akka.actor.{Actor, ActorInitializationException, ActorRef, OneForOneStrategy}
+import akka.actor.{Actor, ActorInitializationException, ActorLogging, ActorRef, OneForOneStrategy}
 import akka.event.Logging
 import akka.http.scaladsl.Http
-import akka.http.scaladsl.model.HttpRequest
+import akka.pattern.GracefulStopSupport
 import akka.routing.RoundRobinPool
 import akka.stream.ActorMaterializer
 import com.typesafe.config.ConfigFactory
-import cromwell.core.Dispatcher
-import cromwell.core.callcaching.docker.DockerHashActor
-import cromwell.core.callcaching.docker.DockerHashActor.{DockerHashActorException, DockerHashContext}
-import cromwell.core.callcaching.docker.registryv2.flows.dockerhub.DockerHubFlow
-import cromwell.core.callcaching.docker.registryv2.flows.gcr.GoogleFlow
+import cromwell.core.actor.StreamActorHelper.ActorRestartException
+import cromwell.core.io.Throttle
+import cromwell.core.{Dispatcher, DockerConfiguration, DockerLocalLookup, DockerRemoteLookup}
+import cromwell.docker.DockerHashActor
+import cromwell.docker.DockerHashActor.DockerHashContext
+import cromwell.docker.local.DockerCliFlow
+import cromwell.docker.registryv2.flows.HttpFlowWithRetry.ContextWithRequest
+import cromwell.docker.registryv2.flows.dockerhub.DockerHubFlow
+import cromwell.docker.registryv2.flows.gcr.GoogleFlow
+import cromwell.docker.registryv2.flows.quay.QuayFlow
 import cromwell.engine.backend.{BackendSingletonCollection, CromwellBackends}
+import cromwell.engine.io.IoActor
 import cromwell.engine.workflow.WorkflowManagerActor
+import cromwell.engine.workflow.WorkflowManagerActor.AbortAllWorkflowsCommand
 import cromwell.engine.workflow.lifecycle.CopyWorkflowLogsActor
-import cromwell.engine.workflow.lifecycle.execution.callcaching.{CallCache, CallCacheReadActor}
+import cromwell.engine.workflow.lifecycle.execution.callcaching.{CallCache, CallCacheReadActor, CallCacheWriteActor}
 import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor
 import cromwell.engine.workflow.workflowstore.{SqlWorkflowStore, WorkflowStore, WorkflowStoreActor}
 import cromwell.jobstore.{JobStore, JobStoreActor, SqlJobStore}
 import cromwell.services.{ServiceRegistryActor, SingletonServicesStore}
 import cromwell.subworkflowstore.{SqlSubWorkflowStore, SubWorkflowStoreActor}
+import cromwell.util.GracefulShutdownHelper
 import net.ceedubs.ficus.Ficus._
 
+import scala.concurrent.Await
 import scala.concurrent.duration._
 import scala.language.postfixOps
+import scala.util.{Failure, Success, Try}
 
 /**
   * An actor which serves as the lord protector for the rest of Cromwell, allowing us to have more fine grain
@@ -36,27 +46,25 @@ import scala.language.postfixOps
   *
   * If any of the actors created by CromwellRootActor fail to initialize the ActorSystem will die, which means that
   * Cromwell will fail to start in a bad state regardless of the entry point.
+  * 
+  * READ THIS: If you add a "system-level" actor here, make sure to consider what should be its
+  * position in the shutdown process and modify CromwellShutdown accordingly.
   */
- abstract class CromwellRootActor extends Actor {
+abstract class CromwellRootActor(gracefulShutdown: Boolean, abortJobsOnTerminate: Boolean)(implicit materializer: ActorMaterializer) extends Actor with ActorLogging with GracefulShutdownHelper {
   import CromwellRootActor._
 
   private val logger = Logging(context.system, this)
   private val config = ConfigFactory.load()
   private implicit val system = context.system
-  private implicit val materializer = ActorMaterializer()
-  
+
   val serverMode: Boolean
 
+  lazy val systemConfig = config.getConfig("system")
   lazy val serviceRegistryActor: ActorRef = context.actorOf(ServiceRegistryActor.props(config), "ServiceRegistryActor")
-  lazy val numberOfWorkflowLogCopyWorkers = config.getConfig("system").as[Option[Int]]("number-of-workflow-log-copy-workers").getOrElse(DefaultNumberOfWorkflowLogCopyWorkers)
-
-  lazy val workflowLogCopyRouter: ActorRef = context.actorOf(RoundRobinPool(numberOfWorkflowLogCopyWorkers)
-      .withSupervisorStrategy(CopyWorkflowLogsActor.strategy)
-      .props(CopyWorkflowLogsActor.props(serviceRegistryActor)),
-      "WorkflowLogCopyRouter")
+  lazy val numberOfWorkflowLogCopyWorkers = systemConfig.as[Option[Int]]("number-of-workflow-log-copy-workers").getOrElse(DefaultNumberOfWorkflowLogCopyWorkers)
 
   lazy val workflowStore: WorkflowStore = SqlWorkflowStore(SingletonServicesStore.databaseInterface)
-  lazy val workflowStoreActor = context.actorOf(WorkflowStoreActor.props(workflowStore, serviceRegistryActor), "WorkflowStoreActor")
+  lazy val workflowStoreActor = context.actorOf(WorkflowStoreActor.props(workflowStore, serviceRegistryActor, SingletonServicesStore.databaseInterface), "WorkflowStoreActor")
 
   lazy val jobStore: JobStore = new SqlJobStore(SingletonServicesStore.databaseInterface)
   lazy val jobStoreActor = context.actorOf(JobStoreActor.props(jobStore), "JobStoreActor")
@@ -64,26 +72,44 @@ import scala.language.postfixOps
   lazy val subWorkflowStore = new SqlSubWorkflowStore(SingletonServicesStore.databaseInterface)
   lazy val subWorkflowStoreActor = context.actorOf(SubWorkflowStoreActor.props(subWorkflowStore), "SubWorkflowStoreActor")
 
+  // Io Actor
+  lazy val throttleElements = systemConfig.as[Option[Int]]("io.number-of-requests").getOrElse(100000)
+  lazy val throttlePer = systemConfig.as[Option[FiniteDuration]]("io.per").getOrElse(100 seconds)
+  lazy val ioThrottle = Throttle(throttleElements, throttlePer, throttleElements)
+  lazy val ioActor = context.actorOf(IoActor.props(1000, Option(ioThrottle)), "IoActor")
+
+  lazy val workflowLogCopyRouter: ActorRef = context.actorOf(RoundRobinPool(numberOfWorkflowLogCopyWorkers)
+    .withSupervisorStrategy(CopyWorkflowLogsActor.strategy)
+    .props(CopyWorkflowLogsActor.props(serviceRegistryActor, ioActor)),
+    "WorkflowLogCopyRouter")
+
   lazy val callCache: CallCache = new CallCache(SingletonServicesStore.databaseInterface)
 
   lazy val numberOfCacheReadWorkers = config.getConfig("system").as[Option[Int]]("number-of-cache-read-workers").getOrElse(DefaultNumberOfCacheReadWorkers)
   lazy val callCacheReadActor = context.actorOf(RoundRobinPool(numberOfCacheReadWorkers)
     .props(CallCacheReadActor.props(callCache)),
     "CallCacheReadActor")
-  
+
+  lazy val callCacheWriteActor = context.actorOf(CallCacheWriteActor.props(callCache), "CallCacheWriteActor")
+
   // Docker Actor
   lazy val ioEc = context.system.dispatchers.lookup(Dispatcher.IoDispatcher)
-  lazy val gcrQueriesPer100Sec = config.getAs[Int]("docker.gcr-api-queries-per-100-seconds") getOrElse 1000
-  lazy val dockerCacheEntryTTL = config.as[Option[FiniteDuration]]("docker.cache-entry-ttl").getOrElse(DefaultCacheTTL)
-  lazy val dockerCacheSize = config.getAs[Long]("docker.cache-size") getOrElse 200L
+  lazy val dockerConf = DockerConfiguration.instance
   // Sets the number of requests that the docker actor will accept before it starts backpressuring (modulo the number of in flight requests)
   lazy val dockerActorQueueSize = 500
-  
-  lazy val dockerHttpPool = Http().superPool[(DockerHashContext, HttpRequest)]()
-  lazy val googleFlow = new GoogleFlow(dockerHttpPool, gcrQueriesPer100Sec)(ioEc, materializer)
-  lazy val dockerHubFlow = new DockerHubFlow(dockerHttpPool)(ioEc, materializer)
-  lazy val dockerFlows = Seq(dockerHubFlow, googleFlow)
-  lazy val dockerHashActor = context.actorOf(DockerHashActor.props(dockerFlows, dockerActorQueueSize, dockerCacheEntryTTL, dockerCacheSize)(materializer).withDispatcher(Dispatcher.IoDispatcher))
+
+  lazy val dockerHttpPool = Http().superPool[ContextWithRequest[DockerHashContext]]()
+  lazy val googleFlow = new GoogleFlow(dockerHttpPool, dockerConf.gcrApiQueriesPer100Seconds)(ioEc, materializer, system.scheduler)
+  lazy val dockerHubFlow = new DockerHubFlow(dockerHttpPool)(ioEc, materializer, system.scheduler)
+  lazy val quayFlow = new QuayFlow(dockerHttpPool)(ioEc, materializer, system.scheduler)
+  lazy val dockerCliFlow = new DockerCliFlow()(ioEc, system.scheduler)
+  lazy val dockerFlows = dockerConf.method match {
+    case DockerLocalLookup => Seq(dockerCliFlow)
+    case DockerRemoteLookup => Seq(dockerHubFlow, googleFlow, quayFlow)
+  }
+
+  lazy val dockerHashActor = context.actorOf(DockerHashActor.props(dockerFlows, dockerActorQueueSize,
+    dockerConf.cacheEntryTtl, dockerConf.cacheSize)(materializer), "DockerHashActor")
 
   lazy val backendSingletons = CromwellBackends.instance.get.backendLifecycleActorFactories map {
     case (name, factory) => name -> (factory.backendSingletonActorProps map context.actorOf)
@@ -92,16 +118,40 @@ import scala.language.postfixOps
 
   lazy val jobExecutionTokenDispenserActor = context.actorOf(JobExecutionTokenDispenserActor.props)
 
-  def abortJobsOnTerminate: Boolean
-
   lazy val workflowManagerActor = context.actorOf(
     WorkflowManagerActor.props(
-      workflowStoreActor, serviceRegistryActor, workflowLogCopyRouter, jobStoreActor, subWorkflowStoreActor, callCacheReadActor,
-      dockerHashActor, jobExecutionTokenDispenserActor, backendSingletonCollection, abortJobsOnTerminate, serverMode),
+      workflowStoreActor, ioActor, serviceRegistryActor, workflowLogCopyRouter, jobStoreActor, subWorkflowStoreActor, callCacheReadActor, callCacheWriteActor,
+      dockerHashActor, jobExecutionTokenDispenserActor, backendSingletonCollection, serverMode),
     "WorkflowManagerActor")
 
+  if (gracefulShutdown) {
+    // If abortJobsOnTerminate is true, aborting all workflows will be handled by the graceful shutdown process
+    CromwellShutdown.registerShutdownTasks(
+      abortJobsOnTerminate,
+      actorSystem = context.system,
+      workflowManagerActor = workflowManagerActor,
+      logCopyRouter = workflowLogCopyRouter,
+      jobStoreActor = jobStoreActor,
+      workflowStoreActor = workflowStoreActor,
+      subWorkflowStoreActor = subWorkflowStoreActor,
+      callCacheWriteActor = callCacheWriteActor,
+      ioActor = ioActor,
+      dockerHashActor = dockerHashActor,
+      serviceRegistryActor = serviceRegistryActor,
+      materializer = materializer
+    )
+  } else if (abortJobsOnTerminate) {
+    // If gracefulShutdown is false but abortJobsOnTerminate is true, set up a classic JVM shutdown hook
+    sys.addShutdownHook {
+      Try(Await.result(gracefulStop(workflowManagerActor, AbortTimeout, AbortAllWorkflowsCommand), AbortTimeout)) match {
+        case Success(_) => logger.info("All workflows aborted")
+        case Failure(f) => logger.error("Failed to abort workflows", f)
+      }
+    }
+  }
+
   override def receive = {
-    case _ => logger.error("CromwellRootActor is receiving a message. It prefers to be left alone!")
+    case message => logger.error(s"Unknown message received by CromwellRootActor: $message")
   }
 
   /**
@@ -109,13 +159,15 @@ import scala.language.postfixOps
     * of Cromwell by passing a Throwable to the guardian.
     */
   override val supervisorStrategy = OneForOneStrategy() {
-    case actorInitializationException: ActorInitializationException => Escalate
-    case dockerHash: DockerHashActorException => Restart
+    case _: ActorInitializationException => Escalate
+    case _: ActorRestartException => Restart
     case t => super.supervisorStrategy.decider.applyOrElse(t, (_: Any) => Escalate)
   }
 }
 
-object CromwellRootActor {
+object CromwellRootActor extends GracefulStopSupport {
+  import net.ceedubs.ficus.Ficus._
+  val AbortTimeout = ConfigFactory.load().as[FiniteDuration]("akka.coordinated-shutdown.phases.abort-all-workflows.timeout")
   val DefaultNumberOfWorkflowLogCopyWorkers = 10
   val DefaultCacheTTL = 20 minutes
   val DefaultNumberOfCacheReadWorkers = 25
diff --git a/engine/src/main/scala/cromwell/server/CromwellServer.scala b/engine/src/main/scala/cromwell/server/CromwellServer.scala
index 56f87ef87..c060dce86 100644
--- a/engine/src/main/scala/cromwell/server/CromwellServer.scala
+++ b/engine/src/main/scala/cromwell/server/CromwellServer.scala
@@ -1,73 +1,66 @@
 package cromwell.server
 
-import java.util.concurrent.TimeoutException
-
-import akka.actor.{ActorContext, ActorSystem, Props}
-import com.typesafe.config.Config
+import akka.actor.{ActorContext, ActorLogging, Props}
+import akka.http.scaladsl.Http
+import akka.http.scaladsl.server.Directives._
+import akka.http.scaladsl.server.Route
+import akka.stream.ActorMaterializer
 import cromwell.core.Dispatcher.EngineDispatcher
-import cromwell.webservice.WorkflowJsonSupport._
-import cromwell.webservice.{APIResponse, CromwellApiService, SwaggerService}
-import lenthall.spray.SprayCanHttpService._
-import lenthall.spray.WrappedRoute._
-import net.ceedubs.ficus.Ficus._
-import spray.http._
-import spray.json._
-import spray.routing.Route
+import cromwell.webservice.{CromwellApiService, SwaggerService}
 
-import scala.concurrent.duration._
-import scala.concurrent.{Await, ExecutionContextExecutor, Future}
+import scala.concurrent.Future
 import scala.util.{Failure, Success}
 
 // Note that as per the language specification, this is instantiated lazily and only used when necessary (i.e. server mode)
 object CromwellServer {
-
-  def run(cromwellSystem: CromwellSystem): Future[Any] = {
-    implicit val executionContext = scala.concurrent.ExecutionContext.Implicits.global
-
-    val actorSystem: ActorSystem = cromwellSystem.actorSystem
-
-    val service = actorSystem.actorOf(CromwellServerActor.props(cromwellSystem.conf), "cromwell-service")
-    val webserviceConf = cromwellSystem.conf.getConfig("webservice")
-
-    val interface = webserviceConf.getString("interface")
-    val port = webserviceConf.getInt("port")
-    val timeout = webserviceConf.as[FiniteDuration]("binding-timeout")
-    val futureBind = service.bind(interface, port)(implicitly, timeout, actorSystem, implicitly)
-    futureBind andThen {
-      case Success(_) =>
-        actorSystem.log.info("Cromwell service started...")
-        Await.result(actorSystem.whenTerminated, Duration.Inf)
-      case Failure(throwable) =>
-        /*
-        TODO:
-        If/when CromwellServer behaves like a better async citizen, we may be less paranoid about our async log messages
-        not appearing due to the actor system shutdown. For now, synchronously print to the stderr so that the user has
-        some idea of why the server failed to start up.
-         */
-        Console.err.println(s"Binding failed interface $interface port $port")
-        throwable.printStackTrace(Console.err)
-        cromwellSystem.shutdownActorSystem()
-    }
+  def run(gracefulShutdown: Boolean, abortJobsOnTerminate: Boolean)(cromwellSystem: CromwellSystem): Future[Any] = {
+    implicit val actorSystem = cromwellSystem.actorSystem
+    implicit val materializer = cromwellSystem.materializer
+    actorSystem.actorOf(CromwellServerActor.props(cromwellSystem, gracefulShutdown, abortJobsOnTerminate), "cromwell-service")
+    actorSystem.whenTerminated
   }
 }
 
-class CromwellServerActor(config: Config) extends CromwellRootActor with CromwellApiService with SwaggerService {
-  implicit def executionContext: ExecutionContextExecutor = actorRefFactory.dispatcher
+class CromwellServerActor(cromwellSystem: CromwellSystem, gracefulShutdown: Boolean, abortJobsOnTerminate: Boolean)(override implicit val materializer: ActorMaterializer)
+  extends CromwellRootActor(gracefulShutdown, abortJobsOnTerminate)
+    with CromwellApiService
+    with SwaggerService
+    with ActorLogging {
+  implicit val actorSystem = context.system
+  override implicit val ec = context.dispatcher
+  override def actorRefFactory: ActorContext = context
 
   override val serverMode = true
-  override val abortJobsOnTerminate = false
 
-  override def actorRefFactory: ActorContext = context
-  override def receive: PartialFunction[Any, Unit] = handleTimeouts orElse runRoute(possibleRoutes)
+  val webserviceConf = cromwellSystem.conf.getConfig("webservice")
+  val interface = webserviceConf.getString("interface")
+  val port = webserviceConf.getInt("port")
+
+  /**
+    * /api routes have special meaning to devops' proxy servers. NOTE: the oauth mentioned on the /api endpoints in
+    * cromwell.yaml is broken unless the swagger index.html is patched. Copy/paste the code from rawls or cromiam if
+    * actual cromwell+swagger+oauth+/api support is needed.
+    */
+  val apiRoutes: Route = pathPrefix("api")(concat(workflowRoutes))
+  val nonApiRoutes: Route = concat(engineRoutes, swaggerUiResourceRoute)
+  val allRoutes: Route = concat(apiRoutes, nonApiRoutes)
+
+  val serverBinding = Http().bindAndHandle(allRoutes, interface, port)
 
-  val routeUnwrapped: Boolean = config.as[Option[Boolean]]("api.routeUnwrapped").getOrElse(false)
-  val possibleRoutes: Route = workflowRoutes.wrapped("api", routeUnwrapped) ~ swaggerUiResourceRoute
-  val timeoutError: String = APIResponse.error(new TimeoutException(
-    "The server was not able to produce a timely response to your request.")).toJson.prettyPrint
+  CromwellShutdown.registerUnbindTask(actorSystem, serverBinding)
 
-  def handleTimeouts: Receive = {
-    case Timedout(_: HttpRequest) =>
-      sender() ! HttpResponse(StatusCodes.InternalServerError, HttpEntity(ContentType(MediaTypes.`application/json`), timeoutError))
+  serverBinding onComplete {
+    case Success(_) => actorSystem.log.info("Cromwell service started...")
+    case Failure(e) =>
+      /*
+        TODO:
+        If/when CromwellServer behaves like a better async citizen, we may be less paranoid about our async log messages
+        not appearing due to the actor system shutdown. For now, synchronously print to the stderr so that the user has
+        some idea of why the server failed to start up.
+      */
+      Console.err.println(s"Binding failed interface $interface port $port")
+      e.printStackTrace(Console.err)
+      cromwellSystem.shutdownActorSystem()
   }
 
   /*
@@ -78,7 +71,7 @@ class CromwellServerActor(config: Config) extends CromwellRootActor with Cromwel
 }
 
 object CromwellServerActor {
-  def props(config: Config): Props = {
-    Props(new CromwellServerActor(config)).withDispatcher(EngineDispatcher)
+  def props(cromwellSystem: CromwellSystem, gracefulShutdown: Boolean, abortJobsOnTerminate: Boolean)(implicit materializer: ActorMaterializer): Props = {
+    Props(new CromwellServerActor(cromwellSystem, gracefulShutdown, abortJobsOnTerminate)).withDispatcher(EngineDispatcher)
   }
 }
diff --git a/engine/src/main/scala/cromwell/server/CromwellShutdown.scala b/engine/src/main/scala/cromwell/server/CromwellShutdown.scala
new file mode 100644
index 000000000..bbb11193a
--- /dev/null
+++ b/engine/src/main/scala/cromwell/server/CromwellShutdown.scala
@@ -0,0 +1,208 @@
+package cromwell.server
+
+import java.util.concurrent.atomic.AtomicBoolean
+
+import akka.Done
+import akka.actor.{ActorRef, ActorSystem, CoordinatedShutdown, _}
+import akka.http.scaladsl.Http
+import akka.pattern.{AskTimeoutException, GracefulStopSupport}
+import akka.routing.Broadcast
+import akka.stream.ActorMaterializer
+import akka.util.Timeout
+import cats.instances.future._
+import cats.syntax.functor._
+import cromwell.engine.workflow.WorkflowManagerActor.{AbortAllWorkflowsCommand, PreventNewWorkflowsFromStarting}
+import cromwell.services.SingletonServicesStore
+import cromwell.util.GracefulShutdownHelper.ShutdownCommand
+import org.slf4j.LoggerFactory
+
+import scala.concurrent.duration._
+import scala.concurrent.{ExecutionContext, Future}
+import scala.util.{Failure, Success}
+
+/**
+  * Collection of methods and objects used to control Cromwell graceful shutdown process.
+  */
+object CromwellShutdown extends GracefulStopSupport {
+  private val logger = LoggerFactory.getLogger("CromwellShutdown")
+  
+  // Includes DB writing actors, I/O Actor and DockerHashActor
+  private val PhaseStopIoActivity = "stop-io-activity"
+  // Shutdown phase allocated when "abort-jobs-on-terminate" is true to give time to the system to abort all workflows
+  // This phase is at the same level as PhaseServiceRequestsDone in the dependency graph.
+  private val PhaseAbortAllWorkflows = "abort-all-workflows"
+
+  private val _shutdownInProgress: AtomicBoolean = new AtomicBoolean(false)
+  private var _coordinatedShutdown: Option[CoordinatedShutdown] = None
+
+  /**
+    * Single instance of CoordinatedShutdown. Assumes only one ActorSystem.
+    */
+  def instance(implicit actorSystem: ActorSystem): CoordinatedShutdown = synchronized {
+    _coordinatedShutdown match {
+      case Some(v) => v
+      case None =>
+        val coordinatedShutdown = CoordinatedShutdown(actorSystem)
+        _coordinatedShutdown = Option(coordinatedShutdown)
+        coordinatedShutdown
+    }
+  }
+
+  /**
+    * Returns true of the coordinated shutdown process is in progress. False otherwise.
+    */
+  def shutdownInProgress(): Boolean = _shutdownInProgress.get()
+
+  /**
+    * Register a task to unbind from the port during the ServiceUnbind phase.
+    */
+  def registerUnbindTask(actorSystem: ActorSystem, serverBinding: Future[Http.ServerBinding]) = {
+    instance(actorSystem).addTask(CoordinatedShutdown.PhaseServiceUnbind, "UnbindingServerPort") { () =>
+      // At this point it's still safe to schedule work on the actor system's dispatcher
+      implicit val ec = actorSystem.dispatcher
+      for {
+        binding <- serverBinding
+        _ <- binding.unbind()
+        _ = logger.info("Http server unbound.") 
+      } yield Done
+    }
+  }
+
+  /**
+    * Register tasks on the coordinated shutdown instance allowing a controlled, ordered shutdown process
+    * meant to prevent data loss and ensure consistency.
+    * Calling this method will add a JVM shutdown hook.
+    */
+  def registerShutdownTasks(
+                             abortJobsOnTerminate: Boolean,
+                             actorSystem: ActorSystem,
+                             workflowManagerActor: ActorRef,
+                             logCopyRouter: ActorRef,
+                             jobStoreActor: ActorRef,
+                             workflowStoreActor: ActorRef,
+                             subWorkflowStoreActor: ActorRef,
+                             callCacheWriteActor: ActorRef,
+                             ioActor: ActorRef,
+                             dockerHashActor: ActorRef,
+                             serviceRegistryActor: ActorRef,
+                             materializer: ActorMaterializer
+                           ): Unit = {
+
+    val coordinatedShutdown = this.instance(actorSystem)
+
+    def shutdownActor(actor: ActorRef,
+                      phase: String,
+                      message: AnyRef,
+                      customTimeout: Option[FiniteDuration] = None)(implicit executionContext: ExecutionContext) = {
+      coordinatedShutdown.addTask(phase, s"stop${actor.path.name.capitalize}") { () =>
+        val action = gracefulStop(actor, customTimeout.getOrElse(coordinatedShutdown.timeout(phase)), message)
+
+        action onComplete {
+          case Success(_) => logger.info(s"${actor.path.name} stopped")
+          case Failure(_: AskTimeoutException) => 
+            logger.error(s"Timed out trying to gracefully stop ${actor.path.name}. Forcefully stopping it.")
+            actorSystem.stop(actor)
+          case Failure(f) => logger.error(s"An error occurred trying to gracefully stop ${actor.path.name}.", f)
+        }
+        
+        action map { _ => Done }
+      }
+    }
+
+    implicit val ec = actorSystem.dispatcher
+
+    // 1) Stop starting new workflows. This will stop the WMA from sending pull messages to the WorkflowStore
+    coordinatedShutdown.addTask(CoordinatedShutdown.PhaseBeforeServiceUnbind, "stopWorkflowPulling") { () =>
+      import akka.pattern.ask
+      _shutdownInProgress.set(true)
+      implicit val timeout = Timeout(coordinatedShutdown.timeout(CoordinatedShutdown.PhaseBeforeServiceUnbind))
+      (workflowManagerActor ? PreventNewWorkflowsFromStarting) map { _ =>
+        logger.info("Workflow polling stopped")
+        Done
+      }
+    }
+
+    /* 2) The socket is unbound from the port in the CoordinatedShutdown.PhaseServiceUnbind
+      * See cromwell.engine.server.CromwellServer
+     */
+
+    /* 3) Finish processing all requests:
+      *  - Stop the WorkflowStore: The port is not bound anymore so we can't have new submissions.
+      *   Process what's left in the message queue and stop. 
+      *   Note that it's possible that some submissions are still asynchronously being prepared at the 
+      *   akka http API layer (CromwellApiService) to be sent to the WorkflowStore.
+      *   Those submissions might be lost if the WorkflowStore shuts itself down when it's finished processing its current work.
+      *   In that case the "ask" over in the CromwellAPIService will fail with a AskTimeoutException and should be handled appropriately.
+      *   This process still ensures that no submission can make it to the database without a response being sent back to the client.
+      *   
+      *  - Stop WorkflowManagerActor: We've already stopped starting new workflows but the Running workflows are still
+      *   going. This is tricky because all the actor hierarchy under the WMA can be in a variety of state combinations.
+      *   Specifically there is an asynchronous gap in several cases between emission of messages towards engine level
+      *   actors (job store, cache store, etc...) and emission of messages towards the metadata service for the same logical
+      *   event (e.g: job complete).
+      *   The current behavior upon restart however is to re-play the graph, skipping execution of completed jobs (determined by
+      *   engine job store) but still re-submitting all related metadata events. This is likely sub-optimal, but is used here
+      *   to simply stop the WMA (which will trigger all its descendants to be stopped recursively) without more coordination.
+      *   Indeed even if the actor is stopped in between the above mentioned gap, metadata will be re-submitted anyway on restart,
+      *   even for completed jobs.
+      *   
+      *  - Stop the LogCopyRouter: it can generate metadata events and must therefore be stopped before the service registry.
+      *   Wrap the ShutdownCommand in a Broadcast message so the router forwards it to all its routees
+      *   Use the ShutdownCommand because a PoisonPill could stop the routees in the middle of "transaction"
+      *   with the IoActor. The routees handle the ShutdownCommand properly and shutdown only when they have
+      *   no outstanding requests to the IoActor. When all routees are dead the router automatically stops itself.
+    */
+    shutdownActor(workflowStoreActor, CoordinatedShutdown.PhaseServiceRequestsDone, ShutdownCommand)
+    shutdownActor(logCopyRouter, CoordinatedShutdown.PhaseServiceRequestsDone, Broadcast(ShutdownCommand))
+    
+    /*
+      * Aborting is only a special case of shutdown. Instead of sending a PoisonPill, send a AbortAllWorkflowsCommand
+      * Also attach this task to a special shutdown phase allowing for a longer timeout.
+     */
+    if (abortJobsOnTerminate) {
+      val abortTimeout = coordinatedShutdown.timeout(PhaseAbortAllWorkflows)
+      shutdownActor(workflowManagerActor, PhaseAbortAllWorkflows, AbortAllWorkflowsCommand, Option(abortTimeout))
+    } else {
+      // This is a pretty rough shutdown of the WMA, depending on which route we go (full let it crash, or more fine grained
+      // state management) we might want to revisit this way of terminating the WMA and its descendants.
+      shutdownActor(workflowManagerActor, CoordinatedShutdown.PhaseServiceRequestsDone, PoisonPill)
+    }
+
+    /* 4) Shutdown connection pools
+      * This will close all akka http opened connection pools tied to the actor system.
+      * The pools stop accepting new work but are given a chance to execute the work submitted prior to the shutdown call.
+      * When this future returns, all outstanding connections to client will be terminated.
+      * Note that this also includes connection pools like the one used to lookup docker hashes.
+     */
+    coordinatedShutdown.addTask(CoordinatedShutdown.PhaseServiceStop, "TerminatingConnections") { () =>
+      Http(actorSystem).shutdownAllConnectionPools() as {
+        logger.info("Connection pools shut down")
+        Done
+      }
+    }
+
+    /* 5) Stop system level actors that require writing to the database or I/O
+      * - SubWorkflowStoreActor
+      * - JobStoreActor
+      * - CallCacheWriteActor
+      * - ServiceRegistryActor
+      * - DockerHashActor
+      * - IoActor
+    */
+    List(subWorkflowStoreActor, jobStoreActor, callCacheWriteActor, serviceRegistryActor, dockerHashActor, ioActor) foreach {
+      shutdownActor(_, PhaseStopIoActivity, ShutdownCommand)
+    }
+
+    // 6) Close database and stream materializer
+    coordinatedShutdown.addTask(CoordinatedShutdown.PhaseBeforeActorSystemTerminate, "closeDatabase") { () =>
+      SingletonServicesStore.databaseInterface.close()
+      logger.info("Database closed")
+      Future.successful(Done)
+    }
+    coordinatedShutdown.addTask(CoordinatedShutdown.PhaseBeforeActorSystemTerminate, "shutdownMaterializer") { () =>
+      materializer.shutdown()
+      logger.info("Stream materializer shut down")
+      Future.successful(Done)
+    }
+  }
+}
diff --git a/engine/src/main/scala/cromwell/server/CromwellSystem.scala b/engine/src/main/scala/cromwell/server/CromwellSystem.scala
index 648850f71..862c53d7d 100644
--- a/engine/src/main/scala/cromwell/server/CromwellSystem.scala
+++ b/engine/src/main/scala/cromwell/server/CromwellSystem.scala
@@ -1,20 +1,45 @@
 package cromwell.server
 
 import akka.actor.{ActorSystem, Terminated}
+import akka.http.scaladsl.Http
+import akka.stream.ActorMaterializer
 import com.typesafe.config.ConfigFactory
 import cromwell.engine.backend.{BackendConfiguration, CromwellBackends}
-import org.slf4j.LoggerFactory
+import cromwell.services.SingletonServicesStore
 
 import scala.concurrent.Future
 
 trait CromwellSystem {
+  /*
+  Initialize the services-store (aka a collection of DAOs), if they haven't been already, before starting any services
+  such as Metadata refresh and especially HTTP server binding. Some services like HTTP binding terminate the app when
+  they failed to start, while the DAOs were still in the middle of non-atomic transaction such as a database upgrade.
+
+  This is the closest we have to:
+  - https://github.com/broadinstitute/firecloud-orchestration/blob/3c9482b/DEVNOTES.md
+  - https://github.com/broadinstitute/clio/blob/7253ec0/clio-server/src/main/scala/org/broadinstitute/clio/server/service/ServerService.scala#L58-L66
+  - https://en.wikipedia.org/wiki/Data_access_object
+   */
+  SingletonServicesStore
+
   protected def systemName = "cromwell-system"
-  protected def newActorSystem(): ActorSystem = ActorSystem(systemName)
   val conf = ConfigFactory.load()
-  val logger = LoggerFactory.getLogger(getClass.getName)
+  
+  protected def newActorSystem(): ActorSystem = ActorSystem(systemName, conf)
   implicit final lazy val actorSystem = newActorSystem()
+  implicit final lazy val materializer = ActorMaterializer()
+  implicit private final lazy val ec = actorSystem.dispatcher
 
   def shutdownActorSystem(): Future[Terminated] = {
+    Http().shutdownAllConnectionPools() flatMap { _ =>
+      shutdownMaterializerAndActorSystem()
+    } recoverWith {
+      case _ => shutdownMaterializerAndActorSystem()
+    }
+  }
+  
+  private def shutdownMaterializerAndActorSystem() = {
+    materializer.shutdown()
     actorSystem.terminate()
   }
 
diff --git a/engine/src/main/scala/cromwell/subworkflowstore/EmptySubWorkflowStoreActor.scala b/engine/src/main/scala/cromwell/subworkflowstore/EmptySubWorkflowStoreActor.scala
index 166d7d685..29b545f70 100644
--- a/engine/src/main/scala/cromwell/subworkflowstore/EmptySubWorkflowStoreActor.scala
+++ b/engine/src/main/scala/cromwell/subworkflowstore/EmptySubWorkflowStoreActor.scala
@@ -2,16 +2,17 @@ package cromwell.subworkflowstore
 
 import akka.actor.{Actor, ActorLogging, Props}
 import cromwell.subworkflowstore.SubWorkflowStoreActor._
+import cromwell.core.Dispatcher.EngineDispatcher
 
 class EmptySubWorkflowStoreActor extends Actor with ActorLogging {
   override def receive: Receive = {
     case register: RegisterSubWorkflow => sender() ! SubWorkflowStoreRegisterSuccess(register) 
     case query: QuerySubWorkflow => sender() ! SubWorkflowNotFound(query)
-    case complete: WorkflowComplete =>sender() ! SubWorkflowStoreCompleteSuccess(complete)
+    case _: WorkflowComplete => // No-op!
     case unknown => log.error(s"SubWorkflowStoreActor received unknown message: $unknown")
   }
 }
 
 object EmptySubWorkflowStoreActor {
-  def props: Props = Props(new EmptySubWorkflowStoreActor())
+  def props: Props = Props(new EmptySubWorkflowStoreActor()).withDispatcher(EngineDispatcher)
 }
diff --git a/engine/src/main/scala/cromwell/subworkflowstore/SubWorkflowStoreActor.scala b/engine/src/main/scala/cromwell/subworkflowstore/SubWorkflowStoreActor.scala
index 25fb62ce0..c5d76c0b8 100644
--- a/engine/src/main/scala/cromwell/subworkflowstore/SubWorkflowStoreActor.scala
+++ b/engine/src/main/scala/cromwell/subworkflowstore/SubWorkflowStoreActor.scala
@@ -3,25 +3,27 @@ package cromwell.subworkflowstore
 import akka.actor.{Actor, ActorLogging, ActorRef, Props}
 import cromwell.core.Dispatcher.EngineDispatcher
 import cromwell.core.ExecutionIndex._
-import cromwell.core.{JobKey, WorkflowId}
+import cromwell.core.{JobKey, MonitoringCompanionHelper, WorkflowId}
 import cromwell.database.sql.tables.SubWorkflowStoreEntry
 import cromwell.subworkflowstore.SubWorkflowStoreActor._
 
 import scala.concurrent.ExecutionContext
 import scala.util.{Failure, Success}
 
-class SubWorkflowStoreActor(database: SubWorkflowStore) extends Actor with ActorLogging {
+class SubWorkflowStoreActor(database: SubWorkflowStore) extends Actor with ActorLogging with MonitoringCompanionHelper {
   
   implicit val ec: ExecutionContext = context.dispatcher
 
-  override def receive = {
+  val subWorkflowStoreReceive: Receive = {
     case register: RegisterSubWorkflow => registerSubWorkflow(sender(), register)
     case query: QuerySubWorkflow => querySubWorkflow(sender(), query)
     case complete: WorkflowComplete => workflowComplete(sender(), complete)
-    case unknown => log.error(s"SubWorkflowStoreActor received unknown message: $unknown")
   }
   
+  override def receive = subWorkflowStoreReceive.orElse(monitoringReceive)
+  
   private def registerSubWorkflow(replyTo: ActorRef, command: RegisterSubWorkflow) = {
+    addWork()
     database.addSubWorkflowStoreEntry(
       command.rootWorkflowExecutionUuid.toString,
       command.parentWorkflowExecutionUuid.toString,
@@ -30,8 +32,12 @@ class SubWorkflowStoreActor(database: SubWorkflowStore) extends Actor with Actor
       command.jobKey.attempt,
       command.subWorkflowExecutionUuid.toString
     ) onComplete { 
-      case Success(_) => replyTo ! SubWorkflowStoreRegisterSuccess(command) 
-      case Failure(ex) => replyTo ! SubWorkflowStoreFailure(command, ex)
+      case Success(_) =>
+        replyTo ! SubWorkflowStoreRegisterSuccess(command) 
+        removeWork()
+      case Failure(ex) =>
+        replyTo ! SubWorkflowStoreFailure(command, ex)
+        removeWork()
     }
   }
 
@@ -45,9 +51,12 @@ class SubWorkflowStoreActor(database: SubWorkflowStore) extends Actor with Actor
   }
 
   private def workflowComplete(replyTo: ActorRef, command: WorkflowComplete) = {
+    addWork()
     database.removeSubWorkflowStoreEntries(command.workflowExecutionUuid.toString) onComplete {
-      case Success(_) => replyTo ! SubWorkflowStoreCompleteSuccess(command)
-      case Failure(ex) => replyTo ! SubWorkflowStoreFailure(command, ex)
+      case Success(_) => removeWork()
+      case Failure(ex) => 
+        replyTo ! SubWorkflowStoreFailure(command, ex)
+        removeWork()
     }
   }
   
@@ -63,7 +72,6 @@ object SubWorkflowStoreActor {
   case class SubWorkflowStoreRegisterSuccess(command: RegisterSubWorkflow) extends SubWorkflowStoreActorResponse
   case class SubWorkflowFound(subWorkflowStoreEntry: SubWorkflowStoreEntry) extends SubWorkflowStoreActorResponse
   case class SubWorkflowNotFound(command: QuerySubWorkflow) extends SubWorkflowStoreActorResponse
-  case class SubWorkflowStoreCompleteSuccess(command: SubWorkflowStoreActorCommand) extends SubWorkflowStoreActorResponse
   
   case class SubWorkflowStoreFailure(command: SubWorkflowStoreActorCommand, failure: Throwable) extends SubWorkflowStoreActorResponse
   
diff --git a/engine/src/main/scala/cromwell/webservice/ApiDataModels.scala b/engine/src/main/scala/cromwell/webservice/ApiDataModels.scala
index 45e7e6643..759abe62d 100644
--- a/engine/src/main/scala/cromwell/webservice/ApiDataModels.scala
+++ b/engine/src/main/scala/cromwell/webservice/ApiDataModels.scala
@@ -2,8 +2,8 @@ package cromwell.webservice
 
 import lenthall.exception.MessageAggregation
 import spray.json._
-import wdl4s.FullyQualifiedName
-import wdl4s.values.WdlValue
+import wdl4s.wdl.FullyQualifiedName
+import wdl4s.wdl.values.WdlValue
 
 
 case class WorkflowStatusResponse(id: String, status: String)
diff --git a/engine/src/main/scala/cromwell/webservice/CromwellApiHandler.scala b/engine/src/main/scala/cromwell/webservice/CromwellApiHandler.scala
deleted file mode 100644
index c1e92690d..000000000
--- a/engine/src/main/scala/cromwell/webservice/CromwellApiHandler.scala
+++ /dev/null
@@ -1,78 +0,0 @@
-package cromwell.webservice
-
-import akka.actor.{Actor, ActorRef, Props}
-import akka.event.Logging
-import cats.data.NonEmptyList
-import com.typesafe.config.ConfigFactory
-import cromwell.core._
-import cromwell.core.Dispatcher.ApiDispatcher
-import cromwell.engine.workflow.WorkflowManagerActor
-import cromwell.engine.workflow.WorkflowManagerActor.WorkflowNotFoundException
-import cromwell.engine.workflow.workflowstore.{WorkflowStoreActor, WorkflowStoreEngineActor, WorkflowStoreSubmitActor}
-import cromwell.webservice.PerRequest.RequestComplete
-import cromwell.webservice.metadata.WorkflowQueryPagination
-import spray.http.{StatusCodes, Uri}
-import spray.httpx.SprayJsonSupport._
-
-
-object CromwellApiHandler {
-  def props(requestHandlerActor: ActorRef): Props = {
-    Props(new CromwellApiHandler(requestHandlerActor)).withDispatcher(ApiDispatcher)
-  }
-
-  sealed trait ApiHandlerMessage
-
-  final case class ApiHandlerWorkflowSubmit(source: WorkflowSourceFilesCollection) extends ApiHandlerMessage
-  final case class ApiHandlerWorkflowSubmitBatch(sources: NonEmptyList[WorkflowSourceFilesCollection]) extends ApiHandlerMessage
-  final case class ApiHandlerWorkflowQuery(uri: Uri, parameters: Seq[(String, String)]) extends ApiHandlerMessage
-  final case class ApiHandlerWorkflowStatus(id: WorkflowId) extends ApiHandlerMessage
-  final case class ApiHandlerWorkflowOutputs(id: WorkflowId) extends ApiHandlerMessage
-  final case class ApiHandlerWorkflowAbort(id: WorkflowId, manager: ActorRef) extends ApiHandlerMessage
-  final case class ApiHandlerCallOutputs(id: WorkflowId, callFqn: String) extends ApiHandlerMessage
-  final case class ApiHandlerCallStdoutStderr(id: WorkflowId, callFqn: String) extends ApiHandlerMessage
-  final case class ApiHandlerWorkflowStdoutStderr(id: WorkflowId) extends ApiHandlerMessage
-  final case class ApiHandlerCallCaching(id: WorkflowId, parameters: QueryParameters, callName: Option[String]) extends ApiHandlerMessage
-  case object ApiHandlerEngineStats extends ApiHandlerMessage
-}
-
-class CromwellApiHandler(requestHandlerActor: ActorRef) extends Actor with WorkflowQueryPagination {
-  import CromwellApiHandler._
-  import WorkflowJsonSupport._
-
-  val log = Logging(context.system, classOf[CromwellApiHandler])
-  val conf = ConfigFactory.load()
-
-  def callNotFound(callFqn: String, id: WorkflowId) = {
-    RequestComplete((StatusCodes.NotFound, APIResponse.error(
-      new RuntimeException(s"Call $callFqn not found for workflow '$id'."))))
-  }
-
-  private def error(t: Throwable)(f: Throwable => RequestComplete[_]): Unit = context.parent ! f(t)
-
-  override def receive = {
-    case ApiHandlerEngineStats => requestHandlerActor ! WorkflowManagerActor.EngineStatsCommand
-    case stats: EngineStatsActor.EngineStats => context.parent ! RequestComplete((StatusCodes.OK, stats))
-    case ApiHandlerWorkflowAbort(id, manager) => requestHandlerActor ! WorkflowStoreActor.AbortWorkflow(id, manager)
-    case WorkflowStoreEngineActor.WorkflowAborted(id) =>
-      context.parent ! RequestComplete((StatusCodes.OK, WorkflowAbortResponse(id.toString, WorkflowAborted.toString)))
-    case WorkflowStoreEngineActor.WorkflowAbortFailed(_, e) =>
-      error(e) {
-        case _: IllegalStateException => RequestComplete((StatusCodes.Forbidden, APIResponse.error(e)))
-        case _: WorkflowNotFoundException => RequestComplete((StatusCodes.NotFound, APIResponse.error(e)))
-        case _ => RequestComplete((StatusCodes.InternalServerError, APIResponse.error(e)))
-      }
-
-    case ApiHandlerWorkflowSubmit(source) => requestHandlerActor ! WorkflowStoreActor.SubmitWorkflow(source)
-
-    case WorkflowStoreSubmitActor.WorkflowSubmittedToStore(id) =>
-      context.parent ! RequestComplete((StatusCodes.Created, WorkflowSubmitResponse(id.toString, WorkflowSubmitted.toString)))
-
-    case ApiHandlerWorkflowSubmitBatch(sources) => requestHandlerActor !
-      WorkflowStoreActor.BatchSubmitWorkflows(sources.map(x => WorkflowSourceFilesCollection(x.wdlSource, x.inputsJson, x.workflowOptionsJson, x.labelsJson, x.importsZipFileOption)))
-
-
-    case WorkflowStoreSubmitActor.WorkflowsBatchSubmittedToStore(ids) =>
-      val responses = ids map { id => WorkflowSubmitResponse(id.toString, WorkflowSubmitted.toString) }
-      context.parent ! RequestComplete((StatusCodes.OK, responses.toList))
-  }
-}
diff --git a/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala b/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala
index 8f690a80c..754f5d793 100644
--- a/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala
+++ b/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala
@@ -1,310 +1,300 @@
 package cromwell.webservice
 
-import akka.actor._
-import cats.data.NonEmptyList
-import com.typesafe.config.{Config, ConfigFactory}
-import cromwell.core.{WorkflowId, WorkflowOptionsJson, WorkflowSourceFilesCollection}
+import java.util.UUID
+
+import akka.actor.{ActorRef, ActorRefFactory}
+import akka.http.scaladsl.server.Directives._
+
+import scala.concurrent.{ExecutionContext, Future}
+import akka.http.scaladsl.model._
+import akka.http.scaladsl.model.Multipart.BodyPart
+import akka.stream.ActorMaterializer
 import cromwell.engine.backend.BackendConfiguration
+import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._
+import cromwell.core.{WorkflowAborted, WorkflowId, WorkflowSubmitted}
+import cromwell.core.Dispatcher.ApiDispatcher
+import cromwell.engine.workflow.workflowstore.{WorkflowStoreActor, WorkflowStoreEngineActor, WorkflowStoreSubmitActor}
+import akka.pattern.{AskTimeoutException, ask}
+import akka.util.{ByteString, Timeout}
+import net.ceedubs.ficus.Ficus._
+import cromwell.engine.workflow.WorkflowManagerActor
 import cromwell.services.metadata.MetadataService._
-import cromwell.webservice.WorkflowJsonSupport._
-import cromwell.webservice.metadata.MetadataBuilderActor
-import lenthall.spray.SwaggerUiResourceHttpService
-import spray.http.MediaTypes._
-import spray.http._
-import spray.httpx.SprayJsonSupport._
-import spray.json._
-import spray.routing._
-import wdl4s.{WdlJson, WdlSource}
-
+import cromwell.webservice.metadata.{MetadataBuilderActor, WorkflowQueryPagination}
+import cromwell.webservice.metadata.MetadataBuilderActor.{BuiltMetadataResponse, FailedMetadataResponse, MetadataBuilderActorResponse}
+import WorkflowJsonSupport._
+import akka.http.scaladsl.server.Route
+import cats.data.NonEmptyList
+import cats.data.Validated.{Invalid, Valid}
+import com.typesafe.config.ConfigFactory
+import cromwell.core.labels.Labels
+import cromwell.engine.workflow.WorkflowManagerActor.WorkflowNotFoundException
+import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheDiffActor.{BuiltCallCacheDiffResponse, CachedCallNotFoundException, CallCacheDiffActorResponse, FailedCallCacheDiffResponse}
+import cromwell.engine.workflow.lifecycle.execution.callcaching.{CallCacheDiffActor, CallCacheDiffQueryParameter}
+import cromwell.engine.workflow.workflowstore.WorkflowStoreEngineActor.WorkflowStoreEngineAbortResponse
+import cromwell.server.CromwellShutdown
+import cromwell.webservice.LabelsManagerActor._
+import lenthall.exception.AggregatedMessageException
+import spray.json.JsObject
+
+import scala.concurrent.duration._
 import scala.util.{Failure, Success, Try}
 
-trait SwaggerService extends SwaggerUiResourceHttpService {
-  override def swaggerServiceName = "cromwell"
-
-  override def swaggerUiVersion = "2.1.1"
-}
-
-trait CromwellApiService extends HttpService with PerRequestCreator {
-  def workflowManagerActor: ActorRef
-  def workflowStoreActor: ActorRef
-  def serviceRegistryActor: ActorRef
-
-  def toMap(someInput: Option[String]): Map[String, JsValue] = {
-    import spray.json._
-    someInput match {
-      case Some(inputs: String) => inputs.parseJson match {
-        case JsObject(inputMap) => inputMap
-        case _ =>
-          throw new RuntimeException(s"Submitted inputs couldn't be processed, please check for syntactical errors")
-      }
-      case None => Map.empty
-    }
-  }
-
-  def mergeMaps(allInputs: Seq[Option[String]]): JsObject = {
-    val convertToMap = allInputs.map(x => toMap(x))
-    JsObject(convertToMap reduce (_ ++ _))
-  }
+trait CromwellApiService {
+  import cromwell.webservice.CromwellApiService._
 
-  def metadataBuilderProps: Props = MetadataBuilderActor.props(serviceRegistryActor)
+  implicit def actorRefFactory: ActorRefFactory
+  implicit val materializer: ActorMaterializer
+  implicit val ec: ExecutionContext
 
-  def handleMetadataRequest(message: AnyRef): Route = {
-    requestContext =>
-      perRequest(requestContext, metadataBuilderProps, message)
-  }
+  val workflowStoreActor: ActorRef
+  val workflowManagerActor: ActorRef
+  val serviceRegistryActor: ActorRef
 
-  def handleQueryMetadataRequest(parameters: Seq[(String, String)]): Route = {
-    requestContext =>
-      perRequest(requestContext, metadataBuilderProps, WorkflowQuery(requestContext.request.uri, parameters))
-  }
+  // Derive timeouts (implicit and not) from akka http's request timeout since there's no point in being higher than that
+  implicit val duration = ConfigFactory.load().as[FiniteDuration]("akka.http.server.request-timeout")
+  implicit val timeout: Timeout = duration
 
-  protected def failBadRequest(t: Throwable, statusCode: StatusCode = StatusCodes.BadRequest) = respondWithMediaType(`application/json`) {
-    complete((statusCode, APIResponse.fail(t).toJson.prettyPrint))
-  }
-
-  val workflowRoutes = queryRoute ~ queryPostRoute ~ workflowOutputsRoute ~ submitRoute ~ submitBatchRoute ~
-    workflowLogsRoute ~ abortRoute ~ metadataRoute ~ timingRoute ~ statusRoute ~ backendRoute ~ statsRoute ~ versionRoute
-
-  protected def withRecognizedWorkflowId(possibleWorkflowId: String)(recognizedWorkflowId: WorkflowId => Route): Route = {
-    def callback(requestContext: RequestContext) = new ValidationCallback {
-      // The submitted value is malformed as a UUID and therefore not possibly recognized.
-      override def onMalformed(possibleWorkflowId: String): Unit = {
-        val exception = new RuntimeException(s"Invalid workflow ID: '$possibleWorkflowId'.")
-        failBadRequest(exception)(requestContext)
-      }
-
-      override def onUnrecognized(possibleWorkflowId: String): Unit = {
-        val exception = new RuntimeException(s"Unrecognized workflow ID: $possibleWorkflowId")
-        failBadRequest(exception, StatusCodes.NotFound)(requestContext)
-      }
-
-      override def onFailure(possibleWorkflowId: String, throwable: Throwable): Unit = {
-        val exception = new RuntimeException(s"Failed lookup attempt for workflow ID $possibleWorkflowId", throwable)
-        failBadRequest(exception)(requestContext)
-      }
-
-      override def onRecognized(workflowId: WorkflowId): Unit = {
-        recognizedWorkflowId(workflowId)(requestContext)
+  val engineRoutes = concat(
+    path("engine" / Segment / "stats") { version =>
+      get {
+        onComplete(workflowManagerActor.ask(WorkflowManagerActor.EngineStatsCommand).mapTo[EngineStatsActor.EngineStats]) {
+          case Success(stats) => complete(stats)
+          case Failure(_) => new RuntimeException("Unable to gather engine stats").failRequest(StatusCodes.InternalServerError)
+        }
       }
-    }
+    },
 
-    requestContext => {
-      val message = ValidateWorkflowIdAndExecute(possibleWorkflowId, callback(requestContext))
-      serviceRegistryActor ! message
+    path("engine" / Segment / "version") { version =>
+      get { complete(versionResponse) }
     }
-  }
+  )
 
-  def statusRoute =
+  val workflowRoutes =
+    path("workflows" / Segment / "backends") { version =>
+      get { complete(backendResponse) }
+    } ~
     path("workflows" / Segment / Segment / "status") { (version, possibleWorkflowId) =>
+      get { metadataBuilderRequest(possibleWorkflowId, (w: WorkflowId) => GetStatus(w)) }
+    } ~
+    path("workflows" / Segment / Segment / "outputs") { (version, possibleWorkflowId) =>
+      get { metadataBuilderRequest(possibleWorkflowId, (w: WorkflowId) => WorkflowOutputs(w)) }
+    } ~
+    path("workflows" / Segment / Segment / "logs") { (version, possibleWorkflowId) =>
+      get { metadataBuilderRequest(possibleWorkflowId, (w: WorkflowId) => GetLogs(w)) }
+    } ~
+    path("workflows" / Segment / "query") { _ =>
       get {
-        withRecognizedWorkflowId(possibleWorkflowId) { id =>
-          handleMetadataRequest(GetStatus(id))
+        parameterSeq { parameters =>
+          extractUri { uri =>
+            metadataQueryRequest(parameters, uri)
+          }
+        }
+      } ~
+      post {
+        entity(as[Seq[Map[String, String]]]) { parameterMap =>
+          extractUri { uri =>
+            metadataQueryRequest(parameterMap.flatMap(_.toSeq), uri)
+          }
         }
       }
-    }
-
-  def queryRoute =
-    path("workflows" / Segment / "query") { version =>
+    } ~
+    encodeResponse {
+      path("workflows" / Segment / Segment / "metadata") { (version, possibleWorkflowId) =>
+        parameters(('includeKey.*, 'excludeKey.*, 'expandSubWorkflows.as[Boolean].?)) { (includeKeys, excludeKeys, expandSubWorkflowsOption) =>
+          val includeKeysOption = NonEmptyList.fromList(includeKeys.toList)
+          val excludeKeysOption = NonEmptyList.fromList(excludeKeys.toList)
+          val expandSubWorkflows = expandSubWorkflowsOption.getOrElse(false)
+
+          (includeKeysOption, excludeKeysOption) match {
+            case (Some(_), Some(_)) =>
+              val e = new IllegalArgumentException("includeKey and excludeKey may not be specified together")
+              e.failRequest(StatusCodes.BadRequest)
+            case (_, _) => metadataBuilderRequest(possibleWorkflowId, (w: WorkflowId) => GetSingleWorkflowMetadataAction(w, includeKeysOption, excludeKeysOption, expandSubWorkflows))
+          }
+        }
+      }
+    } ~
+    path("workflows" / Segment / "callcaching" / "diff") { version =>
       parameterSeq { parameters =>
         get {
-          handleQueryMetadataRequest(parameters)
+          CallCacheDiffQueryParameter.fromParameters(parameters) match {
+            case Valid(queryParameter) =>
+              val diffActor = actorRefFactory.actorOf(CallCacheDiffActor.props(serviceRegistryActor), "CallCacheDiffActor-" + UUID.randomUUID())
+              onComplete(diffActor.ask(queryParameter).mapTo[CallCacheDiffActorResponse]) {
+                case Success(r: BuiltCallCacheDiffResponse) => complete(r.response)
+                case Success(r: FailedCallCacheDiffResponse) => r.reason.errorRequest(StatusCodes.InternalServerError)
+                case Failure(_: AskTimeoutException) if CromwellShutdown.shutdownInProgress() => serviceShuttingDownResponse
+                case Failure(e: CachedCallNotFoundException) => e.errorRequest(StatusCodes.NotFound)
+                case Failure(e) => e.errorRequest(StatusCodes.InternalServerError)
+              }
+            case Invalid(errors) =>
+              val e = AggregatedMessageException("Wrong parameters for call cache diff query", errors.toList)
+              e.errorRequest(StatusCodes.BadRequest)
+          }
         }
       }
-    }
-
-  def queryPostRoute =
-    path("workflows" / Segment / "query") { version =>
-      entity(as[Seq[Map[String, String]]]) { parameterMap =>
-        post {
-          handleQueryMetadataRequest(parameterMap.flatMap(_.toSeq))
-        }
+    } ~
+    path("workflows" / Segment / Segment / "timing") { (version, possibleWorkflowId) =>
+      onComplete(validateWorkflowId(possibleWorkflowId)) {
+        case Success(_) => getFromResource("workflowTimings/workflowTimings.html")
+        case Failure(e) => e.failRequest(StatusCodes.InternalServerError)
       }
-    }
-
-  def abortRoute =
+    } ~
     path("workflows" / Segment / Segment / "abort") { (version, possibleWorkflowId) =>
       post {
-        withRecognizedWorkflowId(possibleWorkflowId) { id =>
-          requestContext => perRequest(requestContext, CromwellApiHandler.props(workflowStoreActor), CromwellApiHandler.ApiHandlerWorkflowAbort(id, workflowManagerActor))
+        val response = validateWorkflowId(possibleWorkflowId) flatMap { w =>
+          workflowStoreActor.ask(WorkflowStoreActor.AbortWorkflow(w, workflowManagerActor)).mapTo[WorkflowStoreEngineAbortResponse]
         }
-      }
-    }
-
-  case class PartialWorkflowSources
-  (
-    wdlSource: Option[WdlSource],
-    workflowInputs: Vector[WdlJson],
-    workflowInputsAux: Map[Int, WdlJson],
-    workflowOptions: Option[WorkflowOptionsJson],
-    customLabels: Option[WdlJson],
-    zippedImports: Option[Array[Byte]])
-  object PartialWorkflowSources {
-    private def workflowInputs(bodyPart: BodyPart): Vector[WdlJson] = {
-      import spray.json._
-      bodyPart.entity.data.asString.parseJson match {
-        case JsArray(Seq(x, xs@_*)) => (Vector(x) ++ xs).map(_.compactPrint)
-        case JsArray(_) => Vector.empty
-        case v: JsValue => Vector(v.compactPrint)
-      }
-    }
-
-    def partialSourcesToSourceCollections(partialSources: Try[PartialWorkflowSources], allowNoInputs: Boolean): Try[Seq[WorkflowSourceFilesCollection]] = {
-      partialSources flatMap {
-        case PartialWorkflowSources(Some(wdlSource), workflowInputs, workflowInputsAux, workflowOptions, labels, wdlDependencies) =>
-          //The order of addition allows for the expected override of colliding keys.
-          val sortedInputAuxes = workflowInputsAux.toSeq.sortBy(_._1).map(x => Option(x._2))
-          val wfInputs: Try[Seq[WdlJson]] = if (workflowInputs.isEmpty) {
-            if (allowNoInputs) Success(Vector("{}")) else Failure(new IllegalArgumentException("No inputs were provided"))
-          } else Success(workflowInputs map { workflowInputSet =>
-            mergeMaps(Seq(Option(workflowInputSet)) ++ sortedInputAuxes).toString
-          })
-          wfInputs.map(_.map(x => WorkflowSourceFilesCollection(wdlSource, x, workflowOptions.getOrElse("{}"), labels.getOrElse("{}"), wdlDependencies)))
-        case other => Failure(new IllegalArgumentException(s"Incomplete workflow submission: $other"))
-      }
-    }
 
-    def fromSubmitRoute(formData: MultipartFormData, allowNoInputs: Boolean): Try[Seq[WorkflowSourceFilesCollection]] = {
-      val partialSources = Try(formData.fields.foldLeft(PartialWorkflowSources(None, Vector.empty, Map.empty, None, None, None)) { (partialSources: PartialWorkflowSources, bodyPart: BodyPart) =>
-        if (bodyPart.name.contains("wdlSource")) {
-          partialSources.copy(wdlSource = Some(bodyPart.entity.data.asString))
-        } else if (bodyPart.name.contains("workflowInputs")) {
-          partialSources.copy(workflowInputs = workflowInputs(bodyPart))
-        } else if (bodyPart.name.forall(_.startsWith("workflowInputs_"))) {
-          val index = bodyPart.name.get.stripPrefix("workflowInputs_").toInt
-          partialSources.copy(workflowInputsAux = partialSources.workflowInputsAux + (index -> bodyPart.entity.data.asString))
-        } else if (bodyPart.name.contains("workflowOptions")) {
-          partialSources.copy(workflowOptions = Some(bodyPart.entity.data.asString))
-        } else if (bodyPart.name.contains("wdlDependencies")) {
-          partialSources.copy(zippedImports = Some(bodyPart.entity.data.toByteArray))
-        } else if (bodyPart.name.contains("customLabels")) {
-          partialSources.copy(customLabels = Some(bodyPart.entity.data.asString))
-        } else {
-          throw new IllegalArgumentException(s"Unexpected body part name: ${bodyPart.name.getOrElse("None")}")
+        onComplete(response) {
+          case Success(WorkflowStoreEngineActor.WorkflowAborted(id)) => complete(WorkflowAbortResponse(id.toString, WorkflowAborted.toString))
+          case Success(WorkflowStoreEngineActor.WorkflowAbortFailed(_, e: IllegalStateException)) => e.errorRequest(StatusCodes.Forbidden)
+          case Success(WorkflowStoreEngineActor.WorkflowAbortFailed(_, e: WorkflowNotFoundException)) => e.errorRequest(StatusCodes.NotFound)
+          case Success(WorkflowStoreEngineActor.WorkflowAbortFailed(_, e)) => e.errorRequest(StatusCodes.InternalServerError)
+          case Failure(_: AskTimeoutException) if CromwellShutdown.shutdownInProgress() => serviceShuttingDownResponse
+          case Failure(e: UnrecognizedWorkflowException) => e.failRequest(StatusCodes.NotFound)
+          case Failure(e: InvalidWorkflowException) => e.failRequest(StatusCodes.BadRequest)
+          case Failure(e) => e.errorRequest(StatusCodes.InternalServerError)
         }
-      })
-      partialSourcesToSourceCollections(partialSources, allowNoInputs)
-    }
-  }
+      }
+    } ~
+    path("workflows" / Segment / Segment / "labels") { (version, possibleWorkflowId) =>
+      entity(as[Map[String, String]]) { parameterMap =>
+        patch {
+          Labels.validateMapOfLabels(parameterMap) match {
+            case Valid(labels) =>
+              val response = validateWorkflowId(possibleWorkflowId) flatMap { id =>
+                val lma = actorRefFactory.actorOf(LabelsManagerActor.props(serviceRegistryActor).withDispatcher(ApiDispatcher))
+                lma.ask(LabelsAddition(LabelsData(id, labels))).mapTo[LabelsManagerActorResponse]
+              }
+              onComplete(response) {
+                case Success(r: BuiltLabelsManagerResponse) => complete(r.response)
+                case Success(e: FailedLabelsManagerResponse) => e.reason.failRequest(StatusCodes.InternalServerError)
+                case Failure(e) => e.errorRequest(StatusCodes.InternalServerError)
 
-  def submitRoute =
-    path("workflows" / Segment) { version =>
-      post {
-        entity(as[MultipartFormData]) { formData =>
-          PartialWorkflowSources.fromSubmitRoute(formData, allowNoInputs = true) match {
-            case Success(workflowSourceFiles) if workflowSourceFiles.size == 1 =>
-              requestContext => {
-                perRequest(requestContext, CromwellApiHandler.props(workflowStoreActor), CromwellApiHandler.ApiHandlerWorkflowSubmit(workflowSourceFiles.head))
               }
-            case Success(workflowSourceFiles) =>
-              failBadRequest(new IllegalArgumentException("To submit more than one workflow at a time, use the batch endpoint."))
-            case Failure(t) =>
-              failBadRequest(t)
+            case Invalid(e) =>
+              val iae = new IllegalArgumentException(e.toList.mkString(","))
+              iae.failRequest(StatusCodes.BadRequest)
           }
         }
       }
-    }
-
-  def submitBatchRoute =
-    path("workflows" / Segment / "batch") { version =>
+    } ~
+  path("workflows" / Segment) { version =>
       post {
-        entity(as[MultipartFormData]) { formData =>
-          PartialWorkflowSources.fromSubmitRoute(formData, allowNoInputs = false) match {
-            case Success(workflowSourceFiles) =>
-              requestContext => {
-                perRequest(requestContext, CromwellApiHandler.props(workflowStoreActor), CromwellApiHandler.ApiHandlerWorkflowSubmitBatch(NonEmptyList.fromListUnsafe(workflowSourceFiles.toList)))
-              }
-            case Failure(t) =>
-              failBadRequest(t)
-          }
+        entity(as[Multipart.FormData]) { formData =>
+          submitRequest(formData, true)
         }
       }
+    } ~
+  path("workflows" / Segment / "batch") { version =>
+    post {
+      entity(as[Multipart.FormData]) { formData =>
+        submitRequest(formData, false)
+      }
     }
+  }
 
-  def workflowOutputsRoute =
-    path("workflows" / Segment / Segment / "outputs") { (version, possibleWorkflowId) =>
-      get {
-        withRecognizedWorkflowId(possibleWorkflowId) { id =>
-          handleMetadataRequest(WorkflowOutputs(id))
-        }
-      }
+  private def submitRequest(formData: Multipart.FormData, isSingleSubmission: Boolean): Route = {
+    val allParts: Future[Map[String, ByteString]] = formData.parts.mapAsync[(String, ByteString)](1) {
+      case b: BodyPart => b.toStrict(duration).map(strict => b.name -> strict.entity.data)
+    }.runFold(Map.empty[String, ByteString])((map, tuple) => map + tuple)
+
+    def toResponse(workflowId: WorkflowId): WorkflowSubmitResponse = {
+      WorkflowSubmitResponse(workflowId.toString, WorkflowSubmitted.toString)
     }
 
-  def workflowLogsRoute =
-    path("workflows" / Segment / Segment / "logs") { (version, possibleWorkflowId) =>
-      get {
-        withRecognizedWorkflowId(possibleWorkflowId) { id =>
-          handleMetadataRequest(GetLogs(id))
-        }
+    def askSubmit(command: WorkflowStoreActor.WorkflowStoreActorSubmitCommand): Route = {
+      // NOTE: Do not blindly coppy the akka-http -to- ask-actor pattern below without knowing the pros and cons.
+      onComplete(workflowStoreActor.ask(command).mapTo[WorkflowStoreSubmitActor.WorkflowStoreSubmitActorResponse]) {
+        case Success(w) =>
+          w match {
+            case WorkflowStoreSubmitActor.WorkflowSubmittedToStore(workflowId) =>
+              complete((StatusCodes.Created, toResponse(workflowId)))
+            case WorkflowStoreSubmitActor.WorkflowsBatchSubmittedToStore(workflowIds) =>
+              complete((StatusCodes.Created, workflowIds.toList map toResponse))
+            case WorkflowStoreSubmitActor.WorkflowSubmitFailed(throwable) =>
+              throwable.failRequest(StatusCodes.BadRequest)
+          }
+        case Failure(_: AskTimeoutException) if CromwellShutdown.shutdownInProgress() => serviceShuttingDownResponse
+        case Failure(e) =>
+          e.failRequest(StatusCodes.InternalServerError)
       }
     }
 
-  def metadataRoute =
-    path("workflows" / Segment / Segment / "metadata") { (version, possibleWorkflowId) =>
-      parameterMultiMap { parameters =>
-        val includeKeysOption = NonEmptyList.fromList(parameters.getOrElse("includeKey", List.empty))
-        val excludeKeysOption = NonEmptyList.fromList(parameters.getOrElse("excludeKey", List.empty))
-        val expandSubWorkflowsOption = {
-          parameters.get("expandSubWorkflows") match {
-            case Some(v :: Nil) => Try(v.toBoolean)
-            case _ => Success(false)
-          }
+    onComplete(allParts) {
+      case Success(data) =>
+        PartialWorkflowSources.fromSubmitRoute(data, allowNoInputs = isSingleSubmission) match {
+          case Success(workflowSourceFiles) if isSingleSubmission && workflowSourceFiles.size == 1 =>
+            askSubmit(WorkflowStoreActor.SubmitWorkflow(workflowSourceFiles.head))
+          // Catches the case where someone has gone through the single submission endpoint w/ more than one workflow
+          case Success(_) if isSingleSubmission =>
+            val e = new IllegalArgumentException("To submit more than one workflow at a time, use the batch endpoint.")
+            e.failRequest(StatusCodes.BadRequest)
+          case Success(workflowSourceFiles) =>
+            askSubmit(WorkflowStoreActor.BatchSubmitWorkflows(NonEmptyList.fromListUnsafe(workflowSourceFiles.toList)))
+          case Failure(t) => t.failRequest(StatusCodes.BadRequest)
         }
+      case Failure(e) => e.failRequest(StatusCodes.InternalServerError)
+    }
+  }
 
-        (includeKeysOption, excludeKeysOption, expandSubWorkflowsOption) match {
-          case (Some(_), Some(_), _) =>
-            failBadRequest(new IllegalArgumentException("includeKey and excludeKey may not be specified together"))
-          case (_, _, Success(expandSubWorkflows)) =>
-            withRecognizedWorkflowId(possibleWorkflowId) { id =>
-              handleMetadataRequest(GetSingleWorkflowMetadataAction(id, includeKeysOption, excludeKeysOption, expandSubWorkflows))
-            }
-          case (_, _, Failure(ex)) => failBadRequest(new IllegalArgumentException(ex))
+  private def validateWorkflowId(possibleWorkflowId: String): Future[WorkflowId] = {
+    Try(WorkflowId.fromString(possibleWorkflowId)) match {
+      case Success(w) =>
+        serviceRegistryActor.ask(ValidateWorkflowId(w)).mapTo[WorkflowValidationResponse] map {
+          case RecognizedWorkflowId => w
+          case UnrecognizedWorkflowId => throw UnrecognizedWorkflowException(s"Unrecognized workflow ID: $w")
+          case FailedToCheckWorkflowId(t) => throw t
         }
-      }
+      case Failure(_) => Future.failed(InvalidWorkflowException(s"Invalid workflow ID: '$possibleWorkflowId'."))
     }
+  }
 
-  def timingRoute =
-    path("workflows" / Segment / Segment / "timing") { (version, possibleWorkflowId) =>
-      withRecognizedWorkflowId(possibleWorkflowId) { id =>
-        getFromResource("workflowTimings/workflowTimings.html")
-      }
+  private def metadataBuilderRequest(possibleWorkflowId: String, request: WorkflowId => ReadAction): Route = {
+    val metadataBuilderActor = actorRefFactory.actorOf(MetadataBuilderActor.props(serviceRegistryActor).withDispatcher(ApiDispatcher), MetadataBuilderActor.uniqueActorName)
+    val response = validateWorkflowId(possibleWorkflowId) flatMap { w => metadataBuilderActor.ask(request(w)).mapTo[MetadataBuilderActorResponse] }
+
+    onComplete(response) {
+      case Success(r: BuiltMetadataResponse) => complete(r.response)
+      case Success(r: FailedMetadataResponse) => r.reason.errorRequest(StatusCodes.InternalServerError)
+      case Failure(_: AskTimeoutException) if CromwellShutdown.shutdownInProgress() => serviceShuttingDownResponse
+      case Failure(e: UnrecognizedWorkflowException) => e.failRequest(StatusCodes.NotFound)
+      case Failure(e: InvalidWorkflowException) => e.failRequest(StatusCodes.BadRequest)
+      case Failure(e) => e.errorRequest(StatusCodes.InternalServerError)
     }
+  }
 
-  def statsRoute =
-    path("engine" / Segment / "stats") { version =>
-      get {
-        requestContext =>
-          perRequest(requestContext, CromwellApiHandler.props(workflowManagerActor), CromwellApiHandler.ApiHandlerEngineStats)
-      }
-    }
+  protected[this] def metadataQueryRequest(parameters: Seq[(String, String)], uri: Uri): Route = {
+    val response = serviceRegistryActor.ask(WorkflowQuery(parameters)).mapTo[MetadataQueryResponse]
 
-  def versionRoute =
-    path("engine" / Segment / "version") { version =>
-      get {
-        complete {
-          lazy val versionConf = ConfigFactory.load("cromwell-version.conf").getConfig("version")
-          versionResponse(versionConf)
+    onComplete(response) {
+      case Success(w: WorkflowQuerySuccess) =>
+        val headers = WorkflowQueryPagination.generateLinkHeaders(uri, w.meta)
+        respondWithHeaders(headers) {
+          complete(w.response)
         }
-      }
+      case Success(w: WorkflowQueryFailure) => w.reason.failRequest(StatusCodes.BadRequest)
+      case Failure(_: AskTimeoutException) if CromwellShutdown.shutdownInProgress() => serviceShuttingDownResponse
+      case Failure(e) => e.errorRequest(StatusCodes.InternalServerError)
     }
+  }
+}
 
-  def versionResponse(versionConf: Config) = JsObject(Map(
-    "cromwell" -> versionConf.getString("cromwell").toJson
-  ))
+object CromwellApiService {
+  import spray.json._
 
-  def backendRoute =
-    path("workflows" / Segment / "backends") { version =>
-      get {
-        complete {
-          // Note that this is not using our standard per-request scheme, since the result is pre-calculated already
-          backendResponse
-        }
-      }
-    }
+  implicit class EnhancedThrowable(val e: Throwable) extends AnyVal {
+    def failRequest(statusCode: StatusCode): Route = complete((statusCode, APIResponse.fail(e).toJson.prettyPrint))
+    def errorRequest(statusCode: StatusCode): Route = complete((statusCode, APIResponse.error(e).toJson.prettyPrint))
+  }
 
-  val backendResponse = JsObject(Map(
-    "supportedBackends" -> BackendConfiguration.AllBackendEntries.map(_.name).sorted.toJson,
-    "defaultBackend" -> BackendConfiguration.DefaultBackendEntry.name.toJson
-  ))
+  final case class BackendResponse(supportedBackends: List[String], defaultBackend: String)
 
-}
+  final case class UnrecognizedWorkflowException(message: String) extends Exception(message)
+  final case class InvalidWorkflowException(message: String) extends Exception(message)
 
+  val backendResponse = BackendResponse(BackendConfiguration.AllBackendEntries.map(_.name).sorted, BackendConfiguration.DefaultBackendEntry.name)
+  val versionResponse = JsObject(Map("cromwell" -> ConfigFactory.load("cromwell-version.conf").getConfig("version").getString("cromwell").toJson))
+  val serviceShuttingDownResponse = new Exception("Cromwell service is shutting down.").failRequest(StatusCodes.ServiceUnavailable)
+}
diff --git a/engine/src/main/scala/cromwell/webservice/LabelsManagerActor.scala b/engine/src/main/scala/cromwell/webservice/LabelsManagerActor.scala
new file mode 100644
index 000000000..7a950ba01
--- /dev/null
+++ b/engine/src/main/scala/cromwell/webservice/LabelsManagerActor.scala
@@ -0,0 +1,68 @@
+package cromwell.webservice
+
+import akka.actor.{Actor, ActorLogging, ActorRef, Props}
+import cromwell.core.labels.Labels
+import cromwell.core.{Dispatcher, WorkflowId, WorkflowMetadataKeys}
+import cromwell.services.metadata.{MetadataEvent, MetadataKey, MetadataValue}
+import cromwell.services.metadata.MetadataService._
+import cromwell.webservice.LabelsManagerActor._
+import spray.json.{DefaultJsonProtocol, JsObject, JsString}
+
+import scala.language.postfixOps
+
+
+object LabelsManagerActor {
+
+  def props(serviceRegistryActor: ActorRef) = Props(new LabelsManagerActor(serviceRegistryActor)).withDispatcher(Dispatcher.ApiDispatcher)
+
+  final case class LabelsData(workflowId: WorkflowId, labels: Labels)
+
+  sealed trait LabelsMessage {
+    def data: LabelsData
+  }
+
+  sealed trait LabelsAction extends LabelsMessage
+  final case class LabelsAddition(data: LabelsData) extends LabelsAction
+
+  sealed trait LabelsResponse extends LabelsMessage
+
+  def processLabelsResponse(workflowId: WorkflowId, labels: Map[String, String]): JsObject = {
+    JsObject(Map(
+      WorkflowMetadataKeys.Id -> JsString(workflowId.toString),
+      WorkflowMetadataKeys.Labels -> JsObject(labels mapValues JsString.apply)
+    ))
+  }
+
+  def metadataEventsToLabels(events: Iterable[MetadataEvent]): Map[String, String] = {
+    events collect { case MetadataEvent(MetadataKey(_, _, key), Some(MetadataValue(value, _)), _) => key.split("\\:").last -> value } toMap
+  }
+
+  def labelsToMetadataEvents(labels: Labels, workflowId: WorkflowId): Iterable[MetadataEvent] = {
+    labels.value map { l => MetadataEvent(MetadataKey(workflowId, None, s"${WorkflowMetadataKeys.Labels}:${l.key}"), MetadataValue(l.value)) }
+  }
+
+  sealed abstract class LabelsManagerActorResponse
+  final case class BuiltLabelsManagerResponse(response: JsObject) extends LabelsManagerActorResponse
+  final case class FailedLabelsManagerResponse(reason: Throwable) extends LabelsManagerActorResponse
+}
+
+class LabelsManagerActor(serviceRegistryActor: ActorRef) extends Actor with ActorLogging with DefaultJsonProtocol {
+
+  implicit val ec = context.dispatcher
+
+  private var wfId: Option[WorkflowId] = None
+  private var target: ActorRef = ActorRef.noSender
+
+  def receive = {
+    case LabelsAddition(data) =>
+      wfId = Option(data.workflowId)
+      target = sender()
+      serviceRegistryActor ! PutMetadataActionAndRespond(labelsToMetadataEvents(data.labels, data.workflowId), self)
+    case MetadataWriteSuccess(events) =>
+      target ! BuiltLabelsManagerResponse(processLabelsResponse(wfId.get, metadataEventsToLabels(events)))
+      context stop self
+    case MetadataWriteFailure(failure, events @ _) =>
+      target ! FailedLabelsManagerResponse(new RuntimeException(s"Unable to update labels for ${wfId.get} due to ${failure.getMessage}"))
+      context stop self
+  }
+}
diff --git a/engine/src/main/scala/cromwell/webservice/PartialWorkflowSources.scala b/engine/src/main/scala/cromwell/webservice/PartialWorkflowSources.scala
new file mode 100644
index 000000000..b4e1a82dd
--- /dev/null
+++ b/engine/src/main/scala/cromwell/webservice/PartialWorkflowSources.scala
@@ -0,0 +1,142 @@
+package cromwell.webservice
+
+import akka.util.ByteString
+import cromwell.core.{WorkflowOptions, WorkflowOptionsJson, WorkflowSourceFilesCollection}
+import wdl4s.wdl.{WorkflowJson, WorkflowSource}
+import cats.data.Validated.{Invalid, Valid}
+import cats.syntax.validated._
+import cats.syntax.cartesian._
+import lenthall.validation.ErrorOr.ErrorOr
+import cromwell.core._
+import org.slf4j.LoggerFactory
+import spray.json.{JsObject, JsValue}
+
+import scala.util.Try
+
+final case class PartialWorkflowSources(workflowSource: Option[WorkflowSource],
+                                        workflowType: Option[WorkflowType],
+                                        workflowTypeVersion: Option[WorkflowTypeVersion],
+                                        workflowInputs: Vector[WorkflowJson],
+                                        workflowInputsAux: Map[Int, WorkflowJson],
+                                        workflowOptions: Option[WorkflowOptionsJson],
+                                        customLabels: Option[WorkflowJson],
+                                        zippedImports: Option[Array[Byte]])
+
+object PartialWorkflowSources {
+  val log = LoggerFactory.getLogger(classOf[PartialWorkflowSources])
+
+  def empty = PartialWorkflowSources(
+    workflowSource = None,
+    // TODO do not hardcode, especially not out here at the boundary layer good gravy
+    workflowType = Option("WDL"),
+    workflowTypeVersion = None,
+    workflowInputs = Vector.empty,
+    workflowInputsAux = Map.empty,
+    workflowOptions = None,
+    customLabels = None,
+    zippedImports = None
+  )
+
+  def fromSubmitRoute(formData: Map[String, ByteString], allowNoInputs: Boolean): Try[Seq[WorkflowSourceFilesCollection]] = {
+    val partialSources = Try(formData.foldLeft(PartialWorkflowSources.empty) { (partialSources: PartialWorkflowSources, kv: (String, ByteString)) =>
+      val name = kv._1
+      val data = kv._2
+
+      if (name == "wdlSource" || name == "workflowSource") {
+        if (name == "wdlSource") deprecationWarning(out = "wdlSource", in = "workflowSource")
+        partialSources.copy(workflowSource = Option(data.utf8String))
+      } else if (name == "workflowType") {
+        partialSources.copy(workflowType = Option(data.utf8String))
+      } else if (name == "workflowTypeVersion") {
+        partialSources.copy(workflowTypeVersion = Option(data.utf8String))
+      } else if (name == "workflowInputs") {
+        partialSources.copy(workflowInputs = workflowInputs(data.utf8String))
+      } else if (name.startsWith("workflowInputs_")) {
+        val index = name.stripPrefix("workflowInputs_").toInt
+        partialSources.copy(workflowInputsAux = partialSources.workflowInputsAux + (index -> data.utf8String))
+      } else if (name == "workflowOptions") {
+        partialSources.copy(workflowOptions = Option(data.utf8String))
+      } else if (name == "wdlDependencies" || name == "workflowDependencies") {
+        if (name == "wdlDependencies") deprecationWarning(out = "wdlDependencies", in = "workflowDependencies")
+        partialSources.copy(zippedImports = Option(data.toArray))
+      } else if (name == "customLabels") {
+        partialSources.copy(customLabels = Option(data.utf8String))
+      } else {
+        throw new IllegalArgumentException(s"Unexpected body part name: $name")
+      }
+    })
+
+    partialSourcesToSourceCollections(partialSources.tryToErrorOr, allowNoInputs).errorOrToTry
+  }
+
+  private def workflowInputs(data: String): Vector[WorkflowJson] = {
+    import spray.json._
+    data.parseJson match {
+      case JsArray(Seq(x, xs@_*)) => (Vector(x) ++ xs).map(_.compactPrint)
+      case JsArray(_) => Vector.empty
+      case v: JsValue => Vector(v.compactPrint)
+    }
+  }
+
+  private def partialSourcesToSourceCollections(partialSources: ErrorOr[PartialWorkflowSources], allowNoInputs: Boolean): ErrorOr[Seq[WorkflowSourceFilesCollection]] = {
+    def validateInputs(pws: PartialWorkflowSources): ErrorOr[Seq[WorkflowJson]] =
+      (pws.workflowInputs.isEmpty, allowNoInputs) match {
+        case (true, true) => Vector("{}").validNel
+        case (true, false) => "No inputs were provided".invalidNel
+        case _ =>
+          val sortedInputAuxes = pws.workflowInputsAux.toSeq.sortBy { case (index, _) => index } map { case(_, inputJson) => Option(inputJson) }
+          (pws.workflowInputs map { workflowInputSet: WorkflowJson => mergeMaps(Seq(Option(workflowInputSet)) ++ sortedInputAuxes).toString }).validNel
+      }
+
+    def validateOptions(options: Option[WorkflowOptionsJson]): ErrorOr[WorkflowOptions] =
+      WorkflowOptions.fromJsonString(options.getOrElse("{}")).tryToErrorOr leftMap { _ map { i => s"Invalid workflow options provided: $i" } }
+
+    def validateWorkflowSource(partialSource: PartialWorkflowSources): ErrorOr[WorkflowJson] = partialSource.workflowSource match {
+      case Some(src) => src.validNel
+      case _ => s"Incomplete workflow submission: $partialSource".invalidNel
+    }
+
+    partialSources match {
+      case Valid(partialSource) =>
+        (validateWorkflowSource(partialSource) |@| validateInputs(partialSource) |@| validateOptions(partialSource.workflowOptions)) map {
+          case (wfSource, wfInputs, wfOptions) =>
+            wfInputs.map(inputsJson => WorkflowSourceFilesCollection(
+              workflowSource = wfSource,
+              workflowType = partialSource.workflowType,
+              workflowTypeVersion = partialSource.workflowTypeVersion,
+              inputsJson = inputsJson,
+              workflowOptionsJson = wfOptions.asPrettyJson,
+              labelsJson = partialSource.customLabels.getOrElse("{}"),
+              importsFile = partialSource.zippedImports))        }
+      case Invalid(err) => err.invalid
+    }
+  }
+
+  private def deprecationWarning(out: String, in: String): Unit = {
+    val warning =
+      s"""
+         |The '$out' parameter name has been deprecated in favor of '$in'.
+         |Support for '$out' will be removed from future versions of Cromwell.
+         |Please switch to using '$in' in future submissions.
+         """.stripMargin
+    log.warn(warning)
+  }
+
+  def mergeMaps(allInputs: Seq[Option[String]]): JsObject = {
+    val convertToMap = allInputs.map(x => toMap(x))
+    JsObject(convertToMap reduce (_ ++ _))
+  }
+
+  private def toMap(someInput: Option[String]): Map[String, JsValue] = {
+    import spray.json._
+    someInput match {
+      case Some(inputs: String) => inputs.parseJson match {
+        case JsObject(inputMap) => inputMap
+        case _ =>
+          throw new RuntimeException(s"Submitted inputs couldn't be processed, please check for syntactical errors")
+      }
+      case None => Map.empty
+    }
+  }
+}
+
diff --git a/engine/src/main/scala/cromwell/webservice/PerRequest.scala b/engine/src/main/scala/cromwell/webservice/PerRequest.scala
deleted file mode 100644
index f4fb9e876..000000000
--- a/engine/src/main/scala/cromwell/webservice/PerRequest.scala
+++ /dev/null
@@ -1,120 +0,0 @@
-package cromwell.webservice
-
-import java.util.UUID
-
-import akka.actor.SupervisorStrategy.Stop
-import akka.actor.{OneForOneStrategy, _}
-import cromwell.core.Dispatcher.ApiDispatcher
-import cromwell.webservice.PerRequest._
-import spray.http.StatusCodes._
-import spray.http._
-import spray.httpx.marshalling.ToResponseMarshaller
-import spray.routing.RequestContext
-
-import scala.concurrent.duration._
-import scala.language.postfixOps
-
-/**
- * This actor controls the lifecycle of a request. It is responsible for forwarding the initial message
- * to a target handling actor. This actor waits for the target actor to signal completion (via a message),
- * timeout, or handle an exception. It is this actors responsibility to respond to the request and
- * shutdown itself and child actors.
- *
- * Request completion can be signaled in 2 ways:
- * 1) with just a response object
- * 2) with a RequestComplete message which can specify http status code as well as the response
- */
-trait PerRequest extends Actor {
-  import context._
-
-  def r: RequestContext
-  def target: ActorRef
-  def message: AnyRef
-  def timeout: Duration
-
-  setReceiveTimeout(timeout)
-  target ! message
-
-  def receive = {
-    // The [Any] type parameter appears to be required for version of Scala > 2.11.2,
-    // the @ unchecked is required to muzzle erasure warnings.
-    case message: RequestComplete[Any] @ unchecked => complete(message.response)(message.marshaller)
-    case message: RequestCompleteWithHeaders[Any] @ unchecked => complete(message.response, message.headers:_*)(message.marshaller)
-    case ReceiveTimeout => complete(GatewayTimeout)
-    case x =>
-      system.log.error("Unsupported response message sent to PreRequest actor: " + Option(x).getOrElse("null").toString)
-      complete(InternalServerError)
-  }
-
-  /**
-   * Complete the request sending the given response and status code
-   * @param response to send to the caller
-   * @param marshaller to use for marshalling the response
-   * @tparam T the type of the response
-   * @return
-   */
-  private def complete[T](response: T, headers: HttpHeader*)(implicit marshaller: ToResponseMarshaller[T]) = {
-    val additionalHeaders = None
-    r.withHttpResponseHeadersMapped(h => h ++ headers ++ additionalHeaders).complete(response)
-    stop(self)
-  }
-
-  override val supervisorStrategy =
-    OneForOneStrategy() {
-      case e =>
-        system.log.error(e, "error processing request: " + r.request.uri)
-        r.complete((InternalServerError, e.getMessage))
-        Stop
-    }
-}
-
-object PerRequest {
-  sealed trait PerRequestMessage
-  /**
-   * Report complete, follows same pattern as spray.routing.RequestContext.complete; examples of how to call
-   * that method should apply here too. E.g. even though this method has only one parameter, it can be called
-   * with 2 where the first is a StatusCode: RequestComplete(StatusCode.Created, response)
-   */
-  case class RequestComplete[T](response: T)(implicit val marshaller: ToResponseMarshaller[T]) extends PerRequestMessage
-
-  /**
-   * Report complete with response headers. To response with a special status code the first parameter can be a
-   * tuple where the first element is StatusCode: RequestCompleteWithHeaders((StatusCode.Created, results), header).
-   * Note that this is here so that RequestComplete above can behave like spray.routing.RequestContext.complete.
-   */
-  case class RequestCompleteWithHeaders[T](response: T, headers: HttpHeader*)(implicit val marshaller: ToResponseMarshaller[T]) extends PerRequestMessage
-
-  /** allows for pattern matching with extraction of marshaller */
-  object RequestComplete_ {
-    def unapply[T](requestComplete: RequestComplete[T]) = Option((requestComplete.response, requestComplete.marshaller))
-  }
-
-  /** allows for pattern matching with extraction of marshaller */
-  object RequestCompleteWithHeaders_ {
-    def unapply[T](requestComplete: RequestCompleteWithHeaders[T]) = Option((requestComplete.response, requestComplete.headers, requestComplete.marshaller))
-  }
-
-  case class WithProps(r: RequestContext, props: Props, message: AnyRef, timeout: Duration, name: String) extends PerRequest {
-    lazy val target = context.actorOf(props.withDispatcher(ApiDispatcher), name)
-  }
-}
-
-/**
- * Provides factory methods for creating per request actors
- */
-trait PerRequestCreator {
-  implicit def actorRefFactory: ActorRefFactory
-
-  def perRequest(r: RequestContext,
-                 props: Props, message: AnyRef,
-                 timeout: Duration = 1 minutes,
-                 name: String = PerRequestCreator.endpointActorName): Unit = {
-    actorRefFactory.actorOf(Props(WithProps(r, props, message, timeout, name)).withDispatcher(ApiDispatcher), name)
-    ()
-  }
-}
-
-object PerRequestCreator {
-  // This scheme was changed away from the Agora System.nanoTime approach due to actor naming collisions (!)
-  def endpointActorName = List("Endpoint", java.lang.Thread.currentThread.getStackTrace()(1).getMethodName, UUID.randomUUID()).mkString("-")
-}
diff --git a/engine/src/main/scala/cromwell/webservice/SwaggerService.scala b/engine/src/main/scala/cromwell/webservice/SwaggerService.scala
new file mode 100644
index 000000000..0af4edd9b
--- /dev/null
+++ b/engine/src/main/scala/cromwell/webservice/SwaggerService.scala
@@ -0,0 +1,7 @@
+package cromwell.webservice
+
+trait SwaggerService extends SwaggerUiResourceHttpService {
+  override def swaggerServiceName = "cromwell"
+
+  override def swaggerUiVersion = "2.1.1"
+}
diff --git a/engine/src/main/scala/cromwell/webservice/SwaggerUiHttpService.scala b/engine/src/main/scala/cromwell/webservice/SwaggerUiHttpService.scala
new file mode 100644
index 000000000..ea0eb98ff
--- /dev/null
+++ b/engine/src/main/scala/cromwell/webservice/SwaggerUiHttpService.scala
@@ -0,0 +1,161 @@
+package cromwell.webservice
+
+import akka.http.scaladsl.model.StatusCodes
+import akka.http.scaladsl.server.Route
+import com.typesafe.config.Config
+import net.ceedubs.ficus.Ficus._
+import akka.http.scaladsl.server.Directives._
+
+/**
+ * Serves up the swagger UI from org.webjars/swagger-ui.
+ */
+trait SwaggerUiHttpService {
+  /**
+   * @return The version of the org.webjars/swagger-ui artifact. For example "2.1.1".
+   */
+  def swaggerUiVersion: String
+
+  /**
+   * Informs the swagger UI of the base of the application url, as hosted on the server.
+   * If your entire app is served under "http://myserver/myapp", then the base URL is "/myapp".
+   * If the app is served at the root of the application, leave this value as the empty string.
+   *
+   * @return The base URL used by the application, or the empty string if there is no base URL. For example "/myapp".
+   */
+  def swaggerUiBaseUrl: String = ""
+
+  /**
+   * @return The path to the swagger UI html documents. For example "swagger"
+   */
+  def swaggerUiPath: String = "swagger"
+
+  /**
+   * The path to the actual swagger documentation in either yaml or json, to be rendered by the swagger UI html.
+   *
+   * @return The path to the api documentation to render in the swagger UI.
+   *         For example "api-docs" or "swagger/lenthall.yaml".
+   */
+  def swaggerUiDocsPath: String = "api-docs"
+
+  /**
+   * @return When true, if someone requests / (or /baseUrl if setup), redirect to the swagger UI.
+   */
+  def swaggerUiFromRoot: Boolean = true
+
+  private def routeFromRoot: Route = get {
+    pathEndOrSingleSlash {
+      // Redirect / to the swagger UI
+      redirect(s"$swaggerUiBaseUrl/$swaggerUiPath", StatusCodes.TemporaryRedirect)
+    }
+  }
+
+  /**
+   * Serves up the swagger UI only. Redirects requests to the root of the UI path to the index.html.
+   *
+   * @return Route serving the swagger UI.
+   */
+  final def swaggerUiRoute: Route = {
+    val route = get {
+      pathPrefix(separateOnSlashes(swaggerUiPath)) {
+        // when the user hits the doc url, redirect to the index.html with api docs specified on the url
+        pathEndOrSingleSlash {
+          redirect(
+            s"$swaggerUiBaseUrl/$swaggerUiPath/index.html?url=$swaggerUiBaseUrl/$swaggerUiDocsPath",
+            StatusCodes.TemporaryRedirect)
+        } ~ getFromResourceDirectory(s"META-INF/resources/webjars/swagger-ui/$swaggerUiVersion")
+      }
+    }
+    if (swaggerUiFromRoot) route ~ routeFromRoot else route
+  }
+
+}
+
+/**
+ * Extends the SwaggerUiHttpService to gets UI configuration values from a provided Typesafe Config.
+ */
+trait SwaggerUiConfigHttpService extends SwaggerUiHttpService {
+  /**
+   * @return The swagger UI config.
+   */
+  def swaggerUiConfig: Config
+
+  override def swaggerUiVersion = swaggerUiConfig.getString("uiVersion")
+
+  abstract override def swaggerUiBaseUrl = swaggerUiConfig.as[Option[String]]("baseUrl").getOrElse(super.swaggerUiBaseUrl)
+
+  abstract override def swaggerUiPath = swaggerUiConfig.as[Option[String]]("uiPath").getOrElse(super.swaggerUiPath)
+
+  abstract override def swaggerUiDocsPath = swaggerUiConfig.as[Option[String]]("docsPath").getOrElse(super.swaggerUiDocsPath)
+}
+
+/**
+ * An extension of HttpService to serve up a resource containing the swagger api as yaml or json. The resource
+ * directory and path on the classpath must match the path for route. The resource can be any file type supported by the
+ * swagger UI, but defaults to "yaml". This is an alternative to spray-swagger's SwaggerHttpService.
+ */
+trait SwaggerResourceHttpService {
+  /**
+   * @return The directory for the resource under the classpath, and in the url
+   */
+  def swaggerDirectory: String = "swagger"
+
+  /**
+   * @return Name of the service, used to map the documentation resource at "/uiPath/serviceName.resourceType".
+   */
+  def swaggerServiceName: String
+
+  /**
+   * @return The type of the resource, usually "yaml" or "json".
+   */
+  def swaggerResourceType: String = "yaml"
+
+  /**
+   * Swagger UI sends HTTP OPTIONS before ALL requests, and expects a status 200 / OK. When true (the default) the
+   * swaggerResourceRoute will return 200 / OK for requests for OPTIONS.
+   *
+   * See also:
+   * - https://github.com/swagger-api/swagger-ui/issues/1209
+   * - https://github.com/swagger-api/swagger-ui/issues/161
+   * - https://groups.google.com/forum/#!topic/swagger-swaggersocket/S6_I6FBjdZ8
+   *
+   * @return True if status code 200 should be returned for HTTP OPTIONS requests for the swagger resource.
+   */
+  def swaggerAllOptionsOk: Boolean = true
+
+  /**
+   * @return The path to the swagger docs.
+   */
+  protected def swaggerDocsPath = s"$swaggerDirectory/$swaggerServiceName.$swaggerResourceType"
+
+  /**
+   * @return A route that returns the swagger resource.
+   */
+  final def swaggerResourceRoute: Route = {
+    val swaggerDocsDirective = path(separateOnSlashes(swaggerDocsPath))
+    val route = get {
+      swaggerDocsDirective {
+        // Return /uiPath/serviceName.resourceType from the classpath resources.
+        getFromResource(swaggerDocsPath)
+      }
+    }
+
+    if (swaggerAllOptionsOk) {
+      route ~ options {
+        // Also return status 200 / OK for all OPTIONS requests.
+        complete(StatusCodes.OK)
+      }
+    } else route
+  }
+}
+
+/**
+ * Extends the SwaggerUiHttpService and SwaggerResourceHttpService to serve up both.
+ */
+trait SwaggerUiResourceHttpService extends SwaggerUiHttpService with SwaggerResourceHttpService {
+  override def swaggerUiDocsPath = swaggerDocsPath
+
+  /**
+   * @return A route that redirects to the swagger UI and returns the swagger resource.
+   */
+  final def swaggerUiResourceRoute: Route = swaggerUiRoute ~ swaggerResourceRoute
+}
diff --git a/engine/src/main/scala/cromwell/webservice/WorkflowJsonSupport.scala b/engine/src/main/scala/cromwell/webservice/WorkflowJsonSupport.scala
index 817cb8dcf..a813278dc 100644
--- a/engine/src/main/scala/cromwell/webservice/WorkflowJsonSupport.scala
+++ b/engine/src/main/scala/cromwell/webservice/WorkflowJsonSupport.scala
@@ -6,10 +6,12 @@ import java.time.OffsetDateTime
 import cromwell.core._
 import cromwell.engine._
 import cromwell.services.metadata.MetadataService
-import MetadataService.{WorkflowQueryResponse, WorkflowQueryResult}
+import MetadataService._
 import cromwell.util.JsonFormatting.WdlValueJsonFormatter
 import WdlValueJsonFormatter._
 import better.files.File
+import cromwell.webservice.CromwellApiService.BackendResponse
+import cromwell.webservice.metadata.MetadataBuilderActor.BuiltMetadataResponse
 import spray.json.{DefaultJsonProtocol, JsString, JsValue, RootJsonFormat}
 
 object WorkflowJsonSupport extends DefaultJsonProtocol {
@@ -19,8 +21,10 @@ object WorkflowJsonSupport extends DefaultJsonProtocol {
   implicit val workflowOutputResponseProtocol = jsonFormat2(WorkflowOutputResponse)
   implicit val callOutputResponseProtocol = jsonFormat3(CallOutputResponse)
   implicit val engineStatsProtocol = jsonFormat2(EngineStatsActor.EngineStats)
+  implicit val BackendResponseFormat = jsonFormat2(BackendResponse)
+  implicit val BuiltStatusResponseFormat = jsonFormat1(BuiltMetadataResponse)
   implicit val callAttempt = jsonFormat2(CallAttempt)
-  implicit val workflowSourceData = jsonFormat4(WorkflowSourceFilesWithoutImports)
+  implicit val workflowSourceData = jsonFormat6(WorkflowSourceFilesWithoutImports)
 
   implicit object fileJsonFormat extends RootJsonFormat[File] {
     override def write(obj: File) = JsString(obj.path.toAbsolutePath.toString)
@@ -30,7 +34,7 @@ object WorkflowJsonSupport extends DefaultJsonProtocol {
     }
   }
 
-  implicit val workflowSourceDataWithImports = jsonFormat5(WorkflowSourceFilesWithDependenciesZip)
+  implicit val workflowSourceDataWithImports = jsonFormat7(WorkflowSourceFilesWithDependenciesZip)
   implicit val errorResponse = jsonFormat3(FailureResponse)
   implicit val successResponse = jsonFormat3(SuccessResponse)
 
@@ -43,7 +47,6 @@ object WorkflowJsonSupport extends DefaultJsonProtocol {
     }
   }
 
-  implicit val unqualifiedFailureEventEntry = jsonFormat2(FailureEventEntry)
   implicit val workflowQueryResult = jsonFormat5(WorkflowQueryResult)
   implicit val workflowQueryResponse = jsonFormat1(WorkflowQueryResponse)
 }
diff --git a/engine/src/main/scala/cromwell/webservice/metadata/IndexedJsonValue.scala b/engine/src/main/scala/cromwell/webservice/metadata/IndexedJsonValue.scala
deleted file mode 100644
index f51e64187..000000000
--- a/engine/src/main/scala/cromwell/webservice/metadata/IndexedJsonValue.scala
+++ /dev/null
@@ -1,63 +0,0 @@
-package cromwell.webservice.metadata
-
-import java.time.OffsetDateTime
-
-import cats.{Monoid, Semigroup}
-import cats.instances.map._
-import cromwell.services.metadata.CallMetadataKeys
-import spray.json._
-
-
-object IndexedJsonValue {
-  private implicit val dateTimeOrdering: Ordering[OffsetDateTime] = scala.Ordering.fromLessThan(_ isBefore _)
-  private val timestampedJsValueOrdering: Ordering[TimestampedJsValue] = scala.Ordering.by(_.timestamp)
-
-  implicit val TimestampedJsonMonoid: Monoid[TimestampedJsValue] = new Monoid[TimestampedJsValue] {
-    def combine(f1: TimestampedJsValue, f2: TimestampedJsValue): TimestampedJsValue = {
-      (f1, f2) match {
-        case (o1: TimestampedJsObject, o2: TimestampedJsObject) =>
-          val sg = implicitly[Semigroup[Map[String, TimestampedJsValue]]]
-          TimestampedJsObject(sg.combine(o1.v, o2.v), dateTimeOrdering.max(o1.timestamp, o2.timestamp))
-        case (o1: TimestampedJsList, o2: TimestampedJsList) =>
-          val sg = implicitly[Semigroup[Map[Int, TimestampedJsValue]]]
-          TimestampedJsList(sg.combine(o1.v, o2.v), dateTimeOrdering.max(o1.timestamp, o2.timestamp))
-        case (o1, o2) => timestampedJsValueOrdering.max(o1, o2)
-      }
-    }
-
-    override def empty: TimestampedJsValue = TimestampedJsObject(Map.empty, OffsetDateTime.now)
-  }
-}
-
-/** Customized version of Json data structure, to account for timestamped values and lazy array creation */
-sealed trait TimestampedJsValue {
-  def toJson(expandedValues: Map[String, JsValue]): JsValue
-  def timestamp: OffsetDateTime
-}
-
-private case class TimestampedJsList(v: Map[Int, TimestampedJsValue], timestamp: OffsetDateTime) extends TimestampedJsValue {
-  override def toJson(expandedValues: Map[String, JsValue]) = JsArray(v.values.toVector map { _.toJson(expandedValues) })
-}
-
-private case class TimestampedJsObject(v: Map[String, TimestampedJsValue], timestamp: OffsetDateTime) extends TimestampedJsValue {
-  override def toJson(expandedValues: Map[String, JsValue]) = {
-    val mappedValues = v map {
-      case (key, subWorkflowId: TimestampedJsPrimitive) if key == CallMetadataKeys.SubWorkflowId =>
-        val subId = subWorkflowId.v.asInstanceOf[JsString]
-        expandedValues.get(subId.value) map { subMetadata =>
-           CallMetadataKeys.SubWorkflowMetadata -> subMetadata
-        } getOrElse {
-          key -> subWorkflowId.v
-        }
-      case (key, value) => key -> value.toJson(expandedValues)
-    }
-    
-    JsObject(mappedValues)
-  }
-}
-
-private class TimestampedJsPrimitive(val v: JsValue, val timestamp: OffsetDateTime) extends TimestampedJsValue {
-  override def toJson(expandedValues: Map[String, JsValue]) = v
-}
-
-private case class TimestampedEmptyJson(override val timestamp: OffsetDateTime) extends TimestampedJsPrimitive(JsObject(Map.empty[String, JsValue]), timestamp)
\ No newline at end of file
diff --git a/engine/src/main/scala/cromwell/webservice/metadata/MetadataBuilderActor.scala b/engine/src/main/scala/cromwell/webservice/metadata/MetadataBuilderActor.scala
index 272e94c75..30ec2757a 100644
--- a/engine/src/main/scala/cromwell/webservice/metadata/MetadataBuilderActor.scala
+++ b/engine/src/main/scala/cromwell/webservice/metadata/MetadataBuilderActor.scala
@@ -1,159 +1,64 @@
 package cromwell.webservice.metadata
 
-import java.time.OffsetDateTime
+import java.util.UUID
 
 import akka.actor.{ActorRef, LoggingFSM, Props}
-import cromwell.webservice.metadata.IndexedJsonValue._
-import cats.instances.list._
-import cats.syntax.foldable._
+import cromwell.webservice.metadata.MetadataComponent._
 import cromwell.core.Dispatcher.ApiDispatcher
 import cromwell.core.ExecutionIndex.ExecutionIndex
 import cromwell.core.{WorkflowId, WorkflowMetadataKeys, WorkflowState}
 import cromwell.services.ServiceRegistryActor.ServiceRegistryFailure
 import cromwell.services.metadata.MetadataService._
 import cromwell.services.metadata._
-import cromwell.webservice.PerRequest.{RequestComplete, RequestCompleteWithHeaders}
-import cromwell.webservice.metadata.MetadataBuilderActor.{Idle, MetadataBuilderActorData, MetadataBuilderActorState, WaitingForMetadataService, WaitingForSubWorkflows}
-import cromwell.webservice.{APIResponse, PerRequestCreator, WorkflowJsonSupport}
+import cromwell.webservice.metadata.MetadataBuilderActor._
 import org.slf4j.LoggerFactory
-import spray.http.{StatusCodes, Uri}
-import spray.httpx.SprayJsonSupport._
 import spray.json._
 
-import scala.collection.immutable.TreeMap
-import scala.language.postfixOps
-import scala.util.{Failure, Success, Try}
-
 
 object MetadataBuilderActor {
+  sealed abstract class MetadataBuilderActorResponse
+  case class BuiltMetadataResponse(response: JsObject) extends MetadataBuilderActorResponse
+  case class FailedMetadataResponse(reason: Throwable) extends MetadataBuilderActorResponse
+
   sealed trait MetadataBuilderActorState
   case object Idle extends MetadataBuilderActorState
   case object WaitingForMetadataService extends MetadataBuilderActorState
   case object WaitingForSubWorkflows extends MetadataBuilderActorState
 
   case class MetadataBuilderActorData(
-                                      originalQuery: MetadataQuery,
-                                      originalEvents: Seq[MetadataEvent],
-                                      subWorkflowsMetadata: Map[String, JsValue],
-                                      waitFor: Int
-                                    ) {
+                                       originalQuery: MetadataQuery,
+                                       originalEvents: Seq[MetadataEvent],
+                                       subWorkflowsMetadata: Map[String, JsValue],
+                                       waitFor: Int
+                                     ) {
     def withSubWorkflow(id: String, metadata: JsValue) = {
       this.copy(subWorkflowsMetadata = subWorkflowsMetadata + ((id, metadata)))
     }
-    
+
     def isComplete = subWorkflowsMetadata.size == waitFor
   }
-  
+
   def props(serviceRegistryActor: ActorRef) = {
     Props(new MetadataBuilderActor(serviceRegistryActor)).withDispatcher(ApiDispatcher)
   }
 
   val log = LoggerFactory.getLogger("MetadataBuilder")
 
-  private val KeySeparator = MetadataKey.KeySeparator
-  private val bracketMatcher = """\[(\d*)\]""".r
-  private val startMatcher = """^([^\[]+)\[""".r
   private val AttemptKey = "attempt"
   private val ShardKey = "shardIndex"
 
-  /** Types of element supported in a dotted key notation */
-  private sealed trait KeyElement {
-    def toIndexedJson(value: TimestampedJsValue): TimestampedJsValue
-  }
-
-  private case class ListElement(name: String, indexes: List[String]) extends KeyElement {
-    def toIndexedJson(innerValue: TimestampedJsValue) = {
-      if (indexes.isEmpty) {
-        TimestampedJsObject(Map(name -> innerValue), innerValue.timestamp)
-      } else {
-       /*
-       * The last index is the one that the innerValue should have in the innerList.
-       * From there lists are fold into one another until we reach the first index.
-       * e.g l[1][2] = "a" means
-       *  "l": [
-       *         [
-       *           "a" <- will have index 2 in the inner list
-       *         ] <- inner list: will have index 1 in the outer list
-       *       ]
-       *
-       * Important note: Indexes are used for sorting purposes ONLY
-       * An index of 2 DOES NOT guarantee that a value will be at index 2 in a list
-       */
-        val list = innerValue match {
-           // Empty value in a list means empty list
-          case TimestampedEmptyJson(timestamp) => TimestampedJsList(Map.empty, timestamp)
-          case nonEmptyValue =>
-             /*
-              * This creates a (possibly nested) list, by folding over the indexes.
-              * The resulting list will be as deep as there are elements in "indexes"
-              * First we create the deepest list, that will contain innerValue (the actual value we want in the list)
-              * e.g with l[1][2] = "a". indexes will be List[1, 2]. innerValue will be "a".
-              * innerList is TimestampedJsList(Map(2 -> "a"), [timestamp of a])
-              */
-            val innerList = TimestampedJsList(TreeMap(indexes.last.toInt -> innerValue), nonEmptyValue.timestamp)
-            /* Then, stating with this innerList, we wrap around it as many lists as (indexes.length - 1) (because we used the last index for the innerValue above)
-             * Continuing with this example, result will be TimestampedJsList(Map(1 -> TimestampedJsList(Map(2 -> "a"))))
-             */
-            indexes.init.foldRight(innerList)((index, acc) => {
-              TimestampedJsList(TreeMap(index.toInt -> acc), acc.timestamp)
-            })
-        }
-
-        TimestampedJsObject(Map(name -> list), list.timestamp)
-      }
-    }
-  }
-  private case class ObjectElement(name: String) extends KeyElement {
-    def toIndexedJson(value: TimestampedJsValue) = TimestampedJsObject(Map(name -> value), value.timestamp)
-  }
-
-  private def parseKeyChunk(chunk: String): KeyElement = {
-    startMatcher.findFirstMatchIn(chunk) match {
-      case Some(listNameRegex) =>
-        val indexes = bracketMatcher.findAllMatchIn(chunk).map(_.group(1)).toList
-        ListElement(listNameRegex.group(1), indexes)
-      case _ => ObjectElement(chunk)
-    }
-  }
-
-  private def metadataValueToIndexedJson(value: Option[MetadataValue], timestamp: OffsetDateTime): TimestampedJsValue = {
-    value map { someValue =>
-      val coerced: Try[TimestampedJsPrimitive] = someValue.valueType match {
-        case MetadataInt => Try(new TimestampedJsPrimitive(JsNumber(someValue.value.toInt), timestamp))
-        case MetadataNumber => Try(new TimestampedJsPrimitive(JsNumber(someValue.value.toDouble), timestamp))
-        case MetadataBoolean => Try(new TimestampedJsPrimitive(JsBoolean(someValue.value.toBoolean), timestamp))
-        case MetadataString => Try(new TimestampedJsPrimitive(JsString(someValue.value), timestamp))
-      }
-
-      coerced match {
-        case Success(v) => v
-        case Failure(e) =>
-          log.warn(s"Failed to coerce ${someValue.value} to ${someValue.valueType}. Falling back to String.", e)
-          new TimestampedJsPrimitive(JsString(someValue.value), timestamp)
-      }
-    } getOrElse TimestampedEmptyJson(timestamp)
-  }
-
-  private def keyValueToIndexedJson(str: String, value: Option[MetadataValue], timestamp: OffsetDateTime): TimestampedJsValue = {
-    val innerValue: TimestampedJsValue = metadataValueToIndexedJson(value, timestamp)
-    str.split(KeySeparator).foldRight(innerValue)((chunk, acc) => { parseKeyChunk(chunk).toIndexedJson(acc) })
-  }
-
+  /**
+    * Metadata for a call attempt
+    */
   private case class MetadataForAttempt(attempt: Int, metadata: JsObject)
-  /** There's one TimestampedJsValue per attempt, hence the list. */
-  private case class MetadataForIndex(index: Int, metadata: List[JsObject])
-
-  implicit val dateTimeOrdering: Ordering[OffsetDateTime] = scala.Ordering.fromLessThan(_ isBefore _)
 
-  /** Sort events by timestamp, transform them into TimestampedJsValues, and merge them together. */
-  private def eventsToIndexedJson(events: Seq[MetadataEvent]): TimestampedJsValue = {
-    // The `List` has a `Foldable` instance defined in scope, and because the `List`'s elements have a `Monoid` instance
-    // defined in scope, `combineAll` can derive a sane `TimestampedJsValue` value even if the `List` of events is empty.
-    events.toList map { e => keyValueToIndexedJson(e.key.key, e.value, e.offsetDateTime) } combineAll
-  }
+  /**
+    * Metadata objects of all attempts for one shard
+    */
+  private case class MetadataForIndex(index: Int, metadata: List[JsObject])
 
-  private def eventsToAttemptMetadata(expandedValues: Map[String, JsValue])(attempt: Int, events: Seq[MetadataEvent]) = {
-    val withAttemptField = JsObject(eventsToIndexedJson(events).toJson(expandedValues).asJsObject.fields + (AttemptKey -> JsNumber(attempt)))
+  private def eventsToAttemptMetadata(subWorkflowMetadata: Map[String, JsValue])(attempt: Int, events: Seq[MetadataEvent]) = {
+    val withAttemptField = JsObject(MetadataComponent(events, subWorkflowMetadata).toJson.asJsObject.fields + (AttemptKey -> JsNumber(attempt)))
     MetadataForAttempt(attempt, withAttemptField)
   }
 
@@ -163,38 +68,62 @@ object MetadataBuilderActor {
     MetadataForIndex(index.getOrElse(-1), metadata)
   }
 
-  private def reduceWorkflowEvents(workflowEvents: Seq[MetadataEvent]): Seq[MetadataEvent] = {
-    // This handles state specially so a sensible final value is returned irrespective of the order in which raw state
-    // events were recorded in the journal.
-    val (workflowStatusEvents, workflowNonStatusEvents) = workflowEvents partition(_.key.key == WorkflowMetadataKeys.Status)
-
-    val ordering = implicitly[Ordering[WorkflowState]]
-    // This orders by value in WorkflowState CRDT resolution, not necessarily the chronologically most recent state.
-    val sortedStateEvents = workflowStatusEvents.filter(_.value.isDefined) sortWith { case (a, b) => ordering.gt(a.value.get.toWorkflowState, b.value.get.toWorkflowState) }
-    workflowNonStatusEvents ++ sortedStateEvents.headOption.toList
-  }
-
-  private def parseWorkflowEventsToTimestampedJsValue(events: Seq[MetadataEvent], includeCallsIfEmpty: Boolean, expandedValues: Map[String, JsValue]): JsObject = {
-    // Partition if sequence of events in a pair of (Workflow level events, Call level events)
+  private def buildMetadataJson(events: Seq[MetadataEvent], includeCallsIfEmpty: Boolean, expandedValues: Map[String, JsValue]): JsObject = {
+    // Partition events into workflow level and call level events
     val (workflowLevel, callLevel) = events partition { _.key.jobKey.isEmpty }
-    val foldedWorkflowValues = eventsToIndexedJson(reduceWorkflowEvents(workflowLevel)).toJson(expandedValues).asJsObject
-
+    val workflowLevelJson = MetadataComponent(workflowLevel, Map.empty).toJson.asJsObject
+
+    /*
+     * Map(
+     *    "fqn" -> Seq[Events],
+     *    "fqn2" -> Seq[Events],
+     *    ...
+     * )
+     * Note that groupBy will preserve the ordering of the events in the Seq, which means that as long as the DB sorts them by timestamp, we can always assume the last one is the newest one.
+     * This is guaranteed by the groupBy invariant and the fact that filter preservers the ordering. (See scala doc for groupBy and filter)
+     */
     val callsGroupedByFQN = callLevel groupBy { _.key.jobKey.get.callFqn }
+    /*
+     * Map(
+     *    "fqn" -> Map( //Shard index
+     *                    Option(0) -> Seq[Events],
+     *                    Option(1) -> Seq[Events]
+     *                    ...
+     *             ),
+     *    ...
+     * )
+     */
     val callsGroupedByFQNAndIndex = callsGroupedByFQN mapValues { _ groupBy { _.key.jobKey.get.index } }
+    /*
+     * Map(
+     *    "fqn" -> Map(
+     *                Option(0) -> Map( //Attempt
+     *                                    1 -> Seq[Events],
+     *                                    2 -> Seq[Events],
+     *                                ...
+     *                             ),   
+     *                ...
+     *             ),
+     *    ...
+     * )
+     */
     val callsGroupedByFQNAndIndexAndAttempt = callsGroupedByFQNAndIndex mapValues { _ mapValues { _ groupBy { _.key.jobKey.get.attempt } } }
 
-    val callsMap = callsGroupedByFQNAndIndexAndAttempt mapValues { eventsForIndex =>
-      eventsForIndex mapValues { eventsForAttempt =>
-        eventsForAttempt map Function.tupled(eventsToAttemptMetadata(expandedValues))
-      } map { Function.tupled(attemptMetadataToIndexMetadata) }
-    } mapValues { md => JsArray(md.toVector.sortBy(_.index) flatMap { _.metadata }) }
+    val eventsToAttemptFunction = Function.tupled(eventsToAttemptMetadata(expandedValues) _)
+    val attemptToIndexFunction = (attemptMetadataToIndexMetadata _).tupled
+
+    val callsMap = callsGroupedByFQNAndIndexAndAttempt mapValues { _ mapValues { _ map eventsToAttemptFunction } map attemptToIndexFunction } mapValues { md =>
+      JsArray(md.toVector.sortBy(_.index) flatMap { _.metadata })
+    }
 
     val wrappedCalls = JsObject(Map(WorkflowMetadataKeys.Calls -> JsObject(callsMap)))
     val callData = if (callsMap.isEmpty && !includeCallsIfEmpty) Nil else wrappedCalls.fields
-    JsObject(foldedWorkflowValues.fields ++ callData)
+    JsObject(workflowLevelJson.fields ++ callData)
   }
 
-  private def parseWorkflowEvents(includeCallsIfEmpty: Boolean, expandedValues: Map[String, JsValue])(events: Seq[MetadataEvent]): JsObject = parseWorkflowEventsToTimestampedJsValue(events, includeCallsIfEmpty, expandedValues)
+  private def parseWorkflowEvents(includeCallsIfEmpty: Boolean, expandedValues: Map[String, JsValue])(events: Seq[MetadataEvent]): JsObject = {
+    buildMetadataJson(events, includeCallsIfEmpty, expandedValues)
+  }
 
   /**
     * Parse a Seq of MetadataEvent into a full Json metadata response.
@@ -203,21 +132,21 @@ object MetadataBuilderActor {
     JsObject(events.groupBy(_.key.workflowId.toString) mapValues parseWorkflowEvents(includeCallsIfEmpty = true, expandedValues))
   }
 
-  implicit class EnhancedMetadataValue(val value: MetadataValue) extends AnyVal {
-    def toWorkflowState: WorkflowState = WorkflowState.fromString(value.value)
-  }
+  def uniqueActorName: String = List("MetadataBuilderActor", UUID.randomUUID()).mkString("-")
 }
 
 class MetadataBuilderActor(serviceRegistryActor: ActorRef) extends LoggingFSM[MetadataBuilderActorState, Option[MetadataBuilderActorData]]
-  with DefaultJsonProtocol with WorkflowQueryPagination {
+  with DefaultJsonProtocol {
+  import MetadataBuilderActor._
 
-  import WorkflowJsonSupport._
+  private var target: ActorRef = ActorRef.noSender
 
   startWith(Idle, None)
   val tag = self.path.name
 
   when(Idle) {
     case Event(action: MetadataServiceAction, _) =>
+      target = sender()
       serviceRegistryActor ! action
       goto(WaitingForMetadataService)
   }
@@ -228,58 +157,49 @@ class MetadataBuilderActor(serviceRegistryActor: ActorRef) extends LoggingFSM[Me
   }
 
   when(WaitingForMetadataService) {
-    case Event(MetadataLookupResponse(query, metadata), None) =>
-      processMetadataResponse(query, metadata)
     case Event(StatusLookupResponse(w, status), _) =>
-      context.parent ! RequestComplete((StatusCodes.OK, processStatusResponse(w, status)))
-      allDone
-    case Event(failure: ServiceRegistryFailure, _) =>
-      val response = APIResponse.fail(new RuntimeException("Can't find metadata service"))
-      context.parent ! RequestComplete((StatusCodes.InternalServerError, response))
-      allDone
-    case Event(WorkflowQuerySuccess(uri: Uri, response, metadata), _) =>
-      context.parent ! RequestCompleteWithHeaders(response, generateLinkHeaders(uri, metadata):_*)
-      allDone
-    case Event(failure: WorkflowQueryFailure, _) =>
-      context.parent ! RequestComplete((StatusCodes.BadRequest, APIResponse.fail(failure.reason)))
+      target ! BuiltMetadataResponse(processStatusResponse(w, status))
       allDone
     case Event(WorkflowOutputsResponse(id, events), _) =>
       // Add in an empty output event if there aren't already any output events.
       val hasOutputs = events exists { _.key.key.startsWith(WorkflowMetadataKeys.Outputs + ":") }
       val updatedEvents = if (hasOutputs) events else MetadataEvent.empty(MetadataKey(id, None, WorkflowMetadataKeys.Outputs)) +: events
-      context.parent ! RequestComplete((StatusCodes.OK, workflowMetadataResponse(id, updatedEvents, includeCallsIfEmpty = false, Map.empty)))
+      target ! BuiltMetadataResponse(workflowMetadataResponse(id, updatedEvents, includeCallsIfEmpty = false, Map.empty))
       allDone
     case Event(LogsResponse(w, l), _) =>
-      context.parent ! RequestComplete((StatusCodes.OK, workflowMetadataResponse(w, l, includeCallsIfEmpty = false, Map.empty)))
+      target ! BuiltMetadataResponse(workflowMetadataResponse(w, l, includeCallsIfEmpty = false, Map.empty))
+      allDone
+    case Event(MetadataLookupResponse(query, metadata), None) => processMetadataResponse(query, metadata)
+    case Event(_: ServiceRegistryFailure, _) =>
+      target ! FailedMetadataResponse(new RuntimeException("Can't find metadata service"))
       allDone
     case Event(failure: MetadataServiceFailure, _) =>
-      context.parent ! RequestComplete((StatusCodes.InternalServerError, APIResponse.error(failure.reason)))
+      target ! FailedMetadataResponse(failure.reason)
       allDone
     case Event(unexpectedMessage, stateData) =>
-      val response = APIResponse.fail(new RuntimeException(s"MetadataBuilderActor $tag(WaitingForMetadataService, $stateData) got an unexpected message: $unexpectedMessage"))
-      context.parent ! RequestComplete((StatusCodes.InternalServerError, response))
+      target ! FailedMetadataResponse(new RuntimeException(s"MetadataBuilderActor $tag(WaitingForMetadataService, $stateData) got an unexpected message: $unexpectedMessage"))
       context stop self
       stay()
   }
-  
+
   when(WaitingForSubWorkflows) {
-    case Event(RequestComplete(metadata), Some(data)) =>
-      processSubWorkflowMetadata(metadata, data)
+    case Event(mbr: MetadataBuilderActorResponse, Some(data)) =>
+      processSubWorkflowMetadata(mbr, data)
   }
-  
+
   whenUnhandled {
     case Event(message, data) =>
       log.error(s"Received unexpected message $message in state $stateName with data $data")
       stay()
   }
-  
-  def processSubWorkflowMetadata(metadataResponse: Any, data: MetadataBuilderActorData) = {
+
+  def processSubWorkflowMetadata(metadataResponse: MetadataBuilderActorResponse, data: MetadataBuilderActorData) = {
     metadataResponse match {
-      case (StatusCodes.OK, js: JsObject) =>
+      case BuiltMetadataResponse(js) =>
         js.fields.get(WorkflowMetadataKeys.Id) match {
           case Some(subId: JsString) =>
             val newData = data.withSubWorkflow(subId.value, js)
-            
+
             if (newData.isComplete) {
               buildAndStop(data.originalQuery, data.originalEvents, newData.subWorkflowsMetadata)
             } else {
@@ -287,21 +207,21 @@ class MetadataBuilderActor(serviceRegistryActor: ActorRef) extends LoggingFSM[Me
             }
           case _ => failAndDie(new RuntimeException("Received unexpected response while waiting for sub workflow metadata."))
         }
-      case _ => failAndDie(new RuntimeException("Failed to retrieve metadata for a sub workflow."))
+      case FailedMetadataResponse(e) => failAndDie(new RuntimeException("Failed to retrieve metadata for a sub workflow.", e))
     }
   }
-  
+
   def failAndDie(reason: Throwable) = {
-    context.parent ! RequestComplete((StatusCodes.InternalServerError, APIResponse.error(reason)))
+    target ! FailedMetadataResponse(reason)
     context stop self
     stay()
   }
-  
+
   def buildAndStop(query: MetadataQuery, eventsList: Seq[MetadataEvent], expandedValues: Map[String, JsValue]) = {
-    context.parent ! RequestComplete((StatusCodes.OK, processMetadataEvents(query, eventsList, expandedValues)))
+    target ! BuiltMetadataResponse(processMetadataEvents(query, eventsList, expandedValues))
     allDone
   }
-  
+
   def processMetadataResponse(query: MetadataQuery, eventsList: Seq[MetadataEvent]) = {
     if (query.expandSubWorkflows) {
       // Scan events for sub workflow ids
@@ -314,7 +234,7 @@ class MetadataBuilderActor(serviceRegistryActor: ActorRef) extends LoggingFSM[Me
       else {
         // Otherwise spin up a metadata builder actor for each sub workflow
         subWorkflowIds foreach { subId =>
-          val subMetadataBuilder = context.actorOf(MetadataBuilderActor.props(serviceRegistryActor), PerRequestCreator.endpointActorName)
+          val subMetadataBuilder = context.actorOf(MetadataBuilderActor.props(serviceRegistryActor), uniqueActorName)
           subMetadataBuilder ! GetMetadataQueryAction(query.copy(workflowId = WorkflowId.fromString(subId)))
         }
         goto(WaitingForSubWorkflows) using Option(MetadataBuilderActorData(query, eventsList, Map.empty, subWorkflowIds.size))
@@ -342,7 +262,10 @@ class MetadataBuilderActor(serviceRegistryActor: ActorRef) extends LoggingFSM[Me
     ))
   }
 
-  private def workflowMetadataResponse(workflowId: WorkflowId, eventsList: Seq[MetadataEvent], includeCallsIfEmpty: Boolean, expandedValues: Map[String, JsValue]) = {
+  private def workflowMetadataResponse(workflowId: WorkflowId,
+                                       eventsList: Seq[MetadataEvent],
+                                       includeCallsIfEmpty: Boolean,
+                                       expandedValues: Map[String, JsValue]): JsObject = {
     JsObject(MetadataBuilderActor.parseWorkflowEvents(includeCallsIfEmpty, expandedValues)(eventsList).fields + ("id" -> JsString(workflowId.toString)))
   }
 }
diff --git a/engine/src/main/scala/cromwell/webservice/metadata/MetadataComponent.scala b/engine/src/main/scala/cromwell/webservice/metadata/MetadataComponent.scala
new file mode 100644
index 000000000..ee70d412a
--- /dev/null
+++ b/engine/src/main/scala/cromwell/webservice/metadata/MetadataComponent.scala
@@ -0,0 +1,156 @@
+package cromwell.webservice.metadata
+
+import cats.{Monoid, Semigroup}
+import cats.instances.map._
+import cats.instances.list._
+import cats.syntax.foldable._
+import cromwell.core.{ExecutionStatus, WorkflowMetadataKeys, WorkflowState}
+import cromwell.services.metadata._
+import spray.json.{JsArray, _}
+
+import scala.collection.immutable.TreeMap
+import scala.language.postfixOps
+import scala.util.{Random, Try}
+
+object MetadataComponent {
+  implicit val MetadataComponentMonoid: Monoid[MetadataComponent] = new Monoid[MetadataComponent] {
+    private lazy val stringKeyMapSg = implicitly[Semigroup[Map[String, MetadataComponent]]]
+    private lazy val intKeyMapSg = implicitly[Semigroup[Map[Int, MetadataComponent]]]
+
+    def combine(f1: MetadataComponent, f2: MetadataComponent): MetadataComponent = {
+      (f1, f2) match {
+        case (MetadataObject(v1), MetadataObject(v2)) => MetadataObject(stringKeyMapSg.combine(v1, v2))
+        case (MetadataList(v1), MetadataList(v2)) => MetadataList(intKeyMapSg.combine(v1, v2))
+        // If there's a custom ordering, use it
+        case (v1 @ MetadataPrimitive(_, Some(o1)), v2 @ MetadataPrimitive(_, Some(o2))) if o1 == o2 => o1.max(v1, v2)
+        // Otherwise assume it's ordered by default and take the new one
+        case (_, o2) => o2
+      }
+    }
+
+    override def empty: MetadataComponent = MetadataObject.empty
+  }
+
+  val metadataPrimitiveJsonWriter: JsonWriter[MetadataPrimitive] = JsonWriter.func2Writer[MetadataPrimitive] {
+    case MetadataPrimitive(MetadataValue(value, MetadataInt), _) => Try(value.toInt) map JsNumber.apply getOrElse JsString(value)
+    case MetadataPrimitive(MetadataValue(value, MetadataNumber), _) => Try(value.toDouble) map JsNumber.apply getOrElse JsString(value)
+    case MetadataPrimitive(MetadataValue(value, MetadataBoolean), _) => Try(value.toBoolean) map JsBoolean.apply getOrElse JsString(value)
+    case MetadataPrimitive(MetadataValue(value, MetadataString), _) => JsString(value)
+  }
+
+  implicit val metadataComponentJsonWriter: JsonWriter[MetadataComponent] = JsonWriter.func2Writer[MetadataComponent] {
+    case MetadataList(values) => JsArray(values.values.toVector map { _.toJson(this.metadataComponentJsonWriter) })
+    case MetadataObject(values) => JsObject(values.mapValues(_.toJson(this.metadataComponentJsonWriter)))
+    case primitive: MetadataPrimitive => metadataPrimitiveJsonWriter.write(primitive)
+    case MetadataEmpty => JsObject.empty
+    case MetadataNull => JsNull
+    case MetadataJsonComponent(jsValue) => jsValue
+  }
+
+  /* ******************************* */
+  /* *** Metadata Events Parsing *** */
+  /* ******************************* */
+  
+  private val KeySeparator = MetadataKey.KeySeparator
+  // Split on every unescaped KeySeparator
+  val KeySplitter = s"(?<!\\\\)$KeySeparator"
+  private val bracketMatcher = """\[(\d*)\]""".r
+  
+  private def parseKeyChunk(chunk: String, innerValue: MetadataComponent): MetadataComponent = {
+    chunk.indexOf('[') match {
+      // If there's no bracket, it's an object. e.g.: "calls"
+      case -1 => MetadataObject(Map(chunk -> innerValue))
+      // If there's a bracket it's a named list. e.g.: "executionEvents[0][1]"
+      case bracketIndex =>
+        // Name: "executionEvents"
+        val objectName = chunk.substring(0, bracketIndex)
+
+        // Empty value means empty list
+        if (innerValue == MetadataEmpty) MetadataObject(Map(objectName -> MetadataList.empty))
+        else {
+          // Brackets: "[0][1]"
+          val brackets = chunk.substring(bracketIndex)
+          // Indices as a list: List(0, 1)
+          val listIndices = for {
+            m <- bracketMatcher.findAllMatchIn(brackets)
+            // It's possible for a bracket pair to be empty, in which case we just give it a random number
+            asInt = if (m.group(1).isEmpty) Random.nextInt() else m.group(1).toInt
+          } yield asInt
+          // Fold into a MetadataList: MetadataList(0 -> MetadataList(1 -> innerValue))
+          val metadataList = listIndices.toList.foldRight(innerValue)((index, acc) => MetadataList(TreeMap(index -> acc)))
+
+          MetadataObject(Map(objectName -> metadataList))
+        }
+    }
+  }
+
+  private def customOrdering(event: MetadataEvent): Option[Ordering[MetadataPrimitive]] = event match {
+    case MetadataEvent(MetadataKey(_, Some(_), key), _, _) if key == CallMetadataKeys.ExecutionStatus => Option(MetadataPrimitive.ExecutionStatusOrdering)
+    case MetadataEvent(MetadataKey(_, None, key), _, _) if key == WorkflowMetadataKeys.Status => Option(MetadataPrimitive.WorkflowStateOrdering)
+    case _ => None
+  }
+
+  private def toMetadataComponent(subWorkflowMetadata: Map[String, JsValue])(event: MetadataEvent) = {
+    lazy val primitive = event.value map { MetadataPrimitive(_, customOrdering(event)) } getOrElse MetadataEmpty
+    lazy val originalKeyAndPrimitive = (event.key.key, primitive)
+
+    val keyAndPrimitive: (String, MetadataComponent) = if (event.key.key.endsWith(CallMetadataKeys.SubWorkflowId)) {
+      (for {
+        metadataValue <- event.value
+        subWorkflowMetadata <- subWorkflowMetadata.get(metadataValue.value)
+        keyWithSubWorkflowMetadata = event.key.key.replace(CallMetadataKeys.SubWorkflowId, CallMetadataKeys.SubWorkflowMetadata)
+        subWorkflowComponent = MetadataJsonComponent(subWorkflowMetadata)
+      } yield (keyWithSubWorkflowMetadata, subWorkflowComponent)) getOrElse originalKeyAndPrimitive
+    } else originalKeyAndPrimitive
+
+    fromMetadataKeyAndPrimitive(keyAndPrimitive._1, keyAndPrimitive._2)
+  }
+  
+  /** Sort events by timestamp, transform them into MetadataComponent, and merge them together. */
+  def apply(events: Seq[MetadataEvent], subWorkflowMetadata: Map[String, JsValue] = Map.empty): MetadataComponent = {
+    // The `List` has a `Foldable` instance defined in scope, and because the `List`'s elements have a `Monoid` instance
+    // defined in scope, `combineAll` can derive a sane `TimestampedJsValue` value even if the `List` of events is empty.
+    events.toList map toMetadataComponent(subWorkflowMetadata) combineAll
+  }
+  
+  def fromMetadataKeyAndPrimitive(metadataKey: String, innerComponent: MetadataComponent) = {
+    import MetadataKey._
+    metadataKey.split(KeySplitter).map(_.unescapeMeta).foldRight(innerComponent)(parseKeyChunk)
+  }
+}
+
+sealed trait MetadataComponent
+case object MetadataEmpty extends MetadataComponent
+case object MetadataNull extends MetadataComponent
+
+// Metadata Object  
+object MetadataObject {
+  def empty = new MetadataObject(Map.empty)
+  def apply(kvPair: (String, MetadataComponent)*) = {
+    new MetadataObject(kvPair.toMap)
+  }
+}
+
+case class MetadataObject(v: Map[String, MetadataComponent]) extends MetadataComponent
+
+// Metadata List
+object MetadataList {
+  def empty = new MetadataList(Map.empty)
+  def apply(components: List[MetadataComponent]) = new MetadataList(components.zipWithIndex.map({case (c, i) => i -> c}).toMap)
+}
+case class MetadataList(v: Map[Int, MetadataComponent]) extends MetadataComponent
+
+// Metadata Primitive
+object MetadataPrimitive {
+  val ExecutionStatusOrdering: Ordering[MetadataPrimitive] = Ordering.by { primitive: MetadataPrimitive =>
+    ExecutionStatus.withName(primitive.v.value)
+  }
+
+  val WorkflowStateOrdering: Ordering[MetadataPrimitive] = Ordering.by { primitive: MetadataPrimitive =>
+    WorkflowState.withName(primitive.v.value)
+  }
+}
+case class MetadataPrimitive(v: MetadataValue, customOrdering: Option[Ordering[MetadataPrimitive]] = None) extends MetadataComponent
+
+// Metadata Component that owns an already computed JsValue
+case class MetadataJsonComponent(jsValue: JsValue) extends MetadataComponent
diff --git a/engine/src/main/scala/cromwell/webservice/metadata/WorkflowQueryPagination.scala b/engine/src/main/scala/cromwell/webservice/metadata/WorkflowQueryPagination.scala
index 8f6650e88..9ae121067 100644
--- a/engine/src/main/scala/cromwell/webservice/metadata/WorkflowQueryPagination.scala
+++ b/engine/src/main/scala/cromwell/webservice/metadata/WorkflowQueryPagination.scala
@@ -1,33 +1,36 @@
 package cromwell.webservice.metadata
 
+import akka.http.scaladsl.model.Uri.Query
+import akka.http.scaladsl.model.headers.{Link, LinkParams}
+import akka.http.scaladsl.model.{HttpHeader, Uri}
 import cromwell.services.metadata.MetadataService.QueryMetadata
-import spray.http.HttpHeaders.Link
-import spray.http.{HttpHeader, Uri}
+
 
 /**
   * Attempts to add query parameters for pagination.
   *
-  * NOTE: This trait is effectively broken, as the returned links are not suitable for use by cromwell clients.
+  * NOTE: This is effectively broken, as the returned links are not suitable for use by cromwell clients.
   *
-  * The trait discards the search parameters for GETs, for example it drops parameters such as "start" and "end". Also
+  * This discards the search parameters for GETs, for example it drops parameters such as "start" and "end". Also
   * generates links incompatible with POSTs, as the endpoints read parameters from the HTTP body during POST, __not__
   * from the URI.
   *
-  * This trait may need to receive an entire `spray.http.HttpRequest` and not just the `spray.http.Uri` to ensure that
-  * it doesn't generate links for POST.
+  * This may need to receive an entire `HttpRequest` and not just the `Uri` to ensure that it doesn't generate links for POST.
   *
   * The existing `CromwellApiServiceSpec` should be updated to verify the expected behavior for both GET and POST.
   *
   * Left behind for legacy reasons, but don't believe anyone has ever used these non-functional links.
+  *
+  * Note: As of 6/7/17 the above is confirmed by JG, but leaving it mostly as-is for now
   */
-trait WorkflowQueryPagination {
+object WorkflowQueryPagination {
 
-  protected def generatePaginationParams(page: Int, pageSize: Int): String = {
-    s"page=$page&pagesize=$pageSize"
+  private def generatePaginationParams(page: Int, pageSize: Int): Query = {
+    Query(s"page=$page&pagesize=$pageSize")
   }
 
   //Generates link headers for pagination navigation https://tools.ietf.org/html/rfc5988#page-6
-  protected def generateLinkHeaders(uri: Uri, metadata: Option[QueryMetadata]): Seq[HttpHeader] = {
+  def generateLinkHeaders(uri: Uri, metadata: Option[QueryMetadata]): List[HttpHeader] = {
     //strip off the query params
     val baseUrl = uri.scheme + ":" + uri.authority + uri.path
     metadata match {
@@ -35,22 +38,21 @@ trait WorkflowQueryPagination {
         (meta.page, meta.pageSize) match {
           case (Some(p), Some(ps)) =>
 
-            val firstLink = Link(Uri(baseUrl).withQuery(generatePaginationParams(1, ps)), Link.first)
+            val firstLink = Link(Uri(baseUrl).withQuery(generatePaginationParams(1, ps)), LinkParams.first)
 
             val prevPage = math.max(p - 1, 1)
-            val prevLink = Link(Uri(baseUrl).withQuery(generatePaginationParams(prevPage, ps)), Link.prev)
+            val prevLink = Link(Uri(baseUrl).withQuery(generatePaginationParams(prevPage, ps)), LinkParams.prev)
 
             val lastPage = math.ceil(meta.totalRecords.getOrElse(1).toDouble / ps.toDouble).toInt
-            val lastLink = Link(Uri(baseUrl).withQuery(generatePaginationParams(lastPage, ps)), Link.last)
+            val lastLink = Link(Uri(baseUrl).withQuery(generatePaginationParams(lastPage, ps)), LinkParams.last)
 
             val nextPage = math.min(p + 1, lastPage)
-            val nextLink = Link(Uri(baseUrl).withQuery(generatePaginationParams(nextPage, ps)), Link.next)
-
-            Seq(firstLink, prevLink, nextLink, lastLink)
+            val nextLink = Link(Uri(baseUrl).withQuery(generatePaginationParams(nextPage, ps)), LinkParams.next)
 
-          case _ => Seq()
+            List(firstLink, prevLink, nextLink, lastLink)
+          case _ => List.empty
         }
-      case None => Seq()
+      case None => List.empty
     }
   }
 }
diff --git a/engine/src/test/resources/swagger/testservice.json b/engine/src/test/resources/swagger/testservice.json
new file mode 100644
index 000000000..3ef100ba2
--- /dev/null
+++ b/engine/src/test/resources/swagger/testservice.json
@@ -0,0 +1,29 @@
+{
+  "swagger": "2.0",
+  "info": {
+    "title": "Test Service API",
+    "description": "Test Service API",
+    "version": "1.2.3"
+  },
+  "produces": [
+    "application/json"
+  ],
+  "paths": {
+    "/hello": {
+      "get": {
+        "responses": {
+          "200": {
+            "description": "Says hello via get"
+          }
+        }
+      },
+      "post": {
+        "responses": {
+          "200": {
+            "description": "Says hello via post"
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/engine/src/test/resources/swagger/testservice.yaml b/engine/src/test/resources/swagger/testservice.yaml
new file mode 100644
index 000000000..186a0b913
--- /dev/null
+++ b/engine/src/test/resources/swagger/testservice.yaml
@@ -0,0 +1,17 @@
+swagger: '2.0'
+info:
+  title: Test Service API
+  description: Test Service API
+  version: 1.2.3
+produces:
+  - application/json
+paths:
+  /hello:
+    get:
+      responses:
+        '200':
+          description: Says hello via get
+    post:
+      responses:
+        '200':
+          description: Says hello via post
diff --git a/engine/src/test/scala/cromwell/ArrayOfArrayCoercionSpec.scala b/engine/src/test/scala/cromwell/ArrayOfArrayCoercionSpec.scala
index e0315d92f..e02b87c4c 100644
--- a/engine/src/test/scala/cromwell/ArrayOfArrayCoercionSpec.scala
+++ b/engine/src/test/scala/cromwell/ArrayOfArrayCoercionSpec.scala
@@ -1,12 +1,12 @@
 package cromwell
 
 import akka.testkit._
-import wdl4s.types.{WdlArrayType, WdlStringType}
-import wdl4s.values.{WdlArray, WdlString}
+import wdl4s.wdl.types.{WdlArrayType, WdlStringType}
+import wdl4s.wdl.values.{WdlArray, WdlString}
 import cromwell.util.SampleWdl
 
 
-class ArrayOfArrayCoercionSpec extends CromwellTestKitSpec {
+class ArrayOfArrayCoercionSpec extends CromwellTestKitWordSpec {
   "A workflow that has an Array[Array[File]] input " should {
     "accept an Array[Array[String]] as the value for the input" in {
       runWdlAndAssertOutputs(
diff --git a/engine/src/test/scala/cromwell/ArrayWorkflowSpec.scala b/engine/src/test/scala/cromwell/ArrayWorkflowSpec.scala
index 0dcf76cee..51f998e72 100644
--- a/engine/src/test/scala/cromwell/ArrayWorkflowSpec.scala
+++ b/engine/src/test/scala/cromwell/ArrayWorkflowSpec.scala
@@ -3,14 +3,14 @@ package cromwell
 import akka.testkit._
 import cromwell.core.path.DefaultPathBuilder
 import cromwell.util.SampleWdl
-import wdl4s.expression.NoFunctions
-import wdl4s.types.{WdlArrayType, WdlFileType, WdlStringType}
-import wdl4s.values.{WdlArray, WdlFile, WdlString}
-import wdl4s.{ImportResolver, WdlNamespaceWithWorkflow}
+import wdl4s.wdl.expression.NoFunctions
+import wdl4s.wdl.types.{WdlArrayType, WdlFileType, WdlStringType}
+import wdl4s.wdl.values.{WdlArray, WdlFile, WdlString}
+import wdl4s.wdl.{ImportResolver, WdlNamespaceWithWorkflow}
 
-class ArrayWorkflowSpec extends CromwellTestKitSpec {
+class ArrayWorkflowSpec extends CromwellTestKitWordSpec {
   val tmpDir = DefaultPathBuilder.createTempDirectory("ArrayWorkflowSpec")
-  val ns = WdlNamespaceWithWorkflow.load(SampleWdl.ArrayLiteral(tmpDir).wdlSource(), Seq.empty[ImportResolver]).get
+  val ns = WdlNamespaceWithWorkflow.load(SampleWdl.ArrayLiteral(tmpDir).workflowSource(), Seq.empty[ImportResolver]).get
   val expectedArray = WdlArray(WdlArrayType(WdlFileType), Seq(WdlFile("f1"), WdlFile("f2"), WdlFile("f3")))
 
   "A static Array[File] declaration" should {
diff --git a/engine/src/test/scala/cromwell/CallCachingWorkflowSpec.scala b/engine/src/test/scala/cromwell/CallCachingWorkflowSpec.scala
index 202dcecf1..24fa0ec22 100644
--- a/engine/src/test/scala/cromwell/CallCachingWorkflowSpec.scala
+++ b/engine/src/test/scala/cromwell/CallCachingWorkflowSpec.scala
@@ -7,11 +7,11 @@ import com.typesafe.config.ConfigFactory
 import cromwell.CallCachingWorkflowSpec._
 import cromwell.core.Tags.{DockerTest, _}
 import cromwell.util.SampleWdl
-import wdl4s.types.{WdlArrayType, WdlIntegerType, WdlStringType}
-import wdl4s.values.{WdlArray, WdlFile, WdlInteger, WdlString}
+import wdl4s.wdl.types.{WdlArrayType, WdlIntegerType, WdlStringType}
+import wdl4s.wdl.values.{WdlArray, WdlFile, WdlInteger, WdlString}
 
 
-class CallCachingWorkflowSpec extends CromwellTestKitSpec {
+class CallCachingWorkflowSpec extends CromwellTestKitWordSpec {
   def cacheHitMessageForCall(name: String) = s"Call Caching: Cache hit. Using UUID\\(.{8}\\):$name\\.*"
 
   val expectedOutputs = Map(
diff --git a/engine/src/test/scala/cromwell/CopyWorkflowOutputsSpec.scala b/engine/src/test/scala/cromwell/CopyWorkflowOutputsSpec.scala
index 690e618b1..643a9c725 100644
--- a/engine/src/test/scala/cromwell/CopyWorkflowOutputsSpec.scala
+++ b/engine/src/test/scala/cromwell/CopyWorkflowOutputsSpec.scala
@@ -8,7 +8,7 @@ import org.scalatest.prop.Tables.Table
 
 import scala.language.postfixOps
 
-class CopyWorkflowOutputsSpec extends CromwellTestKitSpec {
+class CopyWorkflowOutputsSpec extends CromwellTestKitWordSpec {
 
   "CopyWorkflowOutputsCall" should {
     "copy workflow outputs" in {
diff --git a/engine/src/test/scala/cromwell/CromwellTestKitSpec.scala b/engine/src/test/scala/cromwell/CromwellTestKitSpec.scala
index b28a61a3f..318feeaa6 100644
--- a/engine/src/test/scala/cromwell/CromwellTestKitSpec.scala
+++ b/engine/src/test/scala/cromwell/CromwellTestKitSpec.scala
@@ -5,40 +5,40 @@ import java.util.concurrent.atomic.AtomicInteger
 
 import akka.actor.{Actor, ActorRef, ActorSystem, Props, Terminated}
 import akka.pattern.ask
+import akka.stream.ActorMaterializer
 import akka.testkit._
 import com.typesafe.config.{Config, ConfigFactory}
 import cromwell.CromwellTestKitSpec._
 import cromwell.backend._
 import cromwell.core._
-import cromwell.core.callcaching.docker.DockerHashActor.DockerHashResponseSuccess
-import cromwell.core.callcaching.docker.{DockerHashRequest, DockerHashResult}
 import cromwell.core.path.BetterFileMethods.Cmds
 import cromwell.core.path.DefaultPathBuilder
+import cromwell.docker.DockerHashActor.DockerHashSuccessResponse
+import cromwell.docker.{DockerHashRequest, DockerHashResult}
 import cromwell.engine.backend.BackendConfigurationEntry
 import cromwell.engine.workflow.WorkflowManagerActor.RetrieveNewWorkflows
-import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheReadActor.{CacheLookupRequest, CacheResultMatchesForHashes}
-import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.CallCacheHashes
+import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheReadActor.{CacheLookupNoHit, CacheLookupRequest}
+import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheWriteActor.SaveCallCacheHashes
+import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheWriteSuccess
 import cromwell.engine.workflow.workflowstore.WorkflowStoreSubmitActor.WorkflowSubmittedToStore
 import cromwell.engine.workflow.workflowstore.{InMemoryWorkflowStore, WorkflowStoreActor}
 import cromwell.jobstore.JobStoreActor.{JobStoreWriteSuccess, JobStoreWriterCommand}
 import cromwell.server.{CromwellRootActor, CromwellSystem}
 import cromwell.services.ServiceRegistryActor
-import cromwell.services.metadata.MetadataQuery
 import cromwell.services.metadata.MetadataService._
 import cromwell.subworkflowstore.EmptySubWorkflowStoreActor
 import cromwell.util.SampleWdl
-import cromwell.webservice.PerRequest.RequestComplete
 import cromwell.webservice.metadata.MetadataBuilderActor
+import cromwell.webservice.metadata.MetadataBuilderActor.{BuiltMetadataResponse, FailedMetadataResponse, MetadataBuilderActorResponse}
 import org.scalactic.Equality
+import org.scalatest._
 import org.scalatest.concurrent.{Eventually, ScalaFutures}
 import org.scalatest.time.{Millis, Seconds, Span}
-import org.scalatest.{BeforeAndAfterAll, Matchers, OneInstancePerTest, WordSpecLike}
-import spray.http.StatusCode
 import spray.json._
-import wdl4s.TaskCall
-import wdl4s.expression.{NoFunctions, WdlStandardLibraryFunctions}
-import wdl4s.types._
-import wdl4s.values._
+import wdl4s.wdl.WdlTaskCall
+import wdl4s.wdl.expression.{NoFunctions, WdlStandardLibraryFunctions}
+import wdl4s.wdl.types._
+import wdl4s.wdl.values._
 
 import scala.concurrent.duration._
 import scala.concurrent.{Await, ExecutionContext, Future}
@@ -48,12 +48,15 @@ import scala.util.matching.Regex
 case class TestBackendLifecycleActorFactory(configurationDescriptor: BackendConfigurationDescriptor)
   extends BackendLifecycleActorFactory {
   override def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor,
-                                                calls: Set[TaskCall],
-                                                serviceRegistryActor: ActorRef): Option[Props] = None
+                                                ioActor: ActorRef,
+                                                calls: Set[WdlTaskCall],
+                                                serviceRegistryActor: ActorRef,
+                                                restarting: Boolean): Option[Props] = None
 
   override def jobExecutionActorProps(jobDescriptor: BackendJobDescriptor,
                                       initializationData: Option[BackendInitializationData],
                                       serviceRegistryActor: ActorRef,
+                                      ioActor: ActorRef,
                                       backendSingletonActor: Option[ActorRef]): Props = {
     throw new NotImplementedError("this is not implemented")
   }
@@ -96,7 +99,7 @@ object CromwellTestKitSpec {
       |    }
       |
       |    # A dispatcher for engine actors
-      |    # Because backends behaviour is unpredictable (potentially blocking, slow) the engine runs
+      |    # Because backends behavior is unpredictable (potentially blocking, slow) the engine runs
       |    # on its own dispatcher to prevent backends from affecting its performance.
       |    engine-dispatcher {
       |      type = Dispatcher
@@ -115,7 +118,7 @@ object CromwellTestKitSpec {
       |    # Some of our tests fire off a message, then expect a particular event message within 3s (the default).
       |    # Especially on CI, the metadata test does not seem to be returning in time. So, overriding the timeouts
       |    # with slightly higher values. Alternatively, could also adjust the akka.test.timefactor only in CI.
-      |    filter-leeway = 5s
+      |    filter-leeway = 10s
       |    single-expect-default = 5s
       |    default-timeout = 10s
       |  }
@@ -273,11 +276,10 @@ object CromwellTestKitSpec {
     ServiceRegistryActorSystem.actorOf(ServiceRegistryActor.props(ConfigFactory.load()), "ServiceRegistryActor")
   }
 
-  class TestCromwellRootActor(config: Config) extends CromwellRootActor {
+  class TestCromwellRootActor(config: Config)(implicit materializer: ActorMaterializer) extends CromwellRootActor(false, false) {
     override val serverMode = true
     override lazy val serviceRegistryActor = ServiceRegistryActorInstance
     override lazy val workflowStore = new InMemoryWorkflowStore
-    override val abortJobsOnTerminate = false
     def submitWorkflow(sources: WorkflowSourceFilesWithoutImports): WorkflowId = {
       val submitMessage = WorkflowStoreActor.SubmitWorkflow(sources)
       val result = Await.result(workflowStoreActor.ask(submitMessage)(TimeoutDuration), Duration.Inf).asInstanceOf[WorkflowSubmittedToStore].workflowId
@@ -285,15 +287,19 @@ object CromwellTestKitSpec {
       result
     }
   }
+
+  def defaultTwms = new CromwellTestKitSpec.TestWorkflowManagerSystem()
 }
 
-abstract class CromwellTestKitSpec(val twms: TestWorkflowManagerSystem = new CromwellTestKitSpec.TestWorkflowManagerSystem()) extends TestKit(twms.actorSystem)
-  with DefaultTimeout with ImplicitSender with WordSpecLike with Matchers with BeforeAndAfterAll with ScalaFutures with OneInstancePerTest with Eventually {
+abstract class CromwellTestKitWordSpec extends CromwellTestKitSpec with WordSpecLike
+abstract class CromwellTestKitSpec(val twms: TestWorkflowManagerSystem = defaultTwms) extends TestKit(twms.actorSystem)
+  with DefaultTimeout with ImplicitSender with Matchers with ScalaFutures with Eventually with Suite with OneInstancePerTest with BeforeAndAfterAll {
 
   override protected def afterAll() = { twms.shutdownTestActorSystem(); () }
 
   implicit val defaultPatience = PatienceConfig(timeout = Span(200, Seconds), interval = Span(1000, Millis))
   implicit val ec = system.dispatcher
+  implicit val materializer = twms.materializer
 
   val dummyServiceRegistryActor = system.actorOf(Props.empty)
   val dummyLogCopyRouter = system.actorOf(Props.empty)
@@ -308,7 +314,7 @@ abstract class CromwellTestKitSpec(val twms: TestWorkflowManagerSystem = new Cro
     }
   }
 
-  // Allow to use shouldEqual between 2 WdlValues while acknowledging for edge cases and checking for WdlType compatibilty
+  // Allow to use shouldEqual between 2 WdlValues while acknowledging for edge cases and checking for WdlType compatibility
   implicit val wdlEquality = new Equality[WdlValue] {
     def fileEquality(f1: String, f2: String) =
       DefaultPathBuilder.get(f1).getFileName == DefaultPathBuilder.get(f2).getFileName
@@ -339,7 +345,7 @@ abstract class CromwellTestKitSpec(val twms: TestWorkflowManagerSystem = new Cro
   }
 
   private def buildCromwellRootActor(config: Config) = {
-    TestActorRef(new TestCromwellRootActor(config), name = "TestCromwellRootActor")
+    TestActorRef(new TestCromwellRootActor(config), name = "TestCromwellRootActor" + UUID.randomUUID().toString)
   }
 
   def runWdl(sampleWdl: SampleWdl,
@@ -350,7 +356,13 @@ abstract class CromwellTestKitSpec(val twms: TestWorkflowManagerSystem = new Cro
              config: Config = DefaultConfig,
              patienceConfig: PatienceConfig = defaultPatience)(implicit ec: ExecutionContext): Map[FullyQualifiedName, WdlValue] = {
     val rootActor = buildCromwellRootActor(config)
-    val sources = WorkflowSourceFilesWithoutImports(sampleWdl.wdlSource(runtime), sampleWdl.wdlJson, workflowOptions, customLabels)
+    val sources = WorkflowSourceFilesWithoutImports(
+      workflowSource = sampleWdl.workflowSource(runtime),
+      workflowType = Option("WDL"),
+      workflowTypeVersion = None,
+      inputsJson = sampleWdl.workflowJson,
+      workflowOptionsJson = workflowOptions,
+      labelsJson = customLabels)
     val workflowId = rootActor.underlyingActor.submitWorkflow(sources)
     eventually { verifyWorkflowState(rootActor.underlyingActor.serviceRegistryActor, workflowId, terminalState) } (config = patienceConfig, pos = implicitly[org.scalactic.source.Position])
     val outcome = getWorkflowOutputsFromMetadata(workflowId, rootActor.underlyingActor.serviceRegistryActor)
@@ -394,22 +406,6 @@ abstract class CromwellTestKitSpec(val twms: TestWorkflowManagerSystem = new Cro
     workflowId
   }
 
-  def getWorkflowMetadata(workflowId: WorkflowId, serviceRegistryActor: ActorRef, key: Option[String] = None)(implicit ec: ExecutionContext): JsObject = {
-    // MetadataBuilderActor sends its response to context.parent, so we can't just use an ask to talk to it here
-    val message = GetMetadataQueryAction(MetadataQuery(workflowId, None, key, None, None, expandSubWorkflows = false))
-    val parentProbe = TestProbe()
-
-    TestActorRef(MetadataBuilderActor.props(serviceRegistryActor), parentProbe.ref, s"MetadataActor-${UUID.randomUUID()}") ! message
-    val metadata = parentProbe.expectMsgPF(TimeoutDuration) {
-      // Because of type erasure the scala compiler can't check that the RequestComplete generic type will be (StatusCode, JsObject), which would generate a warning
-      // As long as Metadata sends back a JsObject this is safe
-      case response: RequestComplete[(StatusCode, JsObject)] @unchecked => response.response._2
-    }
-
-    system.stop(parentProbe.ref)
-    metadata
-  }
-
   /**
     * Verifies that a state is correct. // TODO: There must be a better way...?
     */
@@ -427,7 +423,16 @@ abstract class CromwellTestKitSpec(val twms: TestWorkflowManagerSystem = new Cro
   }
 
   private def getWorkflowOutputsFromMetadata(id: WorkflowId, serviceRegistryActor: ActorRef): Map[FullyQualifiedName, WdlValue] = {
-    getWorkflowMetadata(id, serviceRegistryActor, None).getFields(WorkflowMetadataKeys.Outputs).toList match {
+    val mba = system.actorOf(MetadataBuilderActor.props(serviceRegistryActor))
+    val response = mba.ask(WorkflowOutputs(id)).mapTo[MetadataBuilderActorResponse] collect {
+      case BuiltMetadataResponse(r) => r
+      case FailedMetadataResponse(e) => throw e
+    }
+    val jsObject = Await.result(response, TimeoutDuration)
+
+    system.stop(mba)
+
+    jsObject.getFields(WorkflowMetadataKeys.Outputs).toList match {
       case head::_ => head.asInstanceOf[JsObject].fields.map( x => (x._1, jsValueToWdlValue(x._2)))
       case _ => Map.empty
     }
@@ -468,7 +473,13 @@ object AlwaysHappyJobStoreActor {
 
 class EmptyCallCacheReadActor extends Actor {
   override def receive: Receive = {
-    case CacheLookupRequest(CallCacheHashes(hashes)) => sender ! CacheResultMatchesForHashes(hashes, Set.empty)
+    case _: CacheLookupRequest => sender ! CacheLookupNoHit
+  }
+}
+
+class EmptyCallCacheWriteActor extends Actor {
+  override def receive: Receive = {
+    case SaveCallCacheHashes => sender ! CallCacheWriteSuccess
   }
 }
 
@@ -476,9 +487,13 @@ object EmptyCallCacheReadActor {
   def props: Props = Props(new EmptyCallCacheReadActor)
 }
 
+object EmptyCallCacheWriteActor {
+  def props: Props = Props(new EmptyCallCacheWriteActor)
+}
+
 class EmptyDockerHashActor extends Actor {
   override def receive: Receive = {
-    case DockerHashRequest(image, _) => sender ! DockerHashResponseSuccess(DockerHashResult("alg", "hash"))
+    case request: DockerHashRequest => sender ! DockerHashSuccessResponse(DockerHashResult("alg", "hash"), request)
   }
 }
 
diff --git a/engine/src/test/scala/cromwell/DeclarationWorkflowSpec.scala b/engine/src/test/scala/cromwell/DeclarationWorkflowSpec.scala
index ca2b56de7..65a541df6 100644
--- a/engine/src/test/scala/cromwell/DeclarationWorkflowSpec.scala
+++ b/engine/src/test/scala/cromwell/DeclarationWorkflowSpec.scala
@@ -1,7 +1,7 @@
 package cromwell
 
-import wdl4s.types.{WdlFileType, WdlStringType}
-import wdl4s.{ImportResolver, WdlNamespaceWithWorkflow, WorkflowInput}
+import wdl4s.wdl.types.{WdlFileType, WdlStringType}
+import wdl4s.wdl.{ImportResolver, WdlNamespaceWithWorkflow, WorkflowInput}
 import cromwell.util.SampleWdl
 import org.scalatest.{Matchers, WordSpecLike}
 
@@ -9,7 +9,7 @@ import org.scalatest.{Matchers, WordSpecLike}
 class DeclarationWorkflowSpec extends Matchers with WordSpecLike {
   "A workflow with declarations in it" should {
     "compute inputs properly" in {
-      WdlNamespaceWithWorkflow.load(SampleWdl.DeclarationsWorkflow.wdlSource(runtime=""), Seq.empty[ImportResolver]).get.workflow.inputs shouldEqual Map(
+      WdlNamespaceWithWorkflow.load(SampleWdl.DeclarationsWorkflow.workflowSource(runtime=""), Seq.empty[ImportResolver]).get.workflow.inputs shouldEqual Map(
         "two_step.cat.file" -> WorkflowInput("two_step.cat.file", WdlFileType),
         "two_step.cgrep.str_decl" -> WorkflowInput("two_step.cgrep.str_decl", WdlStringType),
         "two_step.cgrep.pattern" -> WorkflowInput("two_step.cgrep.pattern", WdlStringType),
diff --git a/engine/src/test/scala/cromwell/FilePassingWorkflowSpec.scala b/engine/src/test/scala/cromwell/FilePassingWorkflowSpec.scala
index d6d059467..6521286dc 100644
--- a/engine/src/test/scala/cromwell/FilePassingWorkflowSpec.scala
+++ b/engine/src/test/scala/cromwell/FilePassingWorkflowSpec.scala
@@ -2,11 +2,11 @@ package cromwell
 
 import akka.testkit._
 import cromwell.util.SampleWdl
-import wdl4s.values.{WdlFile, WdlString}
+import wdl4s.wdl.values.{WdlFile, WdlString}
 
 import scala.concurrent.duration._
 
-class FilePassingWorkflowSpec extends CromwellTestKitSpec {
+class FilePassingWorkflowSpec extends CromwellTestKitWordSpec {
   "A workflow that passes files between tasks" should {
     "pass files properly" in {
       runWdlAndAssertOutputs(
diff --git a/engine/src/test/scala/cromwell/MapWorkflowSpec.scala b/engine/src/test/scala/cromwell/MapWorkflowSpec.scala
index 04031fef5..296de9ad2 100644
--- a/engine/src/test/scala/cromwell/MapWorkflowSpec.scala
+++ b/engine/src/test/scala/cromwell/MapWorkflowSpec.scala
@@ -3,17 +3,17 @@ package cromwell
 import akka.testkit._
 import cromwell.core.path.DefaultPathBuilder
 import cromwell.util.SampleWdl
-import wdl4s.expression.{NoFunctions, WdlFunctions}
-import wdl4s.types.{WdlFileType, WdlIntegerType, WdlMapType, WdlStringType}
-import wdl4s.values._
-import wdl4s.{ImportResolver, WdlNamespaceWithWorkflow}
+import wdl4s.wdl.expression.{NoFunctions, WdlFunctions}
+import wdl4s.wdl.types.{WdlFileType, WdlIntegerType, WdlMapType, WdlStringType}
+import wdl4s.wdl.values._
+import wdl4s.wdl.{ImportResolver, WdlNamespaceWithWorkflow}
 
 import scala.util.{Success, Try}
 
-class MapWorkflowSpec extends CromwellTestKitSpec {
+class MapWorkflowSpec extends CromwellTestKitWordSpec {
   private val pwd = DefaultPathBuilder.get(".")
   private val sampleWdl = SampleWdl.MapLiteral(pwd)
-  val ns = WdlNamespaceWithWorkflow.load(sampleWdl.wdlSource(), Seq.empty[ImportResolver]).get
+  val ns = WdlNamespaceWithWorkflow.load(sampleWdl.workflowSource(), Seq.empty[ImportResolver]).get
   val expectedMap = WdlMap(WdlMapType(WdlFileType, WdlStringType), Map(
     WdlFile("f1") -> WdlString("alice"),
     WdlFile("f2") -> WdlString("bob"),
diff --git a/engine/src/test/scala/cromwell/MetadataWatchActor.scala b/engine/src/test/scala/cromwell/MetadataWatchActor.scala
index cb8802e22..0a47bc38b 100644
--- a/engine/src/test/scala/cromwell/MetadataWatchActor.scala
+++ b/engine/src/test/scala/cromwell/MetadataWatchActor.scala
@@ -1,17 +1,18 @@
 package cromwell
 
-import akka.actor.{Actor, Props}
+import akka.actor.{Actor, ActorLogging, Props}
 import cromwell.core.Dispatcher.EngineDispatcher
 import cromwell.services.metadata.{MetadataEvent, MetadataJobKey, MetadataString, MetadataValue}
 import cromwell.services.metadata.MetadataService.PutMetadataAction
 import MetadataWatchActor._
+import cromwell.services.keyvalue.KeyValueServiceActor.{KvPut, KvPutSuccess}
 
 import scala.concurrent.Promise
 
 // This actor stands in for the service registry and watches for metadata messages that match the optional `matcher`.
 // If there is no predicate then use `ignoringBehavior` which ignores all messages.  This is here because there is no
 // WorkflowManagerActor in this test that would spin up a real ServiceRegistry.
-final case class MetadataWatchActor(promise: Promise[Unit], matchers: Matcher*) extends Actor {
+final case class MetadataWatchActor(promise: Promise[Unit], matchers: Matcher*) extends Actor with ActorLogging {
 
   var unsatisfiedMatchers = matchers
 
@@ -23,7 +24,9 @@ final case class MetadataWatchActor(promise: Promise[Unit], matchers: Matcher*)
         ()
       }
     case PutMetadataAction(_) => // Superfluous message. Ignore
-    case _ => throw new Exception("Invalid message to MetadataWatchActor")
+    // Because the MetadataWatchActor is sometimes used in place of the ServiceRegistryActor, this allows WFs to continue:
+    case kvPut: KvPut => sender ! KvPutSuccess(kvPut)
+    case other => throw new Exception(s"Invalid message to MetadataWatchActor: $other")
   }
 }
 
@@ -69,6 +72,6 @@ object MetadataWatchActor {
     }
   }
 
-  val failurePattern = """failures\[\d*\].message"""
+  val failurePattern = """failures\[\d*\].*\:message"""
   final case class FailureMatcher(value: String) extends KeyMatchesRegexAndValueContainsStringMatcher(failurePattern, value) { }
 }
diff --git a/engine/src/test/scala/cromwell/MultipleFilesWithSameNameWorkflowSpec.scala b/engine/src/test/scala/cromwell/MultipleFilesWithSameNameWorkflowSpec.scala
index aa05c08f6..4d0e4f51a 100644
--- a/engine/src/test/scala/cromwell/MultipleFilesWithSameNameWorkflowSpec.scala
+++ b/engine/src/test/scala/cromwell/MultipleFilesWithSameNameWorkflowSpec.scala
@@ -2,10 +2,10 @@ package cromwell
 
 import akka.testkit._
 import cromwell.util.SampleWdl
-import wdl4s.values.WdlString
+import wdl4s.wdl.values.WdlString
 
 
-class MultipleFilesWithSameNameWorkflowSpec extends CromwellTestKitSpec {
+class MultipleFilesWithSameNameWorkflowSpec extends CromwellTestKitWordSpec {
   "A workflow with two file inputs that have the same name" should {
     "not clobber one file with the contents of another" in {
       runWdlAndAssertOutputs(
diff --git a/engine/src/test/scala/cromwell/OptionalParamWorkflowSpec.scala b/engine/src/test/scala/cromwell/OptionalParamWorkflowSpec.scala
index 4cbfb9ac5..f9db993f9 100644
--- a/engine/src/test/scala/cromwell/OptionalParamWorkflowSpec.scala
+++ b/engine/src/test/scala/cromwell/OptionalParamWorkflowSpec.scala
@@ -1,9 +1,9 @@
 package cromwell
 
 import org.scalatest.{Matchers, WordSpecLike}
-import wdl4s.WdlNamespace
-import wdl4s.expression.NoFunctions
-import wdl4s.values.{WdlFile, WdlString}
+import wdl4s.wdl.WdlNamespace
+import wdl4s.wdl.expression.NoFunctions
+import wdl4s.wdl.values.{WdlFile, WdlString}
 
 
 class OptionalParamWorkflowSpec extends Matchers with WordSpecLike {
diff --git a/engine/src/test/scala/cromwell/PostfixQuantifierWorkflowSpec.scala b/engine/src/test/scala/cromwell/PostfixQuantifierWorkflowSpec.scala
index a436a6287..1935e3409 100644
--- a/engine/src/test/scala/cromwell/PostfixQuantifierWorkflowSpec.scala
+++ b/engine/src/test/scala/cromwell/PostfixQuantifierWorkflowSpec.scala
@@ -1,11 +1,11 @@
 package cromwell
 
 import akka.testkit._
-import wdl4s.values.WdlString
+import wdl4s.wdl.values.WdlString
 import cromwell.util.SampleWdl
 
 
-class PostfixQuantifierWorkflowSpec extends CromwellTestKitSpec {
+class PostfixQuantifierWorkflowSpec extends CromwellTestKitWordSpec {
   "A task which contains a parameter with a zero-or-more postfix quantifier" should {
     "accept an array of size 3" in {
       runWdlAndAssertOutputs(
diff --git a/engine/src/test/scala/cromwell/RestartWorkflowSpec.scala b/engine/src/test/scala/cromwell/RestartWorkflowSpec.scala
index c8f701d4d..acb0afbb9 100644
--- a/engine/src/test/scala/cromwell/RestartWorkflowSpec.scala
+++ b/engine/src/test/scala/cromwell/RestartWorkflowSpec.scala
@@ -6,20 +6,23 @@ import cromwell.core.Tags._
 import cromwell.core._
 import cromwell.engine.workflow.WorkflowDescriptorBuilder
 
-class RestartWorkflowSpec extends CromwellTestKitSpec with WorkflowDescriptorBuilder {
+class RestartWorkflowSpec extends CromwellTestKitWordSpec with WorkflowDescriptorBuilder {
 
   val actorSystem = ActorSystem("RestartWorkflowSpec", ConfigFactory.parseString(CromwellTestKitSpec.ConfigText))
   //val localBackend = new OldStyleLocalBackend(CromwellTestkitSpec.DefaultLocalBackendConfigEntry, actorSystem)
   val sources = WorkflowSourceFilesWithoutImports(
-    wdlSource="""task a {command{}}
-                |workflow w {
-                |  call a
-                |  call a as b
-                |}
-              """.stripMargin,
-    inputsJson="{}",
-    workflowOptionsJson="{}",
-    labelsJson="{}"
+    workflowSource =
+      """task a {command{}}
+        |workflow w {
+        |  call a
+        |  call a as b
+        |}
+      """.stripMargin,
+    workflowType = Option("WDL"),
+    workflowTypeVersion = None,
+    inputsJson = "{}",
+    workflowOptionsJson = "{}",
+    labelsJson = "{}"
   )
 
   "RestartWorkflowSpec" should {
diff --git a/engine/src/test/scala/cromwell/ScatterWorkflowSpec.scala b/engine/src/test/scala/cromwell/ScatterWorkflowSpec.scala
index 306d5b3bd..0f896de48 100644
--- a/engine/src/test/scala/cromwell/ScatterWorkflowSpec.scala
+++ b/engine/src/test/scala/cromwell/ScatterWorkflowSpec.scala
@@ -2,11 +2,11 @@ package cromwell
 
 import akka.testkit._
 import cromwell.core.Tags.DockerTest
-import wdl4s.types.{WdlArrayType, WdlFileType, WdlIntegerType, WdlStringType}
-import wdl4s.values.{WdlArray, WdlFile, WdlInteger, WdlString}
+import wdl4s.wdl.types.{WdlArrayType, WdlFileType, WdlIntegerType, WdlStringType}
+import wdl4s.wdl.values.{WdlArray, WdlFile, WdlInteger, WdlString}
 import cromwell.util.SampleWdl
 
-class ScatterWorkflowSpec extends CromwellTestKitSpec {
+class ScatterWorkflowSpec extends CromwellTestKitWordSpec {
   "A workflow with a stand-alone scatter block in it" should {
     "run properly" in {
       runWdlAndAssertOutputs(
diff --git a/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala b/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala
index 511cbaa27..f67e4e3cd 100644
--- a/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala
+++ b/engine/src/test/scala/cromwell/SimpleWorkflowActorSpec.scala
@@ -7,7 +7,7 @@ import akka.testkit._
 import com.typesafe.config.ConfigFactory
 import cromwell.MetadataWatchActor.{FailureMatcher, Matcher}
 import cromwell.SimpleWorkflowActorSpec._
-import cromwell.core.{WorkflowId, WorkflowSourceFilesWithoutImports}
+import cromwell.core.{SimpleIoActor, WorkflowId, WorkflowSourceFilesWithoutImports}
 import cromwell.engine.backend.BackendSingletonCollection
 import cromwell.engine.workflow.WorkflowActor
 import cromwell.engine.workflow.WorkflowActor._
@@ -28,23 +28,32 @@ object SimpleWorkflowActorSpec {
     promise: Promise[Unit])
 }
 
-class SimpleWorkflowActorSpec extends CromwellTestKitSpec with BeforeAndAfter {
+class SimpleWorkflowActorSpec extends CromwellTestKitWordSpec with BeforeAndAfter {
 
   private def buildWorkflowActor(sampleWdl: SampleWdl,
                                  rawInputsOverride: String,
                                  workflowId: WorkflowId,
                                  matchers: Matcher*): TestableWorkflowActorAndMetadataPromise = {
-    val workflowSources = WorkflowSourceFilesWithoutImports(sampleWdl.wdlSource(), rawInputsOverride, "{}", "{}")
+    val workflowSources = WorkflowSourceFilesWithoutImports(
+      workflowSource = sampleWdl.workflowSource(),
+      workflowType = Option("WDL"),
+      workflowTypeVersion = None,
+      inputsJson = rawInputsOverride,
+      workflowOptionsJson = "{}",
+      labelsJson = "{}"
+    )
     val promise = Promise[Unit]()
     val watchActor = system.actorOf(MetadataWatchActor.props(promise, matchers: _*), s"service-registry-$workflowId-${UUID.randomUUID()}")
     val supervisor = TestProbe()
     val workflowActor = TestFSMRef(
       factory = new WorkflowActor(workflowId, StartNewWorkflow, workflowSources, ConfigFactory.load(),
+        ioActor = system.actorOf(SimpleIoActor.props),
         serviceRegistryActor = watchActor,
         workflowLogCopyRouter = system.actorOf(Props.empty, s"workflow-copy-log-router-$workflowId-${UUID.randomUUID()}"),
         jobStoreActor = system.actorOf(AlwaysHappyJobStoreActor.props),
         subWorkflowStoreActor = system.actorOf(AlwaysHappySubWorkflowStoreActor.props),
         callCacheReadActor = system.actorOf(EmptyCallCacheReadActor.props),
+        callCacheWriteActor = system.actorOf(EmptyCallCacheWriteActor.props),
         dockerHashActor = system.actorOf(EmptyDockerHashActor.props),
         jobTokenDispenserActor = system.actorOf(JobExecutionTokenDispenserActor.props),
         backendSingletonCollection = BackendSingletonCollection(Map("Local" -> None)),
@@ -64,7 +73,7 @@ class SimpleWorkflowActorSpec extends CromwellTestKitSpec with BeforeAndAfter {
 
   "A WorkflowActor" should {
     "start, run, succeed and die" in {
-      val TestableWorkflowActorAndMetadataPromise(workflowActor, supervisor, _) = buildWorkflowActor(SampleWdl.HelloWorld, SampleWdl.HelloWorld.wdlJson, workflowId)
+      val TestableWorkflowActorAndMetadataPromise(workflowActor, supervisor, _) = buildWorkflowActor(SampleWdl.HelloWorld, SampleWdl.HelloWorld.workflowJson, workflowId)
       val probe = TestProbe()
       probe watch workflowActor
       startingCallsFilter("wf_hello.hello") {
@@ -106,7 +115,7 @@ class SimpleWorkflowActorSpec extends CromwellTestKitSpec with BeforeAndAfter {
       try {
         Await.result(promise.future, TestExecutionTimeout)
       } catch {
-        case e: Throwable =>
+        case _: Throwable =>
           val info = failureMatcher.nearMissInformation
           fail(s"We didn't see the expected error message $expectedError within $TestExecutionTimeout. ${info.mkString(", ")}")
       }
@@ -122,7 +131,7 @@ class SimpleWorkflowActorSpec extends CromwellTestKitSpec with BeforeAndAfter {
     "fail when a call fails" in {
       val expectedError = "Job wf_goodbye.goodbye:NA:1 exited with return code 1 which has not been declared as a valid return code. See 'continueOnReturnCode' runtime attribute for more details." 
       val failureMatcher = FailureMatcher(expectedError)
-      val TestableWorkflowActorAndMetadataPromise(workflowActor, supervisor, promise) = buildWorkflowActor(SampleWdl.GoodbyeWorld, SampleWdl.GoodbyeWorld.wdlJson, workflowId, failureMatcher)
+      val TestableWorkflowActorAndMetadataPromise(workflowActor, supervisor, promise) = buildWorkflowActor(SampleWdl.GoodbyeWorld, SampleWdl.GoodbyeWorld.workflowJson, workflowId, failureMatcher)
       val probe = TestProbe()
       probe watch workflowActor
       startingCallsFilter("wf_goodbye.goodbye") {
@@ -141,14 +150,14 @@ class SimpleWorkflowActorSpec extends CromwellTestKitSpec with BeforeAndAfter {
     "gracefully handle malformed WDL" in {
       val expectedError = "Variable 'bfile' not found"
       val failureMatcher = FailureMatcher(expectedError)
-      val TestableWorkflowActorAndMetadataPromise(workflowActor, supervisor, promise) = buildWorkflowActor(SampleWdl.CoercionNotDefined, SampleWdl.CoercionNotDefined.wdlJson, workflowId, failureMatcher)
+      val TestableWorkflowActorAndMetadataPromise(workflowActor, supervisor, promise) = buildWorkflowActor(SampleWdl.CoercionNotDefined, SampleWdl.CoercionNotDefined.workflowJson, workflowId, failureMatcher)
       val probe = TestProbe()
       probe watch workflowActor
       workflowActor ! StartWorkflowCommand
       try {
         Await.result(promise.future, TestExecutionTimeout)
       } catch {
-        case e: Throwable =>
+        case _: Throwable =>
           val info = failureMatcher.nearMissInformation
           val errorString =
             if (info.nonEmpty) "We had a near miss: " + info.mkString(", ")
@@ -168,7 +177,7 @@ class SimpleWorkflowActorSpec extends CromwellTestKitSpec with BeforeAndAfter {
   private def startingCallsFilter[T](callNames: String*)(block: => T): T = {
     import CromwellTestKitSpec.waitForInfo
     within(TestExecutionTimeout) {
-      waitForInfo(s"Starting calls: ${callNames.mkString("", ":NA:1, ", ":NA:1")}$$", 1) {
+      waitForInfo(s"Starting calls: ${callNames.mkString("", ":NA:1, ", ":NA:1")}$$") {
         block
       }
     }
diff --git a/engine/src/test/scala/cromwell/WdlFunctionsAtWorkflowLevelSpec.scala b/engine/src/test/scala/cromwell/WdlFunctionsAtWorkflowLevelSpec.scala
index 186bd0714..2e63123ed 100644
--- a/engine/src/test/scala/cromwell/WdlFunctionsAtWorkflowLevelSpec.scala
+++ b/engine/src/test/scala/cromwell/WdlFunctionsAtWorkflowLevelSpec.scala
@@ -2,11 +2,11 @@ package cromwell
 
 import akka.testkit._
 import cromwell.util.SampleWdl
-import wdl4s.types.{WdlMapType, WdlStringType}
-import wdl4s.values.{WdlMap, WdlString}
+import wdl4s.wdl.types.{WdlMapType, WdlStringType}
+import wdl4s.wdl.values.{WdlMap, WdlString}
 
 
-class WdlFunctionsAtWorkflowLevelSpec extends CromwellTestKitSpec {
+class WdlFunctionsAtWorkflowLevelSpec extends CromwellTestKitWordSpec {
   val outputMap = WdlMap(WdlMapType(WdlStringType, WdlStringType), Map(
     WdlString("k1") -> WdlString("v1"),
     WdlString("k2") -> WdlString("v2"),
diff --git a/engine/src/test/scala/cromwell/WorkflowFailSlowSpec.scala b/engine/src/test/scala/cromwell/WorkflowFailSlowSpec.scala
index 2093cf198..c9c6c5f8a 100644
--- a/engine/src/test/scala/cromwell/WorkflowFailSlowSpec.scala
+++ b/engine/src/test/scala/cromwell/WorkflowFailSlowSpec.scala
@@ -5,7 +5,7 @@ import cromwell.util.SampleWdl
 
 
 // TODO: These tests are (and were) somewhat unsatisfactory. They'd be much better if we use TestFSMRefs and TestProbes to simulate job completions against the WorkflowActor and make sure it only completes the workflow at the appropriate time.
-class WorkflowFailSlowSpec extends CromwellTestKitSpec {
+class WorkflowFailSlowSpec extends CromwellTestKitWordSpec {
   val FailFastOptions =
     """
       |{
diff --git a/engine/src/test/scala/cromwell/WorkflowOutputsSpec.scala b/engine/src/test/scala/cromwell/WorkflowOutputsSpec.scala
index b821dd742..8669051e8 100644
--- a/engine/src/test/scala/cromwell/WorkflowOutputsSpec.scala
+++ b/engine/src/test/scala/cromwell/WorkflowOutputsSpec.scala
@@ -5,7 +5,7 @@ import cromwell.util.SampleWdl
 import cromwell.CromwellTestKitSpec.AnyValueIsFine
 
 
-class WorkflowOutputsSpec extends CromwellTestKitSpec {
+class WorkflowOutputsSpec extends CromwellTestKitWordSpec {
   "Workflow outputs" should {
     "use all outputs if none are specified" in {
       runWdlAndAssertOutputs(
diff --git a/engine/src/test/scala/cromwell/engine/EngineFunctionsSpec.scala b/engine/src/test/scala/cromwell/engine/EngineFunctionsSpec.scala
index cdefcaf6a..deb5c7365 100644
--- a/engine/src/test/scala/cromwell/engine/EngineFunctionsSpec.scala
+++ b/engine/src/test/scala/cromwell/engine/EngineFunctionsSpec.scala
@@ -5,8 +5,8 @@ import cromwell.core.path.{DefaultPathBuilder, Path, PathBuilder}
 import org.scalatest.prop.TableDrivenPropertyChecks._
 import org.scalatest.prop.Tables.Table
 import org.scalatest.{FlatSpec, Matchers}
-import wdl4s.expression.{NoFunctions, PureStandardLibraryFunctionsLike, WdlStandardLibraryFunctions}
-import wdl4s.values.{WdlFile, WdlInteger, WdlString, WdlValue}
+import wdl4s.wdl.expression.{NoFunctions, PureStandardLibraryFunctionsLike, WdlStandardLibraryFunctions}
+import wdl4s.wdl.values.{WdlFile, WdlInteger, WdlString, WdlValue}
 
 import scala.util.{Failure, Success, Try}
 
diff --git a/engine/src/test/scala/cromwell/engine/WorkflowAbortSpec.scala b/engine/src/test/scala/cromwell/engine/WorkflowAbortSpec.scala
index 37ed641c8..0157adec2 100644
--- a/engine/src/test/scala/cromwell/engine/WorkflowAbortSpec.scala
+++ b/engine/src/test/scala/cromwell/engine/WorkflowAbortSpec.scala
@@ -1,8 +1,8 @@
 package cromwell.engine
 
-import cromwell.CromwellTestKitSpec
+import cromwell.CromwellTestKitWordSpec
 
-class WorkflowAbortSpec extends CromwellTestKitSpec {
+class WorkflowAbortSpec extends CromwellTestKitWordSpec {
 
   // TODO: When re-enabled, this test also needs to check that child processes have actually been stopped.
   "A WorkflowManagerActor" should {
@@ -14,7 +14,7 @@ class WorkflowAbortSpec extends CromwellTestKitSpec {
 //        val waitThreshold = 10
 //
 //        // Start the workflow:
-//        val workflowId = messageAndWait[WorkflowId](SubmitWorkflow(TripleSleep.wdlSource(), TripleSleep.wdlJson, TripleSleep.rawInputs))
+//        val workflowId = messageAndWait[WorkflowId](SubmitWorkflow(TripleSleep.workflowSource(), TripleSleep.WorkflowJson, TripleSleep.rawInputs))
 //
 //        def waitForStarted(currentAttempt: Int): Unit = {
 //          val status = messageAndWait[Option[WorkflowState]](WorkflowStatus(workflowId))
diff --git a/engine/src/test/scala/cromwell/engine/WorkflowManagerActorSpec.scala b/engine/src/test/scala/cromwell/engine/WorkflowManagerActorSpec.scala
index b22ebc13f..164c3fedb 100644
--- a/engine/src/test/scala/cromwell/engine/WorkflowManagerActorSpec.scala
+++ b/engine/src/test/scala/cromwell/engine/WorkflowManagerActorSpec.scala
@@ -1,11 +1,11 @@
 package cromwell.engine
 
-import cromwell.CromwellTestKitSpec
+import cromwell.CromwellTestKitWordSpec
 import cromwell.engine.workflow.WorkflowDescriptorBuilder
 import cromwell.util.SampleWdl
 
 
-class WorkflowManagerActorSpec extends CromwellTestKitSpec with WorkflowDescriptorBuilder {
+class WorkflowManagerActorSpec extends CromwellTestKitWordSpec with WorkflowDescriptorBuilder {
   override implicit val actorSystem = system
 
   "A WorkflowManagerActor" should {
diff --git a/engine/src/test/scala/cromwell/engine/WorkflowStoreActorSpec.scala b/engine/src/test/scala/cromwell/engine/WorkflowStoreActorSpec.scala
index e59c3991d..b86fb99df 100644
--- a/engine/src/test/scala/cromwell/engine/WorkflowStoreActorSpec.scala
+++ b/engine/src/test/scala/cromwell/engine/WorkflowStoreActorSpec.scala
@@ -1,8 +1,8 @@
 package cromwell.engine
 
 import cats.data.NonEmptyList
-import cromwell.CromwellTestKitSpec
-import cromwell.core.{WorkflowId, WorkflowSourceFilesCollection}
+import cromwell.core.WorkflowSourceFilesCollection
+import cromwell.database.sql.SqlDatabase
 import cromwell.engine.workflow.workflowstore.WorkflowStoreActor._
 import cromwell.engine.workflow.workflowstore.WorkflowStoreEngineActor.{NewWorkflowsToStart, NoNewWorkflowsToStart}
 import cromwell.engine.workflow.workflowstore.WorkflowStoreSubmitActor.{WorkflowSubmittedToStore, WorkflowsBatchSubmittedToStore}
@@ -12,13 +12,22 @@ import cromwell.services.metadata.MetadataService.{GetMetadataQueryAction, Metad
 import cromwell.services.metadata.impl.ReadMetadataActor
 import cromwell.util.EncryptionSpec
 import cromwell.util.SampleWdl.HelloWorld
-import org.scalatest.Matchers
+import cromwell.{CromwellTestKitSpec, CromwellTestKitWordSpec}
+import org.mockito.Mockito._
+import org.scalatest.concurrent.Eventually
+import org.scalatest.{BeforeAndAfter, Matchers}
+import org.specs2.mock.Mockito
 
 import scala.concurrent.duration._
+import scala.concurrent.{ExecutionContext, Future}
 import scala.language.postfixOps
 
-class WorkflowStoreActorSpec extends CromwellTestKitSpec with Matchers {
+class WorkflowStoreActorSpec extends CromwellTestKitWordSpec with Matchers with BeforeAndAfter with Mockito with Eventually {
   val helloWorldSourceFiles = HelloWorld.asWorkflowSources()
+  val helloCwlWorldSourceFiles = HelloWorld.asWorkflowSources(workflowType = Option("CWL"), workflowTypeVersion = Option("v1.0"))
+
+  val database: SqlDatabase = mock[SqlDatabase]
+  when(database.removeDockerHashStoreEntries(any[String])(any[ExecutionContext])).thenReturn(Future.successful(1))
 
   /**
     * Fold down a list of WorkflowToStart's, checking that their IDs are all unique
@@ -45,14 +54,14 @@ class WorkflowStoreActorSpec extends CromwellTestKitSpec with Matchers {
   "The WorkflowStoreActor" should {
     "return an ID for a submitted workflow" in {
       val store = new InMemoryWorkflowStore
-      val storeActor = system.actorOf(WorkflowStoreActor.props(store, CromwellTestKitSpec.ServiceRegistryActorInstance))
+      val storeActor = system.actorOf(WorkflowStoreActor.props(store, CromwellTestKitSpec.ServiceRegistryActorInstance, database))
       storeActor ! SubmitWorkflow(helloWorldSourceFiles)
       expectMsgType[WorkflowSubmittedToStore](10 seconds)
     }
 
     "return 3 IDs for a batch submission of 3" in {
       val store = new InMemoryWorkflowStore
-      val storeActor = system.actorOf(WorkflowStoreActor.props(store, CromwellTestKitSpec.ServiceRegistryActorInstance))
+      val storeActor = system.actorOf(WorkflowStoreActor.props(store, CromwellTestKitSpec.ServiceRegistryActorInstance, database))
       storeActor ! BatchSubmitWorkflows(NonEmptyList.of(helloWorldSourceFiles, helloWorldSourceFiles, helloWorldSourceFiles))
       expectMsgPF(10 seconds) {
         case WorkflowsBatchSubmittedToStore(ids) => ids.toList.size shouldBe 3
@@ -61,11 +70,10 @@ class WorkflowStoreActorSpec extends CromwellTestKitSpec with Matchers {
 
     "fetch exactly N workflows" in {
       val store = new InMemoryWorkflowStore
-      val storeActor = system.actorOf(WorkflowStoreActor.props(store, CromwellTestKitSpec.ServiceRegistryActorInstance))
-      storeActor ! BatchSubmitWorkflows(NonEmptyList.of(helloWorldSourceFiles, helloWorldSourceFiles, helloWorldSourceFiles))
+      val storeActor = system.actorOf(WorkflowStoreActor.props(store, CromwellTestKitSpec.ServiceRegistryActorInstance, database))
+      storeActor ! BatchSubmitWorkflows(NonEmptyList.of(helloWorldSourceFiles, helloWorldSourceFiles, helloCwlWorldSourceFiles))
       val insertedIds = expectMsgType[WorkflowsBatchSubmittedToStore](10 seconds).workflowIds.toList
 
-
       storeActor ! FetchRunnableWorkflows(2)
       expectMsgPF(10 seconds) {
         case NewWorkflowsToStart(workflowNel) =>
@@ -78,6 +86,19 @@ class WorkflowStoreActorSpec extends CromwellTestKitSpec with Matchers {
               state shouldBe WorkflowStoreState.Submitted
           }
       }
+
+      storeActor ! FetchRunnableWorkflows(1)
+      expectMsgPF(10 seconds) {
+        case NewWorkflowsToStart(workflowNel) =>
+          workflowNel.toList.size shouldBe 1
+          checkDistinctIds(workflowNel.toList) shouldBe true
+          workflowNel map {
+            case WorkflowToStart(id, sources, state) =>
+              insertedIds.contains(id) shouldBe true
+              sources shouldBe prettyOptions(helloCwlWorldSourceFiles)
+              state shouldBe WorkflowStoreState.Submitted
+          }
+      }
     }
 
     "fetch encrypted and cleared workflow options" in {
@@ -92,7 +113,7 @@ class WorkflowStoreActorSpec extends CromwellTestKitSpec with Matchers {
 
 
       val store = new InMemoryWorkflowStore
-      val storeActor = system.actorOf(WorkflowStoreActor.props(store, CromwellTestKitSpec.ServiceRegistryActorInstance))
+      val storeActor = system.actorOf(WorkflowStoreActor.props(store, CromwellTestKitSpec.ServiceRegistryActorInstance, database))
       val readMetadataActor = system.actorOf(ReadMetadataActor.props())
       storeActor ! BatchSubmitWorkflows(NonEmptyList.of(optionedSourceFiles))
       val insertedIds = expectMsgType[WorkflowsBatchSubmittedToStore](10 seconds).workflowIds.toList
@@ -105,7 +126,7 @@ class WorkflowStoreActorSpec extends CromwellTestKitSpec with Matchers {
           workflowNel.toList.foreach {
             case WorkflowToStart(id, sources, state) =>
               insertedIds.contains(id) should be(true)
-              sources.wdlSource should be(optionedSourceFiles.wdlSource)
+              sources.workflowSource should be(optionedSourceFiles.workflowSource)
               sources.inputsJson should be(optionedSourceFiles.inputsJson)
               state should be(WorkflowStoreState.Submitted)
 
@@ -117,14 +138,17 @@ class WorkflowStoreActorSpec extends CromwellTestKitSpec with Matchers {
               encryptedJsObject.fields("refresh_token").asJsObject.fields.keys should contain theSameElementsAs
                 Seq("iv", "ciphertext")
 
-              readMetadataActor ! GetMetadataQueryAction(MetadataQuery.forWorkflow(id))
-              expectMsgPF(10 seconds) {
-                case MetadataLookupResponse(_, eventList) =>
-                  val optionsEvent = eventList.find(_.key.key == "submittedFiles:options").get
-                  val clearedJsObject = optionsEvent.value.get.value.parseJson.asJsObject
-                  clearedJsObject.fields.keys should contain theSameElementsAs Seq("key", "refresh_token")
-                  clearedJsObject.fields("key") should be(JsString("value"))
-                  clearedJsObject.fields("refresh_token") should be(JsString("cleared"))
+              // We need to wait for workflow metadata to be flushed before we can successfully query for it
+              eventually(timeout(15 seconds), interval(5 seconds)) {
+                readMetadataActor ! GetMetadataQueryAction(MetadataQuery.forWorkflow(id))
+                expectMsgPF(10 seconds) {
+                  case MetadataLookupResponse(_, eventList) =>
+                    val optionsEvent = eventList.find(_.key.key == "submittedFiles:options").get
+                    val clearedJsObject = optionsEvent.value.get.value.parseJson.asJsObject
+                    clearedJsObject.fields.keys should contain theSameElementsAs Seq("key", "refresh_token")
+                    clearedJsObject.fields("key") should be(JsString("value"))
+                    clearedJsObject.fields("refresh_token") should be(JsString("cleared"))
+                }
               }
           }
       }
@@ -132,11 +156,10 @@ class WorkflowStoreActorSpec extends CromwellTestKitSpec with Matchers {
 
     "return only the remaining workflows if N is larger than size" in {
       val store = new InMemoryWorkflowStore
-      val storeActor = system.actorOf(WorkflowStoreActor.props(store, CromwellTestKitSpec.ServiceRegistryActorInstance))
+      val storeActor = system.actorOf(WorkflowStoreActor.props(store, CromwellTestKitSpec.ServiceRegistryActorInstance, database))
       storeActor ! BatchSubmitWorkflows(NonEmptyList.of(helloWorldSourceFiles, helloWorldSourceFiles, helloWorldSourceFiles))
       val insertedIds = expectMsgType[WorkflowsBatchSubmittedToStore](10 seconds).workflowIds.toList
 
-
       storeActor ! FetchRunnableWorkflows(100)
       expectMsgPF(10 seconds) {
         case NewWorkflowsToStart(workflowNel) =>
@@ -151,24 +174,9 @@ class WorkflowStoreActorSpec extends CromwellTestKitSpec with Matchers {
       }
     }
 
-    "remove workflows which exist" in {
-      val store = new InMemoryWorkflowStore
-      val storeActor = system.actorOf(WorkflowStoreActor.props(store, CromwellTestKitSpec.ServiceRegistryActorInstance))
-      storeActor ! SubmitWorkflow(helloWorldSourceFiles)
-      val id = expectMsgType[WorkflowSubmittedToStore](10 seconds).workflowId
-      storeActor ! RemoveWorkflow(id)
-      storeActor ! FetchRunnableWorkflows(100)
-      expectMsgPF(10 seconds) {
-        case NoNewWorkflowsToStart => // Great
-        case x => fail(s"Unexpected response from supposedly empty WorkflowStore: $x")
-      }
-    }
-
     "remain responsive if you ask to remove a workflow it doesn't have" in {
       val store = new InMemoryWorkflowStore
-      val storeActor = system.actorOf(WorkflowStoreActor.props(store, CromwellTestKitSpec.ServiceRegistryActorInstance))
-      val id = WorkflowId.randomId()
-      storeActor ! RemoveWorkflow(id)
+      val storeActor = system.actorOf(WorkflowStoreActor.props(store, CromwellTestKitSpec.ServiceRegistryActorInstance, database))
 
       storeActor ! FetchRunnableWorkflows(100)
       expectMsgPF(10 seconds) {
diff --git a/engine/src/test/scala/cromwell/engine/backend/mock/DefaultBackendJobExecutionActor.scala b/engine/src/test/scala/cromwell/engine/backend/mock/DefaultBackendJobExecutionActor.scala
index 0763a1c30..95d778dd2 100644
--- a/engine/src/test/scala/cromwell/engine/backend/mock/DefaultBackendJobExecutionActor.scala
+++ b/engine/src/test/scala/cromwell/engine/backend/mock/DefaultBackendJobExecutionActor.scala
@@ -3,8 +3,8 @@ package cromwell.engine.backend.mock
 import akka.actor.{ActorRef, Props}
 import cromwell.backend.BackendJobExecutionActor.{BackendJobExecutionResponse, JobSucceededResponse}
 import cromwell.backend._
-import wdl4s.TaskCall
-import wdl4s.expression.{NoFunctions, WdlStandardLibraryFunctions}
+import wdl4s.wdl.WdlTaskCall
+import wdl4s.wdl.expression.{NoFunctions, WdlStandardLibraryFunctions}
 
 import scala.concurrent.Future
 
@@ -14,7 +14,7 @@ object DefaultBackendJobExecutionActor {
 
 case class DefaultBackendJobExecutionActor(override val jobDescriptor: BackendJobDescriptor, override val configurationDescriptor: BackendConfigurationDescriptor) extends BackendJobExecutionActor {
   override def execute: Future[BackendJobExecutionResponse] = {
-    Future.successful(JobSucceededResponse(jobDescriptor.key, Some(0), (jobDescriptor.call.task.outputs map taskOutputToJobOutput).toMap, None, Seq.empty))
+    Future.successful(JobSucceededResponse(jobDescriptor.key, Some(0), (jobDescriptor.call.task.outputs map taskOutputToJobOutput).toMap, None, Seq.empty, dockerImageUsed = None))
   }
 
   override def recover = execute
@@ -25,12 +25,15 @@ case class DefaultBackendJobExecutionActor(override val jobDescriptor: BackendJo
 class DefaultBackendLifecycleActorFactory(name: String, configurationDescriptor: BackendConfigurationDescriptor)
   extends BackendLifecycleActorFactory {
   override def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor,
-                                                calls: Set[TaskCall],
-                                                serviceRegistryActor: ActorRef): Option[Props] = None
+                                                ioActor: ActorRef,
+                                                calls: Set[WdlTaskCall],
+                                                serviceRegistryActor: ActorRef,
+                                                restarting: Boolean): Option[Props] = None
 
   override def jobExecutionActorProps(jobDescriptor: BackendJobDescriptor,
                                       initializationData: Option[BackendInitializationData],
                                       serviceRegistryActor: ActorRef,
+                                      ioActor: ActorRef,
                                       backendSingletonActor: Option[ActorRef]): Props = {
     DefaultBackendJobExecutionActor.props(jobDescriptor, configurationDescriptor)
   }
diff --git a/engine/src/test/scala/cromwell/engine/backend/mock/RetryableBackendJobExecutionActor.scala b/engine/src/test/scala/cromwell/engine/backend/mock/RetryableBackendJobExecutionActor.scala
index eaaa04abb..6bc7b04ad 100644
--- a/engine/src/test/scala/cromwell/engine/backend/mock/RetryableBackendJobExecutionActor.scala
+++ b/engine/src/test/scala/cromwell/engine/backend/mock/RetryableBackendJobExecutionActor.scala
@@ -2,7 +2,7 @@ package cromwell.engine.backend.mock
 
 import akka.actor.Props
 import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptor, BackendJobExecutionActor}
-import cromwell.backend.BackendJobExecutionActor.{JobFailedRetryableResponse, BackendJobExecutionResponse, JobSucceededResponse}
+import cromwell.backend.BackendJobExecutionActor.{BackendJobExecutionResponse, JobFailedNonRetryableResponse, JobFailedRetryableResponse}
 
 import scala.concurrent.Future
 
@@ -10,15 +10,17 @@ object RetryableBackendJobExecutionActor {
   def props(jobDescriptor: BackendJobDescriptor, configurationDescriptor: BackendConfigurationDescriptor) = Props(RetryableBackendJobExecutionActor(jobDescriptor, configurationDescriptor))
 }
 
-case class RetryableBackendJobExecutionActor(override val jobDescriptor: BackendJobDescriptor, override val configurationDescriptor: BackendConfigurationDescriptor) extends BackendJobExecutionActor {
+final case class RetryableBackendJobExecutionActor(override val jobDescriptor: BackendJobDescriptor, override val configurationDescriptor: BackendConfigurationDescriptor) extends BackendJobExecutionActor {
 
   val attempts = 3
 
   override def execute: Future[BackendJobExecutionResponse] = {
-    if (jobDescriptor.key.attempt < attempts)
+    if (jobDescriptor.key.attempt < attempts) {
       Future.successful(JobFailedRetryableResponse(jobDescriptor.key, new RuntimeException("An apparent transient Exception!"), None))
-    else
-      Future.successful(JobSucceededResponse(jobDescriptor.key, Some(0), (jobDescriptor.call.task.outputs map taskOutputToJobOutput).toMap, None, Seq.empty))
+    }
+    else {
+      Future.successful(JobFailedNonRetryableResponse(jobDescriptor.key, new RuntimeException("A permanent Exception! Yikes, what a pickle!"), None))
+    }
   }
 
   override def recover = execute
diff --git a/engine/src/test/scala/cromwell/engine/backend/mock/RetryableBackendLifecycleActorFactory.scala b/engine/src/test/scala/cromwell/engine/backend/mock/RetryableBackendLifecycleActorFactory.scala
index c88481610..15639c3f1 100644
--- a/engine/src/test/scala/cromwell/engine/backend/mock/RetryableBackendLifecycleActorFactory.scala
+++ b/engine/src/test/scala/cromwell/engine/backend/mock/RetryableBackendLifecycleActorFactory.scala
@@ -2,18 +2,21 @@ package cromwell.engine.backend.mock
 
 import akka.actor.{ActorRef, Props}
 import cromwell.backend._
-import wdl4s.TaskCall
-import wdl4s.expression.{NoFunctions, WdlStandardLibraryFunctions}
+import wdl4s.wdl.WdlTaskCall
+import wdl4s.wdl.expression.{NoFunctions, WdlStandardLibraryFunctions}
 
 class RetryableBackendLifecycleActorFactory(name: String, configurationDescriptor: BackendConfigurationDescriptor)
   extends BackendLifecycleActorFactory {
   override def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor,
-                                                calls: Set[TaskCall],
-                                                serviceRegistryActor: ActorRef): Option[Props] = None
+                                                ioActor: ActorRef,
+                                                calls: Set[WdlTaskCall],
+                                                serviceRegistryActor: ActorRef,
+                                                restarting: Boolean): Option[Props] = None
 
   override def jobExecutionActorProps(jobDescriptor: BackendJobDescriptor,
                                       initializationData: Option[BackendInitializationData],
                                       serviceRegistryActor: ActorRef,
+                                      ioActor: ActorRef,
                                       backendSingletonActor: Option[ActorRef]): Props = {
     RetryableBackendJobExecutionActor.props(jobDescriptor, configurationDescriptor)
   }
diff --git a/engine/src/test/scala/cromwell/engine/backend/mock/package.scala b/engine/src/test/scala/cromwell/engine/backend/mock/package.scala
index a2f914121..6dbf7e925 100644
--- a/engine/src/test/scala/cromwell/engine/backend/mock/package.scala
+++ b/engine/src/test/scala/cromwell/engine/backend/mock/package.scala
@@ -1,9 +1,9 @@
 package cromwell.engine.backend
 
 import cromwell.core.JobOutput
-import wdl4s.TaskOutput
-import wdl4s.types._
-import wdl4s.values._
+import wdl4s.wdl.TaskOutput
+import wdl4s.wdl.types._
+import wdl4s.wdl.values._
 
 package object mock {
 
diff --git a/engine/src/test/scala/cromwell/engine/io/IoActorGcsBatchSpec.scala b/engine/src/test/scala/cromwell/engine/io/IoActorGcsBatchSpec.scala
new file mode 100644
index 000000000..fa611133b
--- /dev/null
+++ b/engine/src/test/scala/cromwell/engine/io/IoActorGcsBatchSpec.scala
@@ -0,0 +1,103 @@
+package cromwell.engine.io
+
+import java.util.UUID
+
+import akka.stream.ActorMaterializer
+import akka.testkit.{ImplicitSender, TestActorRef}
+import cromwell.core.Tags.IntegrationTest
+import cromwell.core.io._
+import cromwell.core.{TestKitSuite, WorkflowOptions}
+import cromwell.filesystems.gcs.auth.ApplicationDefaultMode
+import cromwell.filesystems.gcs.batch.{GcsBatchCopyCommand, GcsBatchCrc32Command, GcsBatchDeleteCommand, GcsBatchSizeCommand}
+import cromwell.filesystems.gcs.{GcsPathBuilder, GcsPathBuilderFactory}
+import org.scalatest.concurrent.Eventually
+import org.scalatest.{FlatSpecLike, Matchers}
+
+import scala.concurrent.{Await, ExecutionContext}
+import scala.concurrent.duration._
+import scala.language.postfixOps
+
+class IoActorGcsBatchSpec extends TestKitSuite with FlatSpecLike with Matchers with ImplicitSender with Eventually {
+  behavior of "IoActor [GCS Batch]"
+
+  implicit val actorSystem = system
+  implicit val ec: ExecutionContext = system.dispatcher
+  implicit val materializer = ActorMaterializer()
+
+  override def afterAll() = {
+    materializer.shutdown()
+    src.delete(swallowIOExceptions = true)
+    dst.delete(swallowIOExceptions = true)
+    srcRegional.delete(swallowIOExceptions = true)
+    dstMultiRegional.delete(swallowIOExceptions = true)
+    super.afterAll()
+  }
+  
+  lazy val gcsPathBuilder = GcsPathBuilderFactory(ApplicationDefaultMode("default"), "cromwell-test")
+  lazy val pathBuilder: GcsPathBuilder = Await.result(gcsPathBuilder.withOptions(WorkflowOptions.empty), 1 second)
+
+  lazy val randomUUID = UUID.randomUUID().toString
+
+  lazy val src = pathBuilder.build(s"gs://cloud-cromwell-dev/unit-test/$randomUUID/testFile.txt").get
+  lazy val dst = pathBuilder.build(s"gs://cloud-cromwell-dev/unit-test/$randomUUID/testFile-copy.txt").get
+  lazy val srcRegional = pathBuilder.build(s"gs://cloud-cromwell-dev-regional/unit-test/$randomUUID/testRegional.txt").get
+  lazy val dstMultiRegional = pathBuilder.build(s"gs://cloud-cromwell-dev/unit-test/$randomUUID/testFileRegional-copy.txt").get
+  
+  override def beforeAll() = {
+    // Write commands can't be batched, so for the sake of this test, just create a file in GCS synchronously here
+    src.write("hello")
+    srcRegional.write("hello")
+    super.beforeAll()
+  }
+  
+  it should "batch queries" taggedAs IntegrationTest in {
+    val testActor = TestActorRef(new IoActor(10, None))
+
+    val copyCommand = GcsBatchCopyCommand(src, dst, overwrite = false)
+    val sizeCommand = GcsBatchSizeCommand(src)
+    val hashCommand = GcsBatchCrc32Command(src)
+    
+    val deleteSrcCommand = GcsBatchDeleteCommand(src, swallowIOExceptions = false)
+    val deleteDstCommand = GcsBatchDeleteCommand(dst, swallowIOExceptions = false)
+
+    testActor ! copyCommand
+    testActor ! sizeCommand
+    testActor ! hashCommand
+    
+    val received1 = receiveN(3, 10 seconds)
+    
+    received1.size shouldBe 3
+    received1 forall { _.isInstanceOf[IoSuccess[_]] } shouldBe true
+    
+    received1 collect {
+      case IoSuccess(_: GcsBatchSizeCommand, fileSize: Long) => fileSize shouldBe 5
+    }
+
+    received1 collect {
+      case IoSuccess(_: GcsBatchCrc32Command, hash: String) => hash shouldBe "mnG7TA=="
+    }
+
+    testActor ! deleteSrcCommand
+    testActor ! deleteDstCommand
+    
+    val received2 = receiveN(2, 10 seconds)
+
+    received2.size shouldBe 2
+    received2 forall { _.isInstanceOf[IoSuccess[_]] } shouldBe true
+    
+    src.exists shouldBe false
+    dst.exists shouldBe false
+  }
+
+  it should "copy files across GCS storage classes" taggedAs IntegrationTest in {
+    val testActor = TestActorRef(new IoActor(10, None))
+
+    val copyCommand = GcsBatchCopyCommand(srcRegional, dstMultiRegional, overwrite = false)
+    
+    testActor ! copyCommand
+
+    expectMsgClass(30 seconds, classOf[IoSuccess[_]])
+
+    dstMultiRegional.exists shouldBe true
+  }
+}
diff --git a/engine/src/test/scala/cromwell/engine/io/IoActorSpec.scala b/engine/src/test/scala/cromwell/engine/io/IoActorSpec.scala
new file mode 100644
index 000000000..5b0ec5594
--- /dev/null
+++ b/engine/src/test/scala/cromwell/engine/io/IoActorSpec.scala
@@ -0,0 +1,174 @@
+package cromwell.engine.io
+
+import java.net.{SocketException, SocketTimeoutException}
+
+import akka.stream.ActorMaterializer
+import akka.testkit.{ImplicitSender, TestActorRef}
+import better.files.File.OpenOptions
+import com.google.cloud.storage.StorageException
+import cromwell.core.TestKitSuite
+import cromwell.core.io.DefaultIoCommand._
+import cromwell.core.io._
+import cromwell.core.path.{DefaultPathBuilder, Path}
+import cromwell.engine.io.gcs.GcsBatchFlow.BatchFailedException
+import org.scalatest.{FlatSpecLike, Matchers}
+
+import scala.concurrent.ExecutionContext
+import scala.concurrent.duration._
+import scala.language.postfixOps
+
+class IoActorSpec extends TestKitSuite with FlatSpecLike with Matchers with ImplicitSender {
+  behavior of "IoActor"
+  
+  implicit val actorSystem = system
+  implicit val ec: ExecutionContext = system.dispatcher
+  implicit val materializer = ActorMaterializer()
+  
+  override def afterAll() = {
+    materializer.shutdown()
+    super.afterAll()
+  }
+  
+  it should "copy a file" in {
+    val testActor = TestActorRef(new IoActor(1, None))
+    
+    val src = DefaultPathBuilder.createTempFile()
+    val dst: Path = src.parent.resolve(src.name + "-dst")
+    
+    val copyCommand = DefaultIoCopyCommand(src, dst, overwrite = true)
+    
+    testActor ! copyCommand
+    expectMsgPF(5 seconds) {
+      case response: IoSuccess[_] => response.command.isInstanceOf[IoCopyCommand] shouldBe true
+      case response: IoFailure[_] => fail("Expected an IoSuccess", response.failure)
+    }
+    
+    dst.toFile should exist
+    src.delete()
+    dst.delete()
+  }
+
+  it should "write to a file" in {
+    val testActor = TestActorRef(new IoActor(1, None))
+
+    val src = DefaultPathBuilder.createTempFile()
+
+    val writeCommand = DefaultIoWriteCommand(src, "hello", OpenOptions.default)
+
+    testActor ! writeCommand
+    expectMsgPF(5 seconds) {
+      case response: IoSuccess[_] => response.command.isInstanceOf[IoWriteCommand] shouldBe true
+      case response: IoFailure[_] => fail("Expected an IoSuccess", response.failure)
+    }
+
+    src.contentAsString shouldBe "hello"
+    src.delete()
+  }
+
+  it should "delete a file" in {
+    val testActor = TestActorRef(new IoActor(1, None))
+
+    val src = DefaultPathBuilder.createTempFile()
+
+    val deleteCommand = DefaultIoDeleteCommand(src, swallowIOExceptions = false)
+
+    testActor ! deleteCommand
+    expectMsgPF(5 seconds) {
+      case response: IoSuccess[_] => response.command.isInstanceOf[IoDeleteCommand] shouldBe true
+      case response: IoFailure[_] => fail("Expected an IoSuccess", response.failure)
+    }
+
+    src.toFile shouldNot exist
+  }
+
+  it should "read a file" in {
+    val testActor = TestActorRef(new IoActor(1, None))
+
+    val src = DefaultPathBuilder.createTempFile()
+    src.write("hello")
+
+    val readCommand = DefaultIoContentAsStringCommand(src)
+
+    testActor ! readCommand
+    expectMsgPF(5 seconds) {
+      case response: IoSuccess[_] => 
+        response.command.isInstanceOf[IoContentAsStringCommand] shouldBe true
+        response.result.asInstanceOf[String] shouldBe "hello"
+      case response: IoFailure[_] => fail("Expected an IoSuccess", response.failure)
+    }
+
+    src.delete()
+  }
+
+  it should "return a file size" in {
+    val testActor = TestActorRef(new IoActor(1, None))
+
+    val src = DefaultPathBuilder.createTempFile()
+    src.write("hello")
+
+    val sizeCommand = DefaultIoSizeCommand(src)
+
+    testActor ! sizeCommand
+    expectMsgPF(5 seconds) {
+      case response: IoSuccess[_] =>
+        response.command.isInstanceOf[IoSizeCommand] shouldBe true
+        response.result.asInstanceOf[Long] shouldBe 5
+      case response: IoFailure[_] => fail("Expected an IoSuccess", response.failure)
+    }
+
+    src.delete()
+  }
+
+  it should "return a file md5 hash (local)" in {
+    val testActor = TestActorRef(new IoActor(1, None))
+
+    val src = DefaultPathBuilder.createTempFile()
+    src.write("hello")
+
+    val hashCommand = DefaultIoHashCommand(src)
+
+    testActor ! hashCommand
+    expectMsgPF(5 seconds) {
+      case response: IoSuccess[_] =>
+        response.command.isInstanceOf[IoHashCommand] shouldBe true
+        response.result.asInstanceOf[String] shouldBe "5d41402abc4b2a76b9719d911017c592"
+      case response: IoFailure[_] => fail("Expected an IoSuccess", response.failure)
+    }
+
+    src.delete()
+  }
+
+  it should "touch a file (local)" in {
+    val testActor = TestActorRef(new IoActor(1, None))
+
+    val src = DefaultPathBuilder.createTempFile()
+    src.write("hello")
+
+    val touchCommand = DefaultIoTouchCommand(src)
+
+    testActor ! touchCommand
+    expectMsgPF(5 seconds) {
+      case _: IoSuccess[_] =>
+      case response: IoFailure[_] => fail("Expected an IoSuccess", response.failure)
+    }
+
+    src.delete()
+  }
+
+  it should "have correct retryable exceptions" in {
+    val retryables = List(
+      new StorageException(500, "message"),
+      new StorageException(502, "message"),
+      new StorageException(503, "message"),
+      new StorageException(504, "message"),
+      new StorageException(408, "message"),
+      new StorageException(429, "message"),
+      BatchFailedException(new Exception),
+      new SocketException(),
+      new SocketTimeoutException()
+    )
+
+    retryables foreach { IoActor.isRetryable(_) shouldBe true }
+    retryables foreach { IoActor.isFatal(_) shouldBe false }
+  }
+}
diff --git a/engine/src/test/scala/cromwell/engine/io/nio/NioFlowSpec.scala b/engine/src/test/scala/cromwell/engine/io/nio/NioFlowSpec.scala
new file mode 100644
index 000000000..e88669ef9
--- /dev/null
+++ b/engine/src/test/scala/cromwell/engine/io/nio/NioFlowSpec.scala
@@ -0,0 +1,216 @@
+package cromwell.engine.io.nio
+
+import java.nio.file.{FileAlreadyExistsException, NoSuchFileException}
+import java.util.UUID
+
+import akka.actor.ActorRef
+import akka.stream.ActorMaterializer
+import akka.stream.scaladsl.{Keep, Sink, Source}
+import com.google.cloud.storage.StorageException
+import cromwell.core.io._
+import cromwell.core.path.DefaultPathBuilder
+import cromwell.core.{CromwellFatalException, TestKitSuite}
+import cromwell.engine.io.IoActor.DefaultCommandContext
+import cromwell.engine.io.IoCommandContext
+import org.scalatest.mockito.MockitoSugar
+import org.scalatest.{AsyncFlatSpecLike, Matchers}
+
+import scala.concurrent.Future
+
+class NioFlowSpec extends TestKitSuite with AsyncFlatSpecLike with Matchers with MockitoSugar with DefaultIoCommandBuilder {
+
+  behavior of "NioFlowSpec"
+
+  val flow = new NioFlow(1, system.scheduler)(system.dispatcher, system).flow
+  
+  implicit val materializer = ActorMaterializer()
+  val replyTo = mock[ActorRef]
+  val readSink = Sink.head[(IoAck[_], IoCommandContext[_])]
+
+  override def afterAll() = {
+    materializer.shutdown()
+    super.afterAll()
+  }
+
+  it should "write to a Nio Path" in {
+    val testPath = DefaultPathBuilder.createTempFile()
+    val context = DefaultCommandContext(writeCommand(testPath, "hello", Seq.empty), replyTo)
+    val testSource = Source.single(context)
+
+    val stream = testSource.via(flow).toMat(readSink)(Keep.right)
+
+    stream.run() map { _ =>
+      assert(testPath.contentAsString == "hello")
+    }
+  }
+
+  it should "read from a Nio Path" in {
+    val testPath = DefaultPathBuilder.createTempFile()
+    testPath.write("hello")
+    
+    val context = DefaultCommandContext(contentAsStringCommand(testPath), replyTo)
+    val testSource = Source.single(context)
+
+    val stream = testSource.via(flow).toMat(readSink)(Keep.right)
+    
+    stream.run() map {
+      case (success: IoSuccess[_], _) => assert(success.result.asInstanceOf[String] == "hello")
+      case _ => fail("read returned an unexpected message")
+    }
+  }
+
+  it should "get size from a Nio Path" in {
+    val testPath = DefaultPathBuilder.createTempFile()
+    testPath.write("hello")
+
+    val context = DefaultCommandContext(sizeCommand(testPath), replyTo)
+    val testSource = Source.single(context)
+
+    val stream = testSource.via(flow).toMat(readSink)(Keep.right)
+
+    stream.run() map {
+      case (success: IoSuccess[_], _) => assert(success.result.asInstanceOf[Long] == 5)
+      case _ => fail("size returned an unexpected message")
+    }
+  }
+  
+  it should "get hash from a Nio Path" in {
+    val testPath = DefaultPathBuilder.createTempFile()
+    testPath.write("hello")
+
+    val context = DefaultCommandContext(hashCommand(testPath), replyTo)
+    val testSource = Source.single(context)
+
+    val stream = testSource.via(flow).toMat(readSink)(Keep.right)
+
+    stream.run() map {
+      case (success: IoSuccess[_], _) => assert(success.result.asInstanceOf[String] == "5d41402abc4b2a76b9719d911017c592")
+      case _ => fail("hash returned an unexpected message")
+    }
+  }
+
+  it should "copy Nio paths" in {
+    val testPath = DefaultPathBuilder.createTempFile()
+    val testCopyPath = testPath.sibling(UUID.randomUUID().toString)
+
+    val context = DefaultCommandContext(copyCommand(testPath, testCopyPath, overwrite = false), replyTo)
+
+    val testSource = Source.single(context)
+
+    val stream = testSource.via(flow).toMat(readSink)(Keep.right)
+
+    stream.run() map {
+      case (_: IoSuccess[_], _) => assert(testCopyPath.exists)
+      case _ => fail("copy returned an unexpected message")
+    }
+  }
+
+  it should "copy Nio paths with overwrite true" in {
+    val testPath = DefaultPathBuilder.createTempFile()
+    testPath.write("goodbye")
+    
+    val testCopyPath = DefaultPathBuilder.createTempFile()
+    testCopyPath.write("hello")
+    
+    val context = DefaultCommandContext(copyCommand(testPath, testCopyPath, overwrite = true), replyTo)
+
+    val testSource = Source.single(context)
+
+    val stream = testSource.via(flow).toMat(readSink)(Keep.right)
+
+    stream.run() map {
+      case (_: IoSuccess[_], _) =>
+        assert(testCopyPath.exists)
+        assert(testCopyPath.contentAsString == "goodbye")
+      case _ => fail("copy returned an unexpected message")
+    }
+  }
+
+  it should "copy Nio paths with overwrite false" in {
+    val testPath = DefaultPathBuilder.createTempFile()
+    val testCopyPath = DefaultPathBuilder.createTempFile()
+
+    val context = DefaultCommandContext(copyCommand(testPath, testCopyPath, overwrite = false), replyTo)
+
+    val testSource = Source.single(context)
+
+    val stream = testSource.via(flow).toMat(readSink)(Keep.right)
+
+   stream.run() map {
+      case (failure: IoFailure[_], _) =>
+        assert(failure.failure.isInstanceOf[CromwellFatalException])
+        assert(failure.failure.getCause.isInstanceOf[FileAlreadyExistsException])
+      case _ => fail("copy returned an unexpected message")
+    }
+  }
+
+  it should "delete a Nio path" in {
+    val testPath = DefaultPathBuilder.createTempFile()
+    val context = DefaultCommandContext(deleteCommand(testPath, swallowIoExceptions = false), replyTo)
+    val testSource = Source.single(context)
+
+    val stream = testSource.via(flow).toMat(readSink)(Keep.right)
+
+    stream.run() map {
+      case (_: IoSuccess[_], _) => assert(!testPath.exists)
+      case _ => fail("delete returned an unexpected message")
+    }
+  }
+
+  it should "delete a Nio path with swallowIoExceptions true" in {
+    val testPath = DefaultPathBuilder.build("/this/does/not/exist").get
+
+    val context = DefaultCommandContext(deleteCommand(testPath, swallowIoExceptions = true), replyTo)
+
+    val testSource = Source.single(context)
+
+    val stream = testSource.via(flow).toMat(readSink)(Keep.right)
+
+    stream.run() map {
+      case (_: IoSuccess[_], _) => assert(!testPath.exists)
+      case _ => fail("delete returned an unexpected message")
+    }
+  }
+
+  it should "delete a Nio path with swallowIoExceptions false" in {
+    val testPath = DefaultPathBuilder.build("/this/does/not/exist").get
+
+    val context = DefaultCommandContext(deleteCommand(testPath, swallowIoExceptions = false), replyTo)
+
+    val testSource = Source.single(context)
+
+    val stream = testSource.via(flow).toMat(readSink)(Keep.right)
+
+    stream.run() map {
+      case (failure: IoFailure[_], _) =>
+        assert(failure.failure.isInstanceOf[CromwellFatalException])
+        assert(failure.failure.getCause.isInstanceOf[NoSuchFileException])
+      case other @ _ => fail(s"delete returned an unexpected message")
+    }
+  }
+
+  it should "retry on retryable exceptions" in {
+    val testPath = DefaultPathBuilder.build("does/not/matter").get
+
+    val context = DefaultCommandContext(contentAsStringCommand(testPath), replyTo)
+
+    val testSource = Source.single(context)
+
+    val customFlow = new NioFlow(1, system.scheduler, 3)(system.dispatcher, system) {
+      private var tries = 0
+      override def handleSingleCommand(ioSingleCommand: IoCommand[_]) = {
+        tries += 1
+        if (tries < 3) Future.failed(new StorageException(500, "message"))
+        else Future.successful(IoSuccess(ioSingleCommand, "content"))
+       }
+    }.flow
+    
+    val stream = testSource.via(customFlow).toMat(readSink)(Keep.right)
+
+    stream.run() map {
+      case (success: IoSuccess[_], _) => assert(success.result.asInstanceOf[String] == "content")
+      case _ => fail("read returned an unexpected message")
+    }
+  }
+
+}
diff --git a/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala
index 820091fef..049b228d8 100644
--- a/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala
+++ b/engine/src/test/scala/cromwell/engine/workflow/SingleWorkflowRunnerActorSpec.scala
@@ -4,12 +4,14 @@ import java.time.OffsetDateTime
 
 import akka.actor._
 import akka.pattern.ask
-import akka.testkit.TestKit
+import akka.stream.ActorMaterializer
 import akka.util.Timeout
 import com.typesafe.config.ConfigFactory
 import cromwell.CromwellTestKitSpec._
-import cromwell.core.WorkflowSourceFilesCollection
+import cromwell._
 import cromwell.core.path.{DefaultPathBuilder, Path}
+import cromwell.core.{SimpleIoActor, WorkflowSourceFilesCollection}
+import cromwell.database.sql.SqlDatabase
 import cromwell.engine.backend.BackendSingletonCollection
 import cromwell.engine.workflow.SingleWorkflowRunnerActor.RunWorkflow
 import cromwell.engine.workflow.SingleWorkflowRunnerActorSpec._
@@ -17,8 +19,8 @@ import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor
 import cromwell.engine.workflow.workflowstore.{InMemoryWorkflowStore, WorkflowStoreActor}
 import cromwell.util.SampleWdl
 import cromwell.util.SampleWdl.{ExpressionsInInputs, GoodbyeWorld, ThreeStep}
-import cromwell._
 import org.scalatest.prop.{TableDrivenPropertyChecks, TableFor3}
+import org.specs2.mock.Mockito
 import spray.json._
 
 import scala.concurrent.Await
@@ -41,38 +43,41 @@ object SingleWorkflowRunnerActorSpec {
 
   implicit class OptionJsValueEnhancer(val jsValue: Option[JsValue]) extends AnyVal {
     def toOffsetDateTime = OffsetDateTime.parse(jsValue.toStringValue)
-    def toStringValue = jsValue.get.asInstanceOf[JsString].value
+    def toStringValue = jsValue.getOrElse(JsString("{}")).asInstanceOf[JsString].value
     def toFields = jsValue.get.asJsObject.fields
   }
 
   class TestSingleWorkflowRunnerActor(source: WorkflowSourceFilesCollection,
-                                      metadataOutputPath: Option[Path])
-    extends SingleWorkflowRunnerActor(source, metadataOutputPath) {
+                                      metadataOutputPath: Option[Path])(implicit materializer: ActorMaterializer)
+    extends SingleWorkflowRunnerActor(source, metadataOutputPath, false, false) {
     override lazy val serviceRegistryActor = CromwellTestKitSpec.ServiceRegistryActorInstance
   }
 }
 
-abstract class SingleWorkflowRunnerActorSpec extends CromwellTestKitSpec {
-  private val workflowStore = system.actorOf(WorkflowStoreActor.props(new InMemoryWorkflowStore, dummyServiceRegistryActor))
+abstract class SingleWorkflowRunnerActorSpec extends CromwellTestKitWordSpec with Mockito {
+  private val workflowStore = system.actorOf(WorkflowStoreActor.props(new InMemoryWorkflowStore, dummyServiceRegistryActor, mock[SqlDatabase]))
   private val jobStore = system.actorOf(AlwaysHappyJobStoreActor.props)
+  private val ioActor = system.actorOf(SimpleIoActor.props)
   private val subWorkflowStore = system.actorOf(AlwaysHappySubWorkflowStoreActor.props)
   private val callCacheReadActor = system.actorOf(EmptyCallCacheReadActor.props)
+  private val callCacheWriteActor = system.actorOf(EmptyCallCacheWriteActor.props)
   private val dockerHashActor = system.actorOf(EmptyDockerHashActor.props)
   private val jobTokenDispenserActor = system.actorOf(JobExecutionTokenDispenserActor.props)
 
 
   def workflowManagerActor(): ActorRef = {
     val params = WorkflowManagerActorParams(ConfigFactory.load(),
-      workflowStore,
-      dummyServiceRegistryActor,
-      dummyLogCopyRouter,
-      jobStore,
-      subWorkflowStore,
-      callCacheReadActor,
-      dockerHashActor,
-      jobTokenDispenserActor,
-      BackendSingletonCollection(Map.empty),
-      abortJobsOnTerminate = false,
+      workflowStore = workflowStore,
+      ioActor = ioActor,
+      serviceRegistryActor = dummyServiceRegistryActor,
+      workflowLogCopyRouter = dummyLogCopyRouter,
+      jobStoreActor = jobStore,
+      subWorkflowStoreActor = subWorkflowStore,
+      callCacheReadActor = callCacheReadActor,
+      callCacheWriteActor = callCacheWriteActor,
+      dockerHashActor = dockerHashActor,
+      jobTokenDispenserActor = jobTokenDispenserActor,
+      backendSingletonCollection = BackendSingletonCollection(Map.empty),
       serverMode = false)
     system.actorOf(Props(new WorkflowManagerActor(params)), "WorkflowManagerActor")
   }
@@ -99,7 +104,6 @@ class SingleWorkflowRunnerActorNormalSpec extends SingleWorkflowRunnerActorSpec
           singleWorkflowActor()
         }
       }
-      TestKit.shutdownActorSystem(system, TimeoutDuration)
     }
   }
 }
@@ -118,9 +122,7 @@ class SingleWorkflowRunnerActorWithMetadataSpec extends SingleWorkflowRunnerActo
       singleWorkflowActor(
         sampleWdl = wdlFile,
         outputFile = Option(metadataFile))
-        TestKit.shutdownActorSystem(system, TimeoutDuration)
     }
-    
     eventually {
       val metadataFileContent = metadataFile.contentAsString
       val metadata = metadataFileContent.parseJson.asJsObject.fields
@@ -194,7 +196,6 @@ class SingleWorkflowRunnerActorWithMetadataOnFailureSpec extends SingleWorkflowR
       within(TimeoutDuration) {
         singleWorkflowActor(sampleWdl = GoodbyeWorld, outputFile = Option(metadataFile))
       }
-      TestKit.shutdownActorSystem(system, TimeoutDuration)
 
       val metadata = metadataFile.contentAsString.parseJson.asJsObject.fields
       metadata.get("id") shouldNot be(empty)
@@ -256,7 +257,6 @@ class SingleWorkflowRunnerActorWithBadMetadataSpec extends SingleWorkflowRunnerA
           }
         }
       }
-      TestKit.shutdownActorSystem(system, TimeoutDuration)
     }
   }
 }
@@ -276,7 +276,6 @@ class SingleWorkflowRunnerActorFailureSpec extends SingleWorkflowRunnerActorSpec
           case Failure(e) => e.getMessage should include("expected error")
         }
       }
-      TestKit.shutdownActorSystem(system, TimeoutDuration)
     }
   }
 }
@@ -292,7 +291,6 @@ class SingleWorkflowRunnerActorUnexpectedSpec extends SingleWorkflowRunnerActorS
         }
         assert(!system.whenTerminated.isCompleted)
       }
-      TestKit.shutdownActorSystem(system, TimeoutDuration)
     }
   }
 }
diff --git a/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala
index 35bcb2ca1..23134cba1 100644
--- a/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala
+++ b/engine/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala
@@ -21,7 +21,7 @@ import org.scalatest.concurrent.Eventually
 
 import scala.concurrent.duration._
 
-class WorkflowActorSpec extends CromwellTestKitSpec with WorkflowDescriptorBuilder with BeforeAndAfter with Eventually {
+class WorkflowActorSpec extends CromwellTestKitWordSpec with WorkflowDescriptorBuilder with BeforeAndAfter with Eventually {
   override implicit val actorSystem = system
 
   val mockServiceRegistryActor = TestActorRef(new Actor {
@@ -35,16 +35,18 @@ class WorkflowActorSpec extends CromwellTestKitSpec with WorkflowDescriptorBuild
 
   var currentWorkflowId: WorkflowId = _
   val currentLifecycleActor = TestProbe()
-  val wdlSources = ThreeStep.asWorkflowSources(workflowOptions = mockWorkflowOptions)
-  val descriptor = createMaterializedEngineWorkflowDescriptor(WorkflowId.randomId(), workflowSources = wdlSources)
+  val workflowSources = ThreeStep.asWorkflowSources(workflowOptions = mockWorkflowOptions)
+  val descriptor = createMaterializedEngineWorkflowDescriptor(WorkflowId.randomId(), workflowSources = workflowSources)
   val supervisorProbe = TestProbe()
   val deathwatch = TestProbe()
   val finalizationProbe = TestProbe()
-  val copyWorkflowLogsProbe = TestProbe()
+  var copyWorkflowLogsProbe: TestProbe = _
   val AwaitAlmostNothing = 100.milliseconds
 
   before {
     currentWorkflowId = WorkflowId.randomId()
+
+    copyWorkflowLogsProbe = TestProbe()
   }
 
   private def createWorkflowActor(state: WorkflowActorState) = {
@@ -53,13 +55,15 @@ class WorkflowActorSpec extends CromwellTestKitSpec with WorkflowDescriptorBuild
         finalizationProbe = finalizationProbe,
         workflowId = currentWorkflowId,
         startMode = StartNewWorkflow,
-        workflowSources = wdlSources,
+        workflowSources = workflowSources,
         conf = ConfigFactory.load,
+        ioActor = system.actorOf(SimpleIoActor.props),
         serviceRegistryActor = mockServiceRegistryActor,
         workflowLogCopyRouter = copyWorkflowLogsProbe.ref,
         jobStoreActor = system.actorOf(AlwaysHappyJobStoreActor.props),
         subWorkflowStoreActor = system.actorOf(AlwaysHappySubWorkflowStoreActor.props),
         callCacheReadActor = system.actorOf(EmptyCallCacheReadActor.props),
+        callCacheWriteActor = system.actorOf(EmptyCallCacheWriteActor.props),
         dockerHashActor = system.actorOf(EmptyDockerHashActor.props),
         jobTokenDispenserActor = TestProbe().ref
       ),
@@ -156,9 +160,12 @@ class WorkflowActorSpec extends CromwellTestKitSpec with WorkflowDescriptorBuild
     "copy workflow logs in the event of MaterializeWorkflowDescriptorFailureResponse" in {
       val actor = createWorkflowActor(MaterializingWorkflowDescriptorState)
       deathwatch watch actor
+
+      copyWorkflowLogsProbe.expectNoMsg(AwaitAlmostNothing)
       actor ! MaterializeWorkflowDescriptorFailureResponse(new Exception("Intentionally failing workflow materialization to test log copying"))
-      finalizationProbe.expectNoMsg(AwaitAlmostNothing)
       copyWorkflowLogsProbe.expectMsg(CopyWorkflowLogsActor.Copy(currentWorkflowId, mockDir))
+
+      finalizationProbe.expectNoMsg(AwaitAlmostNothing)
       deathwatch.expectTerminated(actor)
     }
   }
@@ -169,13 +176,15 @@ class MockWorkflowActor(val finalizationProbe: TestProbe,
                         startMode: StartMode,
                         workflowSources: WorkflowSourceFilesCollection,
                         conf: Config,
+                        ioActor: ActorRef,
                         serviceRegistryActor: ActorRef,
                         workflowLogCopyRouter: ActorRef,
                         jobStoreActor: ActorRef,
                         subWorkflowStoreActor: ActorRef,
                         callCacheReadActor: ActorRef,
+                        callCacheWriteActor: ActorRef,
                         dockerHashActor: ActorRef,
-                        jobTokenDispenserActor: ActorRef) extends WorkflowActor(workflowId, startMode, workflowSources, conf, serviceRegistryActor, workflowLogCopyRouter, jobStoreActor, subWorkflowStoreActor, callCacheReadActor, dockerHashActor, jobTokenDispenserActor, BackendSingletonCollection(Map.empty), serverMode = true) {
+                        jobTokenDispenserActor: ActorRef) extends WorkflowActor(workflowId, startMode, workflowSources, conf, ioActor, serviceRegistryActor, workflowLogCopyRouter, jobStoreActor, subWorkflowStoreActor, callCacheReadActor, callCacheWriteActor, dockerHashActor, jobTokenDispenserActor, BackendSingletonCollection(Map.empty), serverMode = true) {
 
   override def makeFinalizationActor(workflowDescriptor: EngineWorkflowDescriptor, jobExecutionMap: JobExecutionMap, worfklowOutputs: CallOutputs) = finalizationProbe.ref
 }
diff --git a/engine/src/test/scala/cromwell/engine/workflow/WorkflowDockerLookupActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/WorkflowDockerLookupActorSpec.scala
new file mode 100644
index 000000000..d2255dcce
--- /dev/null
+++ b/engine/src/test/scala/cromwell/engine/workflow/WorkflowDockerLookupActorSpec.scala
@@ -0,0 +1,274 @@
+package cromwell.engine.workflow
+
+import akka.actor.{ActorRef, Props}
+import akka.testkit.{ImplicitSender, TestActorRef, TestProbe}
+import com.typesafe.config.ConfigFactory
+import cromwell.core.actor.StreamIntegration.BackPressure
+import cromwell.core.{TestKitSuite, WorkflowId}
+import cromwell.database.slick.SlickDatabase
+import cromwell.database.sql.tables.DockerHashStoreEntry
+import cromwell.docker.DockerHashActor.{DockerHashFailedResponse, DockerHashSuccessResponse}
+import cromwell.docker.{DockerHashRequest, DockerHashResult, DockerImageIdentifier, DockerImageIdentifierWithoutHash}
+import cromwell.engine.workflow.WorkflowActor.{RestartExistingWorkflow, StartMode, StartNewWorkflow}
+import cromwell.engine.workflow.WorkflowDockerLookupActor.{DockerHashActorTimeout, WorkflowDockerLookupFailure, WorkflowDockerTerminalFailure}
+import cromwell.engine.workflow.WorkflowDockerLookupActorSpec._
+import cromwell.services.ServicesStore._
+import cromwell.services.SingletonServicesStore
+import org.scalatest.{BeforeAndAfter, FlatSpecLike, Matchers}
+import org.specs2.mock.Mockito
+
+import scala.concurrent.duration._
+import scala.concurrent.{ExecutionContext, Future}
+import scala.language.postfixOps
+
+
+class WorkflowDockerLookupActorSpec extends TestKitSuite("WorkflowDockerLookupActorSpecSystem") with FlatSpecLike with Matchers with ImplicitSender with BeforeAndAfter with Mockito {
+  var workflowId: WorkflowId = _
+  var dockerHashingActor: TestProbe = _
+  var numReads: Int = _
+  var numWrites: Int = _
+
+  before {
+    workflowId = WorkflowId.randomId()
+    dockerHashingActor = TestProbe()
+    numReads = 0
+    numWrites = 0
+  }
+
+  it should "wait and resubmit the docker request when it gets a backpressure message" in {
+    val backpressureWaitTime = 2 seconds
+
+    val lookupActor = TestActorRef(Props(new TestWorkflowDockerLookupActor(workflowId, dockerHashingActor.ref, StartNewWorkflow, backpressureWaitTime)), self)
+    lookupActor ! LatestRequest
+
+    dockerHashingActor.expectMsg(LatestRequest)
+    dockerHashingActor.reply(BackPressure(LatestRequest))
+    // Give a couple of seconds of margin to account for test latency etc...
+    dockerHashingActor.expectMsg(backpressureWaitTime.+(2 seconds), LatestRequest)
+  }
+
+  it should "not look up the same tag again after a successful lookup" in {
+    val db = dbWithWrite {
+      numWrites = numWrites + 1
+      Future.successful(())
+    }
+
+    val lookupActor = TestActorRef(WorkflowDockerLookupActor.props(workflowId, dockerHashingActor.ref, StartNewWorkflow, db))
+    lookupActor ! LatestRequest
+
+    // The WorkflowDockerLookupActor should not have the hash for this tag yet and will need to query the dockerHashingActor.
+    dockerHashingActor.expectMsg(LatestRequest)
+    dockerHashingActor.reply(LatestSuccessResponse)
+    // The WorkflowDockerLookupActor should forward the success message to this actor.
+    expectMsg(LatestSuccessResponse)
+    numWrites should equal(1)
+
+    // Now the WorkflowDockerLookupActor should now have this hash in its mappings and should not query the dockerHashingActor again.
+    lookupActor ! LatestRequest
+    dockerHashingActor.expectNoMsg()
+    // The WorkflowDockerLookupActor should forward the success message to this actor.
+    expectMsg(LatestSuccessResponse)
+    numWrites should equal(1)
+  }
+
+  it should "soldier on after docker hashing actor timeouts" in {
+    val lookupActor = TestActorRef(WorkflowDockerLookupActor.props(workflowId, dockerHashingActor.ref, StartNewWorkflow))
+
+    lookupActor ! LatestRequest
+    lookupActor ! OlderRequest
+
+    val timeout = DockerHashActorTimeout(LatestRequest)
+
+    // The WorkflowDockerLookupActor should not have the hash for this tag yet and will need to query the dockerHashingActor.
+    dockerHashingActor.expectMsg(LatestRequest)
+    dockerHashingActor.expectMsg(OlderRequest)
+    dockerHashingActor.reply(timeout)
+    // Send a response for the older request after sending the timeout.  This should cause a mapping to be entered in the
+    // WorkflowDockerLookupActor for the older request, which will keep the WorkflowDockerLookupActor from querying the
+    // DockerHashActor for this hash again.
+    dockerHashingActor.reply(OlderSuccessResponse)
+
+    val results = receiveN(2, 2 seconds).toSet
+    val failedRequests = results collect {
+      case f: WorkflowDockerLookupFailure if f.request == LatestRequest => f.request
+    }
+
+    failedRequests should equal(Set(LatestRequest))
+
+    // Try again.  The hashing actor should receive the latest message and this time won't time out.
+    lookupActor ! LatestRequest
+    lookupActor ! OlderRequest
+    dockerHashingActor.expectMsg(LatestRequest)
+    dockerHashingActor.reply(LatestSuccessResponse)
+
+    val responses = receiveN(2, 2 seconds).toSet
+    val hashResponses = responses collect { case msg: DockerHashSuccessResponse => msg }
+    // Success after transient timeout failures:
+    hashResponses should equal(Set(LatestSuccessResponse, OlderSuccessResponse))
+  }
+
+  it should "respond appropriately to docker hash lookup failures" in {
+    val lookupActor = TestActorRef(WorkflowDockerLookupActor.props(workflowId, dockerHashingActor.ref, StartNewWorkflow))
+    lookupActor ! LatestRequest
+    lookupActor ! OlderRequest
+
+    // The WorkflowDockerLookupActor should not have the hash for this tag yet and will need to query the dockerHashingActor.
+    dockerHashingActor.expectMsg(LatestRequest)
+    dockerHashingActor.expectMsg(OlderRequest)
+    val olderFailedResponse = DockerHashFailedResponse(new RuntimeException("Lookup failed"), OlderRequest)
+
+    dockerHashingActor.reply(LatestSuccessResponse)
+    dockerHashingActor.reply(olderFailedResponse)
+
+    val results = receiveN(2, 2 seconds).toSet
+    val mixedResponses = results collect {
+      case msg: DockerHashSuccessResponse => msg
+      // Scoop out the request here since we can't match the exception on the whole message.
+      case msg: WorkflowDockerLookupFailure if msg.reason.getMessage == "Failed to get docker hash for ubuntu:older Lookup failed" => msg.request
+    }
+
+    Set(LatestSuccessResponse, OlderRequest) should equal(mixedResponses)
+
+    // Try again, I have a good feeling about this.
+    lookupActor ! OlderRequest
+    dockerHashingActor.expectMsg(OlderRequest)
+    dockerHashingActor.reply(OlderSuccessResponse)
+    expectMsg(OlderSuccessResponse)
+  }
+
+  it should "reuse previously looked up hashes following a restart" in {
+    val db = dbWithQuery {
+      Future.successful(
+        Seq(LatestStoreEntry(workflowId), OlderStoreEntry(workflowId)))
+    }
+
+    val lookupActor = TestActorRef(WorkflowDockerLookupActor.props(workflowId, dockerHashingActor.ref, RestartExistingWorkflow, db))
+
+    lookupActor ! LatestRequest
+    lookupActor ! OlderRequest
+
+    dockerHashingActor.expectNoMsg()
+
+    val results = receiveN(2, 2 seconds).toSet
+    val successes = results collect { case result: DockerHashSuccessResponse => result }
+
+    successes should equal(Set(LatestSuccessResponse, OlderSuccessResponse))
+  }
+
+  it should "not try to look up hashes if not restarting" in {
+    val db = dbWithWrite(Future.successful(()))
+    val lookupActor = TestActorRef(WorkflowDockerLookupActor.props(workflowId, dockerHashingActor.ref, StartNewWorkflow, db))
+
+    lookupActor ! LatestRequest
+    lookupActor ! OlderRequest
+
+    dockerHashingActor.expectMsg(LatestRequest)
+    dockerHashingActor.expectMsg(OlderRequest)
+    dockerHashingActor.reply(LatestSuccessResponse)
+    dockerHashingActor.reply(OlderSuccessResponse)
+
+    val results = receiveN(2, 2 seconds).toSet
+    val successes = results collect { case result: DockerHashSuccessResponse => result }
+
+    successes should equal(Set(LatestSuccessResponse, OlderSuccessResponse))
+  }
+
+  it should "handle hash write errors appropriately" in {
+    val db = dbWithWrite {
+      numWrites = numWrites + 1
+      if (numWrites == 1) Future.failed(new RuntimeException("Fake exception from a test.")) else Future.successful(())
+    }
+
+    val lookupActor = TestActorRef(WorkflowDockerLookupActor.props(workflowId, dockerHashingActor.ref, StartNewWorkflow, db))
+    lookupActor ! LatestRequest
+
+    // The WorkflowDockerLookupActor should not have the hash for this tag yet and will need to query the dockerHashingActor.
+    dockerHashingActor.expectMsg(LatestRequest)
+    dockerHashingActor.reply(LatestSuccessResponse)
+    // The WorkflowDockerLookupActor is going to fail when it tries to write to that broken DB.
+    expectMsgClass(classOf[WorkflowDockerLookupFailure])
+    numWrites should equal(1)
+
+    lookupActor ! LatestRequest
+    // The WorkflowDockerLookupActor will query the dockerHashingActor again.
+    dockerHashingActor.expectMsg(LatestRequest)
+    dockerHashingActor.reply(LatestSuccessResponse)
+    // The WorkflowDockerLookupActor should forward the success message to this actor.
+    expectMsg(LatestSuccessResponse)
+    numWrites should equal(2)
+  }
+
+  it should "emit a terminal failure message if failing to read hashes on restart" in {
+    val db = dbWithQuery {
+      numReads = numReads + 1
+      Future.failed(new Exception("Don't worry this is just a dummy failure in a test"))
+    }
+
+    val lookupActor = TestActorRef(WorkflowDockerLookupActor.props(workflowId, dockerHashingActor.ref, RestartExistingWorkflow, db))
+    lookupActor ! LatestRequest
+
+    dockerHashingActor.expectNoMsg()
+    expectMsgClass(classOf[WorkflowDockerTerminalFailure])
+    numReads should equal(1)
+  }
+
+  it should "emit a terminal failure message if unable to parse hashes read from the database on restart" in {
+    val db = dbWithQuery {
+      numReads = numReads + 1
+      Future.successful(Seq(
+        DockerHashStoreEntry(workflowId.toString, Latest, "md5:AAAAA"),
+        // missing the "algorithm:" preceding the hash value so this should fail parsing.
+        DockerHashStoreEntry(workflowId.toString, Older, "BBBBB")
+      ))
+    }
+
+    val lookupActor = TestActorRef(WorkflowDockerLookupActor.props(workflowId, dockerHashingActor.ref, RestartExistingWorkflow, db))
+    lookupActor ! LatestRequest
+
+    dockerHashingActor.expectNoMsg()
+    expectMsgClass(classOf[WorkflowDockerTerminalFailure])
+    numReads should equal(1)
+  }
+
+  def dbWithWrite(writeFn: => Future[Unit]): SlickDatabase = {
+    databaseInterface(write = _ => writeFn)
+  }
+
+  def dbWithQuery(queryFn: => Future[Seq[DockerHashStoreEntry]]): SlickDatabase = {
+    databaseInterface(query = _ => queryFn)
+  }
+
+  def databaseInterface(query: String => Future[Seq[DockerHashStoreEntry]] = abjectFailure,
+                        write: DockerHashStoreEntry => Future[Unit] = abjectFailure): SlickDatabase = {
+    new SlickDatabase(DatabaseConfig) {
+      override def queryDockerHashStoreEntries(workflowExecutionUuid: String)(implicit ec: ExecutionContext): Future[Seq[DockerHashStoreEntry]] = query(workflowExecutionUuid)
+
+      override def addDockerHashStoreEntry(dockerHashStoreEntry: DockerHashStoreEntry)(implicit ec: ExecutionContext): Future[Unit] = write(dockerHashStoreEntry)
+    }.initialized
+  }
+}
+
+
+object WorkflowDockerLookupActorSpec {
+  val Latest = "ubuntu:latest"
+  val Older = "ubuntu:older"
+
+  val LatestImageId = DockerImageIdentifier.fromString(Latest).get.asInstanceOf[DockerImageIdentifierWithoutHash]
+  val OlderImageId = DockerImageIdentifier.fromString(Older).get.asInstanceOf[DockerImageIdentifierWithoutHash]
+
+  val LatestRequest = DockerHashRequest(LatestImageId)
+  val OlderRequest = DockerHashRequest(OlderImageId)
+
+  def LatestStoreEntry(workflowId: WorkflowId): DockerHashStoreEntry = DockerHashStoreEntry(workflowId.toString, Latest, "md5:AAAAAAAA")
+  def OlderStoreEntry(workflowId: WorkflowId): DockerHashStoreEntry = DockerHashStoreEntry(workflowId.toString, Older, "md5:BBBBBBBB")
+
+  val LatestSuccessResponse = DockerHashSuccessResponse(DockerHashResult("md5", "AAAAAAAA"), LatestRequest)
+  val OlderSuccessResponse = DockerHashSuccessResponse(DockerHashResult("md5", "BBBBBBBB"), OlderRequest)
+
+  val DatabaseConfig = ConfigFactory.load.getConfig("database")
+
+  def abjectFailure[A, B]: A => Future[B] = _ => Future.failed(new RuntimeException("Should not be called!"))
+
+  class TestWorkflowDockerLookupActor(workflowId: WorkflowId, dockerHashingActor: ActorRef, startMode: StartMode, override val backpressureTimeout: FiniteDuration)
+    extends WorkflowDockerLookupActor(workflowId, dockerHashingActor, startMode, SingletonServicesStore.databaseInterface)
+}
diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActorSpec.scala
index 8ae601d85..9cc7cff8d 100644
--- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActorSpec.scala
+++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/MaterializeWorkflowDescriptorActorSpec.scala
@@ -3,7 +3,7 @@ package cromwell.engine.workflow.lifecycle
 import akka.actor.Props
 import akka.testkit.TestDuration
 import com.typesafe.config.ConfigFactory
-import cromwell.CromwellTestKitSpec
+import cromwell.CromwellTestKitWordSpec
 import cromwell.core.labels.{Label, Labels}
 import cromwell.core.{WorkflowId, WorkflowOptions, WorkflowSourceFilesWithoutImports}
 import cromwell.engine.backend.{BackendConfigurationEntry, CromwellBackends}
@@ -13,11 +13,11 @@ import org.scalatest.BeforeAndAfter
 import org.scalatest.mockito.MockitoSugar
 import spray.json.DefaultJsonProtocol._
 import spray.json._
-import wdl4s.values.WdlString
+import wdl4s.wdl.values.WdlString
 
 import scala.concurrent.duration._
 
-class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitSpec with BeforeAndAfter with MockitoSugar {
+class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitWordSpec with BeforeAndAfter with MockitoSugar {
 
   val workflowId = WorkflowId.randomId()
   val minimumConf = ConfigFactory.parseString(
@@ -43,22 +43,28 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitSpec with Be
   val badCustomLabelsFile="""{ "Label1": "valu£1", "--label2": "valuevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevalue" }"""
 
   val validInputsJson = HelloWorld.rawInputs.toJson.toString()
-  val wdlSourceWithDocker = HelloWorld.wdlSource(""" runtime { docker: "ubuntu:latest" } """)
-  val wdlSourceNoDocker = HelloWorld.wdlSource(""" runtime { } """)
+  val workflowSourceWithDocker = HelloWorld.workflowSource(""" runtime { docker: "ubuntu:latest" } """)
+  val workflowSourceNoDocker = HelloWorld.workflowSource(""" runtime { } """)
   val Timeout = 10.second.dilated
-  val NoBehaviourActor = system.actorOf(Props.empty)
+  val NoBehaviorActor = system.actorOf(Props.empty)
 
   before {
   }
 
   after {
-    system.stop(NoBehaviourActor)
+    system.stop(NoBehaviorActor)
   }
 
   "MaterializeWorkflowDescriptorActor" should {
     "accept valid WDL, inputs and options files" in {
-      val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviourActor, workflowId, importLocalFilesystem = false))
-      val sources = WorkflowSourceFilesWithoutImports(wdlSourceNoDocker, validInputsJson, validOptionsFile, validCustomLabelsFile)
+      val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviorActor, workflowId, importLocalFilesystem = false))
+      val sources = WorkflowSourceFilesWithoutImports(
+        workflowSource = workflowSourceNoDocker,
+        workflowType = Option("WDL"),
+        workflowTypeVersion = None,
+        inputsJson = validInputsJson,
+        workflowOptionsJson = validOptionsFile,
+        labelsJson = validCustomLabelsFile)
       materializeWfActor ! MaterializeWorkflowDescriptorCommand(sources, minimumConf)
 
       within(Timeout) {
@@ -75,7 +81,7 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitSpec with Be
             // Default backend assignment is "Local":
             wfDesc.backendAssignments foreach {
               case (call, assignment) if call.task.name.equals("hello") => assignment shouldBe "Local"
-              case (call, assignment) => fail(s"Unexpected call: ${call.task.name}")
+              case (call, _) => fail(s"Unexpected call: ${call.task.name}")
             }
             wfDesc.pathBuilders.size shouldBe 1
           case MaterializeWorkflowDescriptorFailureResponse(reason) => fail(s"Materialization failed with $reason")
@@ -109,8 +115,14 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitSpec with Be
           |  }
           |}
         """.stripMargin
-      val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviourActor, workflowId, importLocalFilesystem = false))
-      val sources = WorkflowSourceFilesWithoutImports(wdl, "{}", defaultDocker, validCustomLabelsFile)
+      val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviorActor, workflowId, importLocalFilesystem = false))
+      val sources = WorkflowSourceFilesWithoutImports(
+        workflowSource = wdl,
+        workflowType = Option("WDL"),
+        workflowTypeVersion = None,
+        inputsJson = "{}",
+        workflowOptionsJson = defaultDocker,
+        labelsJson = validCustomLabelsFile)
       materializeWfActor ! MaterializeWorkflowDescriptorCommand(sources, minimumConf)
 
       within(Timeout) {
@@ -156,8 +168,14 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitSpec with Be
       val cromwellBackends = CromwellBackends(fauxBackendEntries)
 
       // Run the test:
-      val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviourActor, workflowId, cromwellBackends, importLocalFilesystem = false))
-      val sources = WorkflowSourceFilesWithoutImports(wdl, "{}", "{}", validCustomLabelsFile)
+      val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviorActor, workflowId, cromwellBackends, importLocalFilesystem = false))
+      val sources = WorkflowSourceFilesWithoutImports(
+        workflowSource = wdl,
+        workflowType = Option("WDL"),
+        workflowTypeVersion = None,
+        inputsJson = "{}",
+        workflowOptionsJson = "{}",
+        labelsJson = validCustomLabelsFile)
       materializeWfActor ! MaterializeWorkflowDescriptorCommand(sources, differentDefaultBackendConf)
 
       within(Timeout) {
@@ -191,8 +209,14 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitSpec with Be
           |}
         """.stripMargin
 
-      val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviourActor, workflowId, importLocalFilesystem = false))
-      val sources = WorkflowSourceFilesWithoutImports(wdl, "{}", "{}", "{}")
+      val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviorActor, workflowId, importLocalFilesystem = false))
+      val sources = WorkflowSourceFilesWithoutImports(
+        workflowSource = wdl,
+        workflowType = Option("WDL"),
+        workflowTypeVersion = None,
+        inputsJson = "{}",
+        workflowOptionsJson = "{}",
+        labelsJson = "{}")
       materializeWfActor ! MaterializeWorkflowDescriptorCommand(sources, differentDefaultBackendConf)
 
       within(Timeout) {
@@ -200,7 +224,7 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitSpec with Be
           case MaterializeWorkflowDescriptorFailureResponse(reason) =>
             if (!reason.getMessage.contains("Backend for call foo.a ('NoSuchBackend') not registered in configuration file"))
               fail(s"Unexpected failure message from MaterializeWorkflowDescriptorActor: ${reason.getMessage}")
-          case MaterializeWorkflowDescriptorSuccessResponse(wfDesc) => fail("This materialization should not have succeeded!")
+          case _: MaterializeWorkflowDescriptorSuccessResponse => fail("This materialization should not have succeeded!")
           case unknown =>
             fail(s"Unexpected materialization response: $unknown")
         }
@@ -210,15 +234,21 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitSpec with Be
     }
 
     "reject an invalid WDL source" in {
-      val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviourActor, workflowId, importLocalFilesystem = false))
-      val sources = WorkflowSourceFilesWithoutImports(unstructuredFile, validInputsJson, validOptionsFile, validCustomLabelsFile)
+      val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviorActor, workflowId, importLocalFilesystem = false))
+      val sources = WorkflowSourceFilesWithoutImports(
+        workflowSource = unstructuredFile,
+        workflowType = Option("WDL"),
+        workflowTypeVersion = None,
+        inputsJson = validInputsJson,
+        workflowOptionsJson = validOptionsFile,
+        labelsJson = validCustomLabelsFile)
       materializeWfActor ! MaterializeWorkflowDescriptorCommand(sources, minimumConf)
 
       within(Timeout) {
         expectMsgPF() {
           case MaterializeWorkflowDescriptorFailureResponse(reason) =>
             reason.getMessage should startWith("Workflow input processing failed:\nUnable to load namespace from workflow: ERROR: Finished parsing without consuming all tokens.")
-          case MaterializeWorkflowDescriptorSuccessResponse(wfDesc) => fail("This materialization should not have succeeded!")
+          case _: MaterializeWorkflowDescriptorSuccessResponse => fail("This materialization should not have succeeded!")
           case unknown =>
             fail(s"Unexpected materialization response: $unknown")
         }
@@ -234,15 +264,21 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitSpec with Be
           |
           |# no workflow foo { ... } block!!
         """.stripMargin
-      val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviourActor, workflowId, importLocalFilesystem = false))
-      val sources = WorkflowSourceFilesWithoutImports(noWorkflowWdl, validInputsJson, validOptionsFile, validCustomLabelsFile)
+      val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviorActor, workflowId, importLocalFilesystem = false))
+      val sources = WorkflowSourceFilesWithoutImports(
+        workflowSource = noWorkflowWdl,
+        workflowType = Option("WDL"),
+        workflowTypeVersion = None,
+        inputsJson = validInputsJson,
+        workflowOptionsJson = validOptionsFile,
+        labelsJson = validCustomLabelsFile)
       materializeWfActor ! MaterializeWorkflowDescriptorCommand(sources, minimumConf)
 
       within(Timeout) {
         expectMsgPF() {
           case MaterializeWorkflowDescriptorFailureResponse(reason) =>
             reason.getMessage should startWith("Workflow input processing failed:\nUnable to load namespace from workflow: Namespace does not have a local workflow to run")
-          case MaterializeWorkflowDescriptorSuccessResponse(wfDesc) => fail("This materialization should not have succeeded!")
+          case _: MaterializeWorkflowDescriptorSuccessResponse => fail("This materialization should not have succeeded!")
           case unknown =>
             fail(s"Unexpected materialization response: $unknown")
         }
@@ -259,15 +295,21 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitSpec with Be
           |
           |workflow foo {  }
         """.stripMargin
-      val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviourActor, workflowId, importLocalFilesystem = false))
-      val badWdlSources = WorkflowSourceFilesWithoutImports(noWorkflowWdl, validInputsJson, validOptionsFile, validCustomLabelsFile)
+      val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviorActor, workflowId, importLocalFilesystem = false))
+      val badWdlSources = WorkflowSourceFilesWithoutImports(
+        workflowSource = noWorkflowWdl,
+        workflowType = Option("WDL"),
+        workflowTypeVersion = None,
+        inputsJson = validInputsJson,
+        workflowOptionsJson = validOptionsFile,
+        labelsJson = validCustomLabelsFile)
       materializeWfActor ! MaterializeWorkflowDescriptorCommand(badWdlSources, minimumConf)
 
       within(Timeout) {
         expectMsgPF() {
           case MaterializeWorkflowDescriptorFailureResponse(reason) =>
             reason.getMessage should startWith("Workflow input processing failed:\nUnable to load namespace from workflow: Namespace does not have a local workflow to run")
-          case MaterializeWorkflowDescriptorSuccessResponse(wfDesc) => fail("This materialization should not have succeeded!")
+          case _: MaterializeWorkflowDescriptorSuccessResponse => fail("This materialization should not have succeeded!")
           case unknown =>
             fail(s"Unexpected materialization response: $unknown")
         }
@@ -278,15 +320,21 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitSpec with Be
 
 
     "reject an invalid options file" in {
-      val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviourActor, workflowId, importLocalFilesystem = false))
-      val sources = WorkflowSourceFilesWithoutImports(wdlSourceNoDocker, validInputsJson, unstructuredFile, validCustomLabelsFile)
+      val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviorActor, workflowId, importLocalFilesystem = false))
+      val sources = WorkflowSourceFilesWithoutImports(
+        workflowSource = workflowSourceNoDocker,
+        workflowType = Option("WDL"),
+        workflowTypeVersion = None,
+        inputsJson = validInputsJson,
+        workflowOptionsJson = unstructuredFile,
+        labelsJson = validCustomLabelsFile)
       materializeWfActor ! MaterializeWorkflowDescriptorCommand(sources, minimumConf)
 
       within(Timeout) {
         expectMsgPF() {
           case MaterializeWorkflowDescriptorFailureResponse(reason) =>
             reason.getMessage should startWith("Workflow input processing failed:\nWorkflow contains invalid options JSON")
-          case MaterializeWorkflowDescriptorSuccessResponse(wfDesc) => fail("This materialization should not have succeeded!")
+          case _: MaterializeWorkflowDescriptorSuccessResponse => fail("This materialization should not have succeeded!")
           case unknown =>
             fail(s"Unexpected materialization response: $unknown")
         }
@@ -296,8 +344,14 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitSpec with Be
     }
 
     "reject an unstructured labels file" in {
-      val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviourActor, workflowId, importLocalFilesystem = false))
-      val sources = WorkflowSourceFilesWithoutImports(wdlSourceNoDocker, validInputsJson, validOptionsFile, unstructuredFile)
+      val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviorActor, workflowId, importLocalFilesystem = false))
+      val sources = WorkflowSourceFilesWithoutImports(
+        workflowSource = workflowSourceNoDocker,
+        workflowType = Option("WDL"),
+        workflowTypeVersion = None,
+        inputsJson = validInputsJson,
+        workflowOptionsJson = validOptionsFile,
+        labelsJson = unstructuredFile)
       materializeWfActor ! MaterializeWorkflowDescriptorCommand(sources, minimumConf)
 
       within(Timeout) {
@@ -306,7 +360,7 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitSpec with Be
             reason.getMessage should startWith(
               """Workflow input processing failed:
                 |Workflow contains invalid labels JSON: Unexpected character 'u'""".stripMargin)
-          case MaterializeWorkflowDescriptorSuccessResponse(wfDesc) => fail("This materialization should not have succeeded!")
+          case _: MaterializeWorkflowDescriptorSuccessResponse => fail("This materialization should not have succeeded!")
           case unknown =>
             fail(s"Unexpected materialization response: $unknown")
         }
@@ -316,20 +370,27 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitSpec with Be
     }
 
     "reject invalid labels" in {
-      val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviourActor, workflowId, importLocalFilesystem = false))
-      val sources = WorkflowSourceFilesWithoutImports(wdlSourceNoDocker, validInputsJson, validOptionsFile, badCustomLabelsFile)
+      val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviorActor, workflowId, importLocalFilesystem = false))
+      val sources = WorkflowSourceFilesWithoutImports(
+        workflowSource = workflowSourceNoDocker,
+        workflowType = Option("WDL"),
+        workflowTypeVersion = None,
+        inputsJson = validInputsJson,
+        workflowOptionsJson = validOptionsFile,
+        labelsJson = badCustomLabelsFile)
       materializeWfActor ! MaterializeWorkflowDescriptorCommand(sources, minimumConf)
 
       within(Timeout) {
         expectMsgPF() {
           case MaterializeWorkflowDescriptorFailureResponse(reason) =>
-            reason.getMessage should be(
+            val expectedMessage =
               s"""Workflow input processing failed:
-                |Invalid label: Label1 did not match the regex ${Label.LabelRegexPattern}
-                |Invalid label: valu£1 did not match the regex ${Label.LabelRegexPattern}
-                |Invalid label: --label2 did not match the regex ${Label.LabelRegexPattern}
-                |Invalid label: valuevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevalue was 65 characters. The maximum is 63""".stripMargin)
-          case MaterializeWorkflowDescriptorSuccessResponse(wfDesc) => fail("This materialization should not have succeeded!")
+                  |Invalid label: `Label1` did not match the regex ${Label.LabelKeyRegex}.
+                  |Invalid label: `valu£1` did not match the regex ${Label.LabelValueRegex}.
+                  |Invalid label: `--label2` did not match the regex ${Label.LabelKeyRegex}.
+                  |Invalid label: `valuevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevalue` is 65 characters. The maximum is 63.""".stripMargin
+            reason.getMessage shouldBe expectedMessage
+          case _: MaterializeWorkflowDescriptorSuccessResponse => fail("This materialization should not have succeeded!")
           case unknown =>
             fail(s"Unexpected materialization response: $unknown")
         }
@@ -339,15 +400,21 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitSpec with Be
     }
 
     "reject an invalid workflow inputs file" in {
-      val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviourActor, workflowId, importLocalFilesystem = false))
-      val sources = WorkflowSourceFilesWithoutImports(wdlSourceNoDocker, unstructuredFile, validOptionsFile, validCustomLabelsFile)
+      val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviorActor, workflowId, importLocalFilesystem = false))
+      val sources = WorkflowSourceFilesWithoutImports(
+        workflowSource = workflowSourceNoDocker,
+        workflowType = Option("WDL"),
+        workflowTypeVersion = None,
+        inputsJson = unstructuredFile,
+        workflowOptionsJson = validOptionsFile,
+        labelsJson = validCustomLabelsFile)
       materializeWfActor ! MaterializeWorkflowDescriptorCommand(sources, minimumConf)
 
       within(Timeout) {
         expectMsgPF() {
           case MaterializeWorkflowDescriptorFailureResponse(reason) =>
             reason.getMessage should startWith("Workflow input processing failed:\nWorkflow contains invalid inputs JSON")
-          case MaterializeWorkflowDescriptorSuccessResponse(wfDesc) => fail("This materialization should not have succeeded!")
+          case _: MaterializeWorkflowDescriptorSuccessResponse => fail("This materialization should not have succeeded!")
           case unknown =>
             fail(s"Unexpected materialization response: $unknown")
         }
@@ -357,16 +424,22 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitSpec with Be
     }
 
     "reject requests if any required inputs are missing" in {
-      val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviourActor, workflowId, importLocalFilesystem = false))
+      val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviorActor, workflowId, importLocalFilesystem = false))
       val noInputsJson = "{}"
-      val badOptionsSources = WorkflowSourceFilesWithoutImports(wdlSourceNoDocker, noInputsJson, validOptionsFile, validCustomLabelsFile)
+      val badOptionsSources = WorkflowSourceFilesWithoutImports(
+        workflowSource = workflowSourceNoDocker,
+        workflowType = Option("WDL"),
+        workflowTypeVersion = None,
+        inputsJson = noInputsJson,
+        workflowOptionsJson = validOptionsFile,
+        labelsJson = validCustomLabelsFile)
       materializeWfActor ! MaterializeWorkflowDescriptorCommand(badOptionsSources, minimumConf)
 
       within(Timeout) {
         expectMsgPF() {
           case MaterializeWorkflowDescriptorFailureResponse(reason) =>
             reason.getMessage should startWith("Workflow input processing failed:\nRequired workflow input 'wf_hello.hello.addressee' not specified")
-          case MaterializeWorkflowDescriptorSuccessResponse(wfDesc) => fail("This materialization should not have succeeded!")
+          case _: MaterializeWorkflowDescriptorSuccessResponse => fail("This materialization should not have succeeded!")
           case unknown =>
             fail(s"Unexpected materialization response: $unknown")
         }
@@ -384,15 +457,57 @@ class MaterializeWorkflowDescriptorActorSpec extends CromwellTestKitSpec with Be
           |  call bar
           |}
         """.stripMargin
-      val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviourActor, workflowId, importLocalFilesystem = false))
-      val sources = WorkflowSourceFilesWithoutImports(wdl, "{}", validOptionsFile, validCustomLabelsFile)
+      val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviorActor, workflowId, importLocalFilesystem = false))
+      val sources = WorkflowSourceFilesWithoutImports(
+        workflowSource = wdl,
+        workflowType = Option("WDL"),
+        workflowTypeVersion = None,
+        inputsJson = "{}",
+        workflowOptionsJson = validOptionsFile,
+        labelsJson = validCustomLabelsFile)
       materializeWfActor ! MaterializeWorkflowDescriptorCommand(sources, minimumConf)
 
       within(Timeout) {
         expectMsgPF() {
           case MaterializeWorkflowDescriptorFailureResponse(reason) =>
             reason.getMessage should startWith("Workflow input processing failed:\nUnable to load namespace from workflow: ERROR: Value for j is not coerceable into a Int")
-          case MaterializeWorkflowDescriptorSuccessResponse(wfDesc) => fail("This materialization should not have succeeded!")
+          case _: MaterializeWorkflowDescriptorSuccessResponse => fail("This materialization should not have succeeded!")
+          case unknown => fail(s"Unexpected materialization response: $unknown")
+        }
+      }
+
+      system.stop(materializeWfActor)
+    }
+
+    "identify all malformed input file names in an input json" in {
+      val wdl =
+        """
+          |task bar { command { echo foobar } }
+          |workflow foo {
+          |  File bad_one
+          |  File good_one
+          |  File bad_two
+          |
+          |  call bar
+          |}
+        """.stripMargin
+      val jsonInput = Map("foo.bad_one" -> "\"gs://this/is/a/bad/gcs/path.txt", "foo.good_one" -> "\"/local/path/is/ok.txt", "foo.bad_two" -> "\"gs://another/bad/gcs/path.txt").toJson.toString
+      val materializeWfActor = system.actorOf(MaterializeWorkflowDescriptorActor.props(NoBehaviorActor, workflowId, importLocalFilesystem = false))
+      val sources = WorkflowSourceFilesWithoutImports(
+        workflowSource = wdl,
+        workflowType = Option("WDL"),
+        workflowTypeVersion = None,
+        inputsJson = jsonInput,
+        workflowOptionsJson = validOptionsFile,
+        labelsJson = validCustomLabelsFile)
+      materializeWfActor ! MaterializeWorkflowDescriptorCommand(sources, minimumConf)
+
+      within(Timeout) {
+        expectMsgPF() {
+          case MaterializeWorkflowDescriptorFailureResponse(reason) =>
+            reason.getMessage should equal("Workflow input processing failed:\nInvalid value for File input 'foo.bad_one': \"gs://this/is/a/bad/gcs/path.txt starts with a '\"' " +
+              "\nInvalid value for File input 'foo.bad_two': \"gs://another/bad/gcs/path.txt starts with a '\"' ")
+          case _: MaterializeWorkflowDescriptorSuccessResponse => fail("This materialization should not have succeeded!")
           case unknown => fail(s"Unexpected materialization response: $unknown")
         }
       }
diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ExecutionStoreBenchmark.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ExecutionStoreBenchmark.scala
new file mode 100644
index 000000000..a70f68e49
--- /dev/null
+++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ExecutionStoreBenchmark.scala
@@ -0,0 +1,60 @@
+package cromwell.engine.workflow.lifecycle.execution
+
+import cromwell.backend.BackendJobDescriptorKey
+import cromwell.core.ExecutionStatus.{apply => _, _}
+import cromwell.core.{ExecutionStatus, JobKey}
+import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor.CollectorKey
+import cromwell.util.SampleWdl
+import org.scalameter.api._
+import wdl4s.wdl.{WdlTaskCall, WdlNamespaceWithWorkflow}
+import org.scalameter.picklers.Implicits._
+
+/**
+  * Benchmarks the performance of the execution store using ScalaMeter (http://scalameter.github.io/)
+  * This is not run automatically by "sbt test". To run this test specifically, either use intellij integration, or run
+  * sbt "project engine"  "benchmark:test-only cromwell.engine.workflow.lifecycle.execution.ExecutionStoreBenchmark"
+  * sbt benchmark:test will run all ScalaMeter tests
+  */
+object ExecutionStoreBenchmark extends Bench[Double] {
+
+  /* Benchmark configuration */
+  lazy val measurer = new Measurer.Default
+  lazy val executor = SeparateJvmsExecutor(new Executor.Warmer.Default, Aggregator.average, measurer)
+  lazy val reporter = new LoggingReporter[Double]
+  lazy val persistor = Persistor.None
+  
+  val wdl = WdlNamespaceWithWorkflow.load(SampleWdl.PrepareScatterGatherWdl().workflowSource(), Seq.empty).get
+  val prepareCall: WdlTaskCall = wdl.workflow.findCallByName("do_prepare").get.asInstanceOf[WdlTaskCall]
+  val scatterCall: WdlTaskCall = wdl.workflow.findCallByName("do_scatter").get.asInstanceOf[WdlTaskCall]
+  val scatter = wdl.workflow.namespace.scatters.head
+  
+  def makeKey(call: WdlTaskCall, executionStatus: ExecutionStatus)(index: Int) = {
+    BackendJobDescriptorKey(call, Option(index), 1) -> executionStatus
+  }
+  
+  // Generates numbers from 1000 to 10000 with 1000 gap:
+  // 1000, 2000, ..., 10000
+  val sizes: Gen[Int] = Gen.range("size")(10000, 100000, 10000)
+  
+  // Generates executionStores using the given above sizes
+  // Each execution store contains X simulated shards of "prepareCall" in status Done and X simulated shards of "scatterCall" in status NotStarted
+  // This provides a good starting point to evaluate the speed of "runnableCalls", as it needs to iterate over all "NotStarted" keys, and for each one
+  // look for their upstreams keys in status "Done"
+  val executionStores: Gen[ExecutionStore] = for {
+    size <- sizes
+    doneMap = (0 until size map makeKey(prepareCall, ExecutionStatus.Done)).toMap
+    collectorKey = Map(CollectorKey(scatterCall, scatter, size) -> ExecutionStatus.NotStarted)
+    notStartedMap = (0 until size map makeKey(scatterCall, ExecutionStatus.NotStarted)).toMap ++ collectorKey
+    finalMap: Map[JobKey, ExecutionStatus] = doneMap ++ notStartedMap
+  } yield new ExecutionStore(finalMap, true)
+  
+  performance of "ExecutionStore" in {
+    // Measures how fast the execution store can find runnable calls with lots of "Done" calls and "NotStarted" calls.
+    // Other "shapes" would be valuable to get a better sense of how this method behaves in various situations (with Collector Keys etc...)  
+    measure method "runnableCalls" in {
+      using(executionStores) in { es =>
+        es.runnableScopes
+      }
+    }
+  }
+}
diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/SubWorkflowExecutionActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/SubWorkflowExecutionActorSpec.scala
index 6cae26763..f3466a7da 100644
--- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/SubWorkflowExecutionActorSpec.scala
+++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/SubWorkflowExecutionActorSpec.scala
@@ -19,7 +19,7 @@ import cromwell.subworkflowstore.SubWorkflowStoreActor.{QuerySubWorkflow, SubWor
 import org.scalatest.concurrent.Eventually
 import org.scalatest.{FlatSpecLike, Matchers}
 import org.specs2.mock.Mockito
-import wdl4s.{WdlNamespaceWithWorkflow, Workflow, WorkflowCall}
+import wdl4s.wdl._
 
 import scala.concurrent.duration._
 import scala.language.postfixOps
@@ -32,7 +32,9 @@ class SubWorkflowExecutionActorSpec extends TestKitSuite with FlatSpecLike with
   val jobStoreProbe = TestProbe()
   val subWorkflowStoreProbe = TestProbe()
   val callCacheReadActorProbe = TestProbe()
+  val callCacheWriteActorProbe = TestProbe()
   val dockerHashActorProbe = TestProbe()
+  val ioActorProbe = TestProbe()
   val jobTokenDispenserProbe = TestProbe()
   val preparationActor = TestProbe()
   val subWorkflowActor = TestProbe()
@@ -49,9 +51,9 @@ class SubWorkflowExecutionActorSpec extends TestKitSuite with FlatSpecLike with
     List.empty,
     CallCachingOff
   )
-  val subWorkflow = mock[Workflow]
+  val subWorkflow = mock[WdlWorkflow]
   subWorkflow.unqualifiedName returns "sub_wf"
-  val subWorkflowCall = mock[WorkflowCall]
+  val subWorkflowCall = mock[WdlWorkflowCall]
   subWorkflowCall.fullyQualifiedName returns "foo.bar"
   subWorkflowCall.callable returns subWorkflow
   val subKey = SubWorkflowKey(subWorkflowCall, None, 1)
@@ -64,10 +66,12 @@ class SubWorkflowExecutionActorSpec extends TestKitSuite with FlatSpecLike with
         subKey,
         WorkflowExecutionActorData.empty(parentWorkflowDescriptor),
         Map.empty,
+        ioActorProbe.ref,
         serviceRegistryProbe.ref,
         jobStoreProbe.ref,
         subWorkflowStoreProbe.ref,
         callCacheReadActorProbe.ref,
+        callCacheWriteActorProbe.ref,
         dockerHashActorProbe.ref,
         jobTokenDispenserProbe.ref,
         BackendSingletonCollection(Map.empty),
diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorSpec.scala
index f3106bc21..c60b31dae 100644
--- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorSpec.scala
+++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActorSpec.scala
@@ -3,22 +3,22 @@ package cromwell.engine.workflow.lifecycle.execution
 import akka.actor.{Actor, Props}
 import akka.testkit.{EventFilter, TestActorRef, TestDuration, TestProbe}
 import com.typesafe.config.ConfigFactory
+import cromwell._
 import cromwell.backend.AllBackendInitializationData
-import cromwell.core.WorkflowId
+import cromwell.core.{SimpleIoActor, WorkflowId}
 import cromwell.engine.backend.{BackendConfigurationEntry, BackendSingletonCollection, CromwellBackends}
 import cromwell.engine.workflow.WorkflowDescriptorBuilder
-import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor.ExecuteWorkflowCommand
+import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor.{ExecuteWorkflowCommand, WorkflowExecutionFailedResponse}
 import cromwell.engine.workflow.tokens.JobExecutionTokenDispenserActor
 import cromwell.services.ServiceRegistryActor
 import cromwell.services.metadata.MetadataService
 import cromwell.util.SampleWdl
-import cromwell._
-import org.scalatest.BeforeAndAfter
+import org.scalatest.{BeforeAndAfter, FlatSpecLike, Matchers}
 
 import scala.concurrent.duration._
 import scala.concurrent.{Await, Promise}
 
-class WorkflowExecutionActorSpec extends CromwellTestKitSpec with BeforeAndAfter with WorkflowDescriptorBuilder {
+class WorkflowExecutionActorSpec extends CromwellTestKitSpec with FlatSpecLike with Matchers with BeforeAndAfter with WorkflowDescriptorBuilder {
 
   override implicit val actorSystem = system
   implicit val DefaultDuration = 20.seconds.dilated
@@ -41,82 +41,97 @@ class WorkflowExecutionActorSpec extends CromwellTestKitSpec with BeforeAndAfter
       |}
     """.stripMargin
 
-  "WorkflowExecutionActor" should {
-    "retry a job 2 times and succeed in the third attempt" in {
-      import MetadataWatchActor.metadataKeyAttemptChecker
-      val metadataSuccessPromise = Promise[Unit]()
-      val requiredMetadataMatchers: Seq[MetadataWatchActor.Matcher] = List(
-        MetadataWatchActor.JobKeyMetadataKeyAndValueContainStringMatcher(metadataKeyAttemptChecker(1), "executionStatus", "Preempted"),
-        MetadataWatchActor.JobKeyMetadataKeyAndValueContainStringMatcher(metadataKeyAttemptChecker(2), "executionStatus", "Preempted"),
-        MetadataWatchActor.JobKeyMetadataKeyAndValueContainStringMatcher(metadataKeyAttemptChecker(3), "executionStatus", "Done")
-      )
-      val metadataWatcherProps = Props(MetadataWatchActor(metadataSuccessPromise, requiredMetadataMatchers: _*))
-      val serviceRegistryActor = system.actorOf(ServiceRegistryActor.props(ConfigFactory.load(), overrides = Map(MetadataService.MetadataServiceName -> metadataWatcherProps)))
-      val jobStoreActor = system.actorOf(AlwaysHappyJobStoreActor.props)
-      val subWorkflowStoreActor = system.actorOf(AlwaysHappySubWorkflowStoreActor.props)
-      val jobTokenDispenserActor = system.actorOf(JobExecutionTokenDispenserActor.props)
-      val MockBackendConfigEntry = BackendConfigurationEntry(
-        name = "Mock",
-        lifecycleActorFactoryClass = "cromwell.engine.backend.mock.RetryableBackendLifecycleActorFactory",
-        stubbedConfig
-      )
-      CromwellBackends.initBackends(List(MockBackendConfigEntry))
-
-      val workflowId = WorkflowId.randomId()
-      val engineWorkflowDescriptor = createMaterializedEngineWorkflowDescriptor(workflowId, SampleWdl.HelloWorld.asWorkflowSources(runtime = runtimeSection))
-      val callCacheReadActor = TestProbe()
-      val dockerHashActor = TestProbe()
-
-      val workflowExecutionActor = system.actorOf(
-        WorkflowExecutionActor.props(engineWorkflowDescriptor, serviceRegistryActor, jobStoreActor, subWorkflowStoreActor,
-          callCacheReadActor.ref, dockerHashActor.ref, jobTokenDispenserActor, MockBackendSingletonCollection, AllBackendInitializationData.empty, restarting = false),
-        "WorkflowExecutionActor")
-
-      EventFilter.info(pattern = ".*Final Outputs", occurrences = 1).intercept {
-        EventFilter.info(pattern = "Starting calls: wf_hello.hello", occurrences = 3).intercept {
-          workflowExecutionActor ! ExecuteWorkflowCommand
+  behavior of "WorkflowExecutionActor"
+
+  it should "allow a backend to tell it to retry... up to a point" in {
+    import MetadataWatchActor.metadataKeyAttemptChecker
+    val metadataSuccessPromise = Promise[Unit]()
+    val requiredMetadataMatchers: Seq[MetadataWatchActor.Matcher] = List(
+      MetadataWatchActor.JobKeyMetadataKeyAndValueContainStringMatcher(metadataKeyAttemptChecker(1), "executionStatus", "RetryableFailure"),
+      MetadataWatchActor.JobKeyMetadataKeyAndValueContainStringMatcher(metadataKeyAttemptChecker(2), "executionStatus", "RetryableFailure"),
+      MetadataWatchActor.JobKeyMetadataKeyAndValueContainStringMatcher(metadataKeyAttemptChecker(3), "executionStatus", "Failed")
+    )
+    val metadataWatcherActor = TestActorRef[MetadataWatchActor](Props(MetadataWatchActor(metadataSuccessPromise, requiredMetadataMatchers: _*)))
+    val serviceRegistryActor = system.actorOf(ServiceRegistryActor.props(ConfigFactory.load(), overrides = Map(MetadataService.MetadataServiceName -> metadataWatcherActor.props)))
+    val jobStoreActor = system.actorOf(AlwaysHappyJobStoreActor.props)
+    val ioActor = system.actorOf(SimpleIoActor.props)
+    val subWorkflowStoreActor = system.actorOf(AlwaysHappySubWorkflowStoreActor.props)
+    val jobTokenDispenserActor = system.actorOf(JobExecutionTokenDispenserActor.props)
+    val MockBackendConfigEntry = BackendConfigurationEntry(
+      name = "Mock",
+      lifecycleActorFactoryClass = "cromwell.engine.backend.mock.RetryableBackendLifecycleActorFactory",
+      stubbedConfig
+    )
+    CromwellBackends.initBackends(List(MockBackendConfigEntry))
+
+    val workflowId = WorkflowId.randomId()
+    val engineWorkflowDescriptor = createMaterializedEngineWorkflowDescriptor(workflowId, SampleWdl.HelloWorld.asWorkflowSources(runtime = runtimeSection))
+    val callCacheReadActor = TestProbe()
+    val callCacheWriteActor = TestProbe()
+    val dockerHashActor = TestProbe()
+
+    val weaSupervisor = TestProbe()
+    val workflowExecutionActor = TestActorRef(
+      props = WorkflowExecutionActor.props(engineWorkflowDescriptor, ioActor, serviceRegistryActor, jobStoreActor, subWorkflowStoreActor,
+        callCacheReadActor.ref, callCacheWriteActor.ref, dockerHashActor.ref, jobTokenDispenserActor, MockBackendSingletonCollection, AllBackendInitializationData.empty, restarting = false),
+      name = "WorkflowExecutionActor",
+      supervisor = weaSupervisor.ref)
+
+    EventFilter.info(pattern = "Starting calls: wf_hello.hello", occurrences = 3).intercept {
+      workflowExecutionActor ! ExecuteWorkflowCommand
+    }
+
+    weaSupervisor.expectMsgClass(classOf[WorkflowExecutionFailedResponse])
+
+    // Super-helpful debug in case the metadata watcher is still unhappy:
+    if(metadataWatcherActor.underlyingActor.unsatisfiedMatchers.nonEmpty) {
+      requiredMetadataMatchers foreach { matcher =>
+        matcher.nearMissInformation.foreach { info =>
+          System.out.println("A matcher had a near miss (it might still get a matching value later!): " + info.replace("\n", "..."))
         }
       }
+    }
 
-      // TODO: Yes, this might be slow... I'd advocate for refactoring away from the run-a-wdl style, but (shrug)
-      // (but in fact, this never really takes 2 minutes. That's just for safety)
-      Await.result(awaitable = metadataSuccessPromise.future, atMost = 2.minutes.dilated)
+    // TODO: Yes, this might be slow... I'd advocate for refactoring away from the run-a-wdl style, but (shrug)
+    // (but in fact, this never really takes 2 minutes. That's just for safety)
+    Await.result(awaitable = metadataSuccessPromise.future, atMost = 2.minutes.dilated)
 
-      system.stop(serviceRegistryActor)
-    }
+    system.stop(serviceRegistryActor)
+  }
 
-    "execute a workflow with scatters" in {
-      val serviceRegistry = mockServiceRegistryActor
-      val jobStore = system.actorOf(AlwaysHappyJobStoreActor.props)
-      val subWorkflowStoreActor = system.actorOf(AlwaysHappySubWorkflowStoreActor.props)
-      val callCacheReadActor = system.actorOf(EmptyCallCacheReadActor.props)
-      val dockerHashActor = system.actorOf(EmptyDockerHashActor.props)
-      val jobTokenDispenserActor = system.actorOf(JobExecutionTokenDispenserActor.props)
-
-      val MockBackendConfigEntry = BackendConfigurationEntry(
-        name = MockBackendName,
-        lifecycleActorFactoryClass = "cromwell.engine.backend.mock.DefaultBackendLifecycleActorFactory",
-        stubbedConfig
-      )
-      CromwellBackends.initBackends(List(MockBackendConfigEntry))
-
-      val workflowId = WorkflowId.randomId()
-      val engineWorkflowDescriptor = createMaterializedEngineWorkflowDescriptor(workflowId, SampleWdl.SimpleScatterWdl.asWorkflowSources(runtime = runtimeSection))
-      val workflowExecutionActor = system.actorOf(
-        WorkflowExecutionActor.props(engineWorkflowDescriptor, serviceRegistry, jobStore, subWorkflowStoreActor,
-          callCacheReadActor, dockerHashActor, jobTokenDispenserActor, MockBackendSingletonCollection, AllBackendInitializationData.empty, restarting = false),
-        "WorkflowExecutionActor")
-
-      val scatterLog = "Starting calls: scatter0.inside_scatter:0:1, scatter0.inside_scatter:1:1, scatter0.inside_scatter:2:1, scatter0.inside_scatter:3:1, scatter0.inside_scatter:4:1"
-
-      EventFilter.info(pattern = ".*Final Outputs", occurrences = 1).intercept {
-        EventFilter.info(pattern = scatterLog, occurrences = 1).intercept {
-          EventFilter.info(pattern = "Starting calls: scatter0.outside_scatter:NA:1", occurrences = 1).intercept {
-            workflowExecutionActor ! ExecuteWorkflowCommand
-          }
+  it should "execute a workflow with scatters" in {
+    val serviceRegistry = mockServiceRegistryActor
+    val jobStore = system.actorOf(AlwaysHappyJobStoreActor.props)
+    val subWorkflowStoreActor = system.actorOf(AlwaysHappySubWorkflowStoreActor.props)
+    val callCacheReadActor = system.actorOf(EmptyCallCacheReadActor.props)
+    val callCacheWriteActor = system.actorOf(EmptyCallCacheWriteActor.props)
+    val dockerHashActor = system.actorOf(EmptyDockerHashActor.props)
+    val ioActor = system.actorOf(SimpleIoActor.props)
+    val jobTokenDispenserActor = system.actorOf(JobExecutionTokenDispenserActor.props)
+
+    val MockBackendConfigEntry = BackendConfigurationEntry(
+      name = MockBackendName,
+      lifecycleActorFactoryClass = "cromwell.engine.backend.mock.DefaultBackendLifecycleActorFactory",
+      stubbedConfig
+    )
+    CromwellBackends.initBackends(List(MockBackendConfigEntry))
+
+    val workflowId = WorkflowId.randomId()
+    val engineWorkflowDescriptor = createMaterializedEngineWorkflowDescriptor(workflowId, SampleWdl.SimpleScatterWdl.asWorkflowSources(runtime = runtimeSection))
+    val workflowExecutionActor = system.actorOf(
+      WorkflowExecutionActor.props(engineWorkflowDescriptor, ioActor, serviceRegistry, jobStore, subWorkflowStoreActor,
+        callCacheReadActor, callCacheWriteActor, dockerHashActor, jobTokenDispenserActor, MockBackendSingletonCollection, AllBackendInitializationData.empty, restarting = false),
+      "WorkflowExecutionActor")
+
+    val scatterLog = "Starting calls: scatter0.inside_scatter:0:1, scatter0.inside_scatter:1:1, scatter0.inside_scatter:2:1, scatter0.inside_scatter:3:1, scatter0.inside_scatter:4:1"
+
+    EventFilter.info(pattern = ".*Final Outputs", occurrences = 1).intercept {
+      EventFilter.info(pattern = scatterLog, occurrences = 1).intercept {
+        EventFilter.info(pattern = "Starting calls: scatter0.outside_scatter:NA:1", occurrences = 1).intercept {
+          workflowExecutionActor ! ExecuteWorkflowCommand
         }
       }
-      system.stop(serviceRegistry)
     }
+    system.stop(serviceRegistry)
   }
 }
diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheDiffActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheDiffActorSpec.scala
new file mode 100644
index 000000000..801e5bcad
--- /dev/null
+++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheDiffActorSpec.scala
@@ -0,0 +1,267 @@
+package cromwell.engine.workflow.lifecycle.execution.callcaching
+
+import akka.testkit.{ImplicitSender, TestFSMRef, TestProbe}
+import cats.data.NonEmptyList
+import cromwell.core.{TestKitSuite, WorkflowId}
+import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheDiffActor._
+import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheDiffQueryParameter.CallCacheDiffQueryCall
+import cromwell.services.metadata.MetadataService.{GetMetadataQueryAction, MetadataLookupResponse, MetadataServiceKeyLookupFailed}
+import cromwell.services.metadata._
+import org.scalatest.concurrent.Eventually
+import org.scalatest.{FlatSpecLike, Matchers}
+
+class CallCacheDiffActorSpec extends TestKitSuite with FlatSpecLike with Matchers with ImplicitSender with Eventually {
+
+  behavior of "CallCacheDiffActor"
+
+  val workflowIdA = WorkflowId.fromString("971652a6-139c-4ef3-96b5-aeb611a40dbf")
+  val workflowIdB = WorkflowId.fromString("bb85b3ec-e179-4f12-b90f-5191216da598")
+
+  val callFqnA = "callFqnA"
+  val callFqnB = "callFqnB"
+  
+  val metadataJobKeyA = Option(MetadataJobKey(callFqnA, Option(1), 1))
+  val metadataJobKeyB = Option(MetadataJobKey(callFqnB, None, 1))
+  
+  val callA = CallCacheDiffQueryCall(workflowIdA.toString, callFqnA, Option(1))
+  val callB = CallCacheDiffQueryCall(workflowIdB.toString, callFqnB, None)
+
+  val queryA = MetadataQuery(
+    workflowIdA,
+    Option(MetadataQueryJobKey(callFqnA, Option(1), None)),
+    None,
+    Option(NonEmptyList.of("callCaching", "executionStatus")),
+    None,
+    expandSubWorkflows = false
+  )
+
+  val queryB = MetadataQuery(
+    workflowIdB,
+    Option(MetadataQueryJobKey(callFqnB, None, None)),
+    None,
+    Option(NonEmptyList.of("callCaching", "executionStatus")),
+    None,
+    expandSubWorkflows = false
+  )
+  
+  val eventsA = List(
+      MetadataEvent(MetadataKey(workflowIdA, metadataJobKeyA, "executionStatus"), MetadataValue("Done")),
+      MetadataEvent(MetadataKey(workflowIdA, metadataJobKeyA, "callCaching:allowResultReuse"), MetadataValue(true)),
+      MetadataEvent(MetadataKey(workflowIdA, metadataJobKeyA, "callCaching:hashes: hash in only in A"), MetadataValue("hello")),
+      MetadataEvent(MetadataKey(workflowIdA, metadataJobKeyA, "callCaching:hashes: hash in A and B with same value"), MetadataValue(1)),
+      MetadataEvent(MetadataKey(workflowIdA, metadataJobKeyA, "callCaching:hashes: hash in A and B with different value"), MetadataValue("I'm the hash for A !"))
+  )
+
+  val eventsB = List(
+    MetadataEvent(MetadataKey(workflowIdB, metadataJobKeyB, "executionStatus"), MetadataValue("Failed")),
+    MetadataEvent(MetadataKey(workflowIdB, metadataJobKeyB, "callCaching:allowResultReuse"), MetadataValue(false)),
+    MetadataEvent(MetadataKey(workflowIdB, metadataJobKeyB, "callCaching:hashes: hash in only in B"), MetadataValue("hello")),
+    MetadataEvent(MetadataKey(workflowIdB, metadataJobKeyB, "callCaching:hashes: hash in A and B with same value"), MetadataValue(1)),
+    MetadataEvent(MetadataKey(workflowIdA, metadataJobKeyA, "callCaching:hashes: hash in A and B with different value"), MetadataValue("I'm the hash for B !"))
+  )
+  
+  it should "send correct queries to MetadataService when receiving a CallCacheDiffRequest" in {
+    val mockServiceRegistryActor = TestProbe()
+    val actor = TestFSMRef(new CallCacheDiffActor(mockServiceRegistryActor.ref))
+
+    actor ! CallCacheDiffQueryParameter(callA, callB)
+
+    mockServiceRegistryActor.expectMsg(GetMetadataQueryAction(queryA))
+    mockServiceRegistryActor.expectMsg(GetMetadataQueryAction(queryB))
+  }
+
+  it should "save response for callA and wait for callB" in {
+    val mockServiceRegistryActor = TestProbe()
+    val actor = TestFSMRef(new CallCacheDiffActor(mockServiceRegistryActor.ref))
+    
+    actor.setState(WaitingForMetadata, CallCacheDiffWithRequest(queryA, queryB, None, None, self))
+
+    val response = MetadataLookupResponse(queryA, eventsA)
+    actor ! response
+   
+    eventually {
+      actor.stateData shouldBe CallCacheDiffWithRequest(queryA, queryB, Some(response), None, self)
+      actor.stateName shouldBe WaitingForMetadata
+    }
+  }
+
+  it should "save response for callB and wait for callA" in {
+    val mockServiceRegistryActor = TestProbe()
+    val actor = TestFSMRef(new CallCacheDiffActor(mockServiceRegistryActor.ref))
+
+    actor.setState(WaitingForMetadata, CallCacheDiffWithRequest(queryA, queryB, None, None, self))
+
+    val response = MetadataLookupResponse(queryB, eventsB)
+    actor ! response
+
+    eventually {
+      actor.stateData shouldBe CallCacheDiffWithRequest(queryA, queryB, None, Some(response), self)
+      actor.stateName shouldBe WaitingForMetadata
+    }
+  }
+
+  it should "build the response when receiving response for A and already has B" in {
+    val mockServiceRegistryActor = TestProbe()
+    val actor = TestFSMRef(new CallCacheDiffActor(mockServiceRegistryActor.ref))
+    watch(actor)
+    val responseB = MetadataLookupResponse(queryB, eventsB)
+
+    actor.setState(WaitingForMetadata, CallCacheDiffWithRequest(queryA, queryB, None, Option(responseB), self))
+
+    actor ! MetadataLookupResponse(queryA, eventsA)
+
+    expectMsgClass(classOf[CallCacheDiffActorResponse])
+    expectTerminated(actor)
+  }
+
+  it should "build the response when receiving response for B and already has A" in {
+    val mockServiceRegistryActor = TestProbe()
+    val actor = TestFSMRef(new CallCacheDiffActor(mockServiceRegistryActor.ref))
+    watch(actor)
+    val responseA = MetadataLookupResponse(queryA, eventsA)
+
+    actor.setState(WaitingForMetadata, CallCacheDiffWithRequest(queryA, queryB, Option(responseA), None, self))
+
+    actor ! MetadataLookupResponse(queryB, eventsB)
+
+    expectMsgClass(classOf[CallCacheDiffActorResponse])
+    expectTerminated(actor)
+  }
+
+  it should "build a correct response" in {
+    import cromwell.services.metadata.MetadataService.MetadataLookupResponse
+    import spray.json._
+    
+    val mockServiceRegistryActor = TestProbe()
+    val actor = TestFSMRef(new CallCacheDiffActor(mockServiceRegistryActor.ref))
+    watch(actor)
+    actor.setState(WaitingForMetadata, CallCacheDiffWithRequest(queryA, queryB, None, None, self))
+
+    actor ! MetadataLookupResponse(queryB, eventsB)
+    actor ! MetadataLookupResponse(queryA, eventsA)
+
+    val expectedJson: JsObject =
+      s"""
+         |{  
+         |   "callA":{  
+         |      "executionStatus": "Done",
+         |      "allowResultReuse": true,
+         |      "callFqn": "callFqnA",
+         |      "jobIndex": 1,
+         |      "workflowId": "971652a6-139c-4ef3-96b5-aeb611a40dbf"
+         |   },
+         |   "callB":{  
+         |      "executionStatus": "Failed",
+         |      "allowResultReuse": false,
+         |      "callFqn": "callFqnB",
+         |      "jobIndex": -1,
+         |      "workflowId": "bb85b3ec-e179-4f12-b90f-5191216da598"
+         |   },
+         |   "hashDifferential":[  
+         |      {  
+         |         "hashKey": "hash in only in A",
+         |         "callA":"hello",
+         |         "callB":null
+         |      },
+         |      {  
+         |         "hashKey": "hash in A and B with different value",
+         |         "callA":"I'm the hash for A !",
+         |         "callB":"I'm the hash for B !"
+         |      },
+         |      {  
+         |         "hashKey": "hash in only in B",
+         |         "callA":null,
+         |         "callB":"hello"
+         |      }
+         |   ]
+         |}
+       """.stripMargin.parseJson.asJsObject
+    
+    val expectedResponse = BuiltCallCacheDiffResponse(expectedJson)
+    
+    expectMsg(expectedResponse)
+    expectTerminated(actor)
+  }
+
+  it should "fail properly" in {
+    import scala.concurrent.duration._
+    import scala.language.postfixOps
+
+    val mockServiceRegistryActor = TestProbe()
+    val actor = TestFSMRef(new CallCacheDiffActor(mockServiceRegistryActor.ref))
+    watch(actor)
+    val exception = new Exception("Query lookup failed - but it's ok ! this is a test !")
+    val responseA = MetadataServiceKeyLookupFailed(queryA, exception)
+
+    actor.setState(WaitingForMetadata, CallCacheDiffWithRequest(queryA, queryB, None, None, self))
+
+    actor ! responseA
+
+    expectMsgPF(1 second) {
+      case FailedCallCacheDiffResponse(e: Throwable) =>
+        e.getMessage shouldBe "Query lookup failed - but it's ok ! this is a test !"
+    }
+    
+    expectTerminated(actor)
+  }
+
+  it should "Respond with an appropriate message if hashes are missing" in {
+    import scala.concurrent.duration._
+    import scala.language.postfixOps
+
+    val mockServiceRegistryActor = TestProbe()
+    val actor = TestFSMRef(new CallCacheDiffActor(mockServiceRegistryActor.ref))
+    watch(actor)
+    val responseB = MetadataLookupResponse(queryB, eventsB.filterNot(_.key.key.contains("hashes")))
+
+    actor.setState(WaitingForMetadata, CallCacheDiffWithRequest(queryA, queryB, None, Option(responseB), self))
+
+    actor ! MetadataLookupResponse(queryA, eventsA.filterNot(_.key.key.contains("hashes")))
+
+    expectMsgPF(1 second) {
+      case FailedCallCacheDiffResponse(e) =>
+         e.getMessage shouldBe "callA and callB have not finished yet, or were run on a previous version of Cromwell on which this endpoint was not supported."
+    }
+    expectTerminated(actor)
+  }
+
+  it should "Respond with CachedCallNotFoundException if a call is missing" in {
+    import scala.concurrent.duration._
+    import scala.language.postfixOps
+
+    val mockServiceRegistryActor = TestProbe()
+    val actor = TestFSMRef(new CallCacheDiffActor(mockServiceRegistryActor.ref))
+    watch(actor)
+    val responseB = MetadataLookupResponse(queryB, eventsB.filterNot(_.key.key.contains("hashes")))
+
+    actor.setState(WaitingForMetadata, CallCacheDiffWithRequest(queryA, queryB, None, Option(responseB), self))
+
+    actor ! MetadataLookupResponse(queryA, List.empty)
+
+    expectMsgPF(1 second) {
+      case FailedCallCacheDiffResponse(e) =>
+        e.getMessage shouldBe "Cannot find call 971652a6-139c-4ef3-96b5-aeb611a40dbf:callFqnA:1"
+    }
+    expectTerminated(actor)
+  }
+
+  it should "Respond with CachedCallNotFoundException if both calls are missing" in {
+    import scala.concurrent.duration._
+    import scala.language.postfixOps
+
+    val mockServiceRegistryActor = TestProbe()
+    val actor = TestFSMRef(new CallCacheDiffActor(mockServiceRegistryActor.ref))
+    watch(actor)
+    val responseB = MetadataLookupResponse(queryB, List.empty)
+
+    actor.setState(WaitingForMetadata, CallCacheDiffWithRequest(queryA, queryB, None, Option(responseB), self))
+
+    actor ! MetadataLookupResponse(queryA, List.empty)
+
+    expectMsgPF(1 second) {
+      case FailedCallCacheDiffResponse(e) =>
+        e.getMessage shouldBe "Cannot find calls 971652a6-139c-4ef3-96b5-aeb611a40dbf:callFqnA:1, bb85b3ec-e179-4f12-b90f-5191216da598:callFqnB:-1"
+    }
+    expectTerminated(actor)
+  }
+}
diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheHashingJobActorDataSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheHashingJobActorDataSpec.scala
new file mode 100644
index 000000000..1f1c874f1
--- /dev/null
+++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheHashingJobActorDataSpec.scala
@@ -0,0 +1,92 @@
+package cromwell.engine.workflow.lifecycle.execution.callcaching
+
+import cats.data.NonEmptyList
+import cromwell.backend._
+import cromwell.backend.standard.callcaching.StandardFileHashingActor.SingleFileHashRequest
+import cromwell.core.TestKitSuite
+import cromwell.core.callcaching._
+import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheHashingJobActor.{CallCacheHashingJobActorData, CompleteFileHashingResult, NoFileHashesResult, PartialFileHashingResult}
+import org.scalatest.concurrent.Eventually
+import org.scalatest.prop.TableDrivenPropertyChecks
+import org.scalatest.{FlatSpecLike, Matchers}
+
+class CallCacheHashingJobActorDataSpec extends TestKitSuite with FlatSpecLike with BackendSpec with Matchers with Eventually with TableDrivenPropertyChecks {
+  behavior of "CallCacheReadingJobActorData"
+
+  val fileHash1 = HashResult(HashKey("key"), HashValue("value"))
+  val fileHash2 = HashResult(HashKey("key2"), HashValue("value2"))
+  val fileHash3 = HashResult(HashKey("key3"), HashValue("value3"))
+  val fileHashRequest1 = SingleFileHashRequest(null, fileHash1.hashKey, null, null)
+  val fileHashRequest2 = SingleFileHashRequest(null, fileHash2.hashKey, null, null)
+  val fileHashRequest3 = SingleFileHashRequest(null, fileHash3.hashKey, null, null)
+  
+  val testCases = Table(
+    ("dataBefore", "dataAfter", "result"),
+    // No fileHashRequestsRemaining
+    (
+      CallCacheHashingJobActorData(
+        List.empty, List.empty, None
+      ),
+      CallCacheHashingJobActorData(
+        List.empty, List(fileHash1), None
+      ),
+      Option(NoFileHashesResult)
+    ),
+    // Last fileHashRequestsRemaining
+    (
+      CallCacheHashingJobActorData(
+        List(List(fileHashRequest1)), List.empty, None
+      ),
+      CallCacheHashingJobActorData(
+        List.empty, List(fileHash1), None
+      ),
+      Option(CompleteFileHashingResult(Set(fileHash1), "6A02F950958AEDA3DBBF83FBB306A030"))
+    ),
+    // Last batch and not last value
+    (
+      CallCacheHashingJobActorData(
+        List(List(fileHashRequest1, fileHashRequest2)), List.empty, None
+      ),
+      CallCacheHashingJobActorData(
+        List(List(fileHashRequest2)), List(fileHash1), None
+      ),
+      None
+    ),
+    // Not last batch but last value of this batch
+    (
+      CallCacheHashingJobActorData(
+        List(List(fileHashRequest1), List(fileHashRequest2)), List.empty, None
+      ),
+      CallCacheHashingJobActorData(
+        List(List(fileHashRequest2)), List(fileHash1), None
+      ),
+      Option(PartialFileHashingResult(NonEmptyList.of(fileHash1)))
+    ),
+    // Not last batch and not last value of this batch
+    (
+      CallCacheHashingJobActorData(
+        List(List(fileHashRequest1, fileHashRequest2), List(fileHashRequest3)), List.empty, None
+      ),
+      CallCacheHashingJobActorData(
+        List(List(fileHashRequest2), List(fileHashRequest3)), List(fileHash1), None
+      ),
+      None
+    ),
+    // Makes sure new hash is added at the front of the list
+    (
+      CallCacheHashingJobActorData(
+        List(List(fileHashRequest1, fileHashRequest2), List(fileHashRequest3)), List(fileHash2), None
+      ),
+      CallCacheHashingJobActorData(
+        List(List(fileHashRequest2), List(fileHashRequest3)), List(fileHash1, fileHash2), None
+      ),
+      None
+    )
+  )
+  
+  it should "process new file hashes" in {
+    forAll(testCases) { case ((oldData, newData, result)) =>
+      oldData.withFileHash(fileHash1) shouldBe (newData -> result)
+    }
+  }
+}
diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheHashingJobActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheHashingJobActorSpec.scala
new file mode 100644
index 000000000..a3ea45458
--- /dev/null
+++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheHashingJobActorSpec.scala
@@ -0,0 +1,313 @@
+package cromwell.engine.workflow.lifecycle.execution.callcaching
+
+import akka.actor.{ActorRef, Props}
+import akka.testkit.{TestFSMRef, TestProbe}
+import cats.data.NonEmptyList
+import cromwell.backend._
+import cromwell.backend.standard.callcaching.StandardFileHashingActor.{FileHashResponse, SingleFileHashRequest}
+import cromwell.core.callcaching.{HashingFailedMessage, _}
+import cromwell.core.{LocallyQualifiedName, TestKitSuite}
+import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheHashingJobActor.{CCHJAFileHashResponse, CallCacheHashingJobActorData, CompleteFileHashingResult, HashingFiles, InitialHashingResult, NextBatchOfFileHashesRequest, NoFileHashesResult, PartialFileHashingResult, WaitingForHashFileRequest}
+import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.CacheMiss
+import org.mockito.Mockito._
+import org.scalatest.concurrent.Eventually
+import org.scalatest.prop.TableDrivenPropertyChecks
+import org.scalatest.{FlatSpecLike, Matchers}
+import wdl4s.wdl.values.{WdlFile, WdlInteger, WdlString, WdlValue}
+import wdl4s.wdl.{WdlTask, WdlTaskCall}
+
+class CallCacheHashingJobActorSpec extends TestKitSuite with FlatSpecLike with BackendSpec with Matchers with Eventually with TableDrivenPropertyChecks {
+  behavior of "CallCacheReadingJobActor"
+
+  def templateJobDescriptor(inputs: Map[LocallyQualifiedName, WdlValue] = Map.empty) = {
+    val task = mock[WdlTask]
+    val call = mock[WdlTaskCall]
+    when(task.commandTemplateString).thenReturn("Do the stuff... now!!")
+    when(task.outputs).thenReturn(List.empty)
+    when(call.task).thenReturn(task)
+    val workflowDescriptor = mock[BackendWorkflowDescriptor]
+    val runtimeAttributes = Map(
+      "cpu" -> WdlInteger(1),
+      "memory" -> WdlString("3 GB"),
+      "continueOnReturnCode" -> WdlInteger(0),
+      "docker" -> WdlString("ubuntu:latest")
+    )
+    val jobDescriptor = BackendJobDescriptor(workflowDescriptor, BackendJobDescriptorKey(call, None, 1), runtimeAttributes, fqnMapToDeclarationMap(inputs), NoDocker, Map.empty)
+    jobDescriptor
+  }
+
+  it should "die immediately if created without cache read actor and write to cache turned off" in {
+    val parent = TestProbe()
+    val testActor = TestFSMRef(new CallCacheHashingJobActor(
+      templateJobDescriptor(),
+      None,
+      None,
+      Set.empty,
+      "backedName",
+      Props.empty,
+      false,
+      DockerWithHash("ubuntu@sha256:blablablba")
+    ), parent.ref)
+    watch(testActor)
+    expectTerminated(testActor)
+    parent.expectMsgClass(classOf[InitialHashingResult])
+    parent.expectMsg(CacheMiss)
+  }
+
+  it should "send a correct InitialHashingResult upon starting" in {
+    val parent = TestProbe()
+    val inputs = Map(
+      "stringInput" -> WdlString("hello"),
+      "fileInput" -> WdlFile("world")
+    )
+    // Do not include "memory" on purpose, even though it's in the map of runtime attributes.
+    // This way we can verify that only attributes with a RuntimeAttributeDefinition are used for hashing
+    // Vice versa include a "failOnStderr" definition even though it's not in the map.
+    // This ensures that we still record the fact that there was no failOnStderr attribute with a "N/A" value
+    val runtimeAttributeDefinitions = Set(
+      RuntimeAttributeDefinition("docker", None, usedInCallCaching = true),
+      RuntimeAttributeDefinition("failOnStderr", None, usedInCallCaching = true),
+      RuntimeAttributeDefinition("continueOnReturnCode", Option(WdlInteger(0)), usedInCallCaching = true),
+      RuntimeAttributeDefinition("cpu", None, usedInCallCaching = false)
+    )
+    val callCacheRead = TestProbe()
+    val jobDescriptor: BackendJobDescriptor = templateJobDescriptor(inputs)
+    val actorUnderTest = TestFSMRef(new CallCacheHashingJobActor(
+      jobDescriptor,
+      Option(callCacheRead.ref),
+      None,
+      runtimeAttributeDefinitions,
+      "backedName",
+      Props.empty,
+      true,
+      DockerWithHash("ubuntu@sha256:blablablba")
+    ), parent.ref)
+    
+    val expectedInitialHashes = Set(
+      // md5 of Do the stuff... now
+      HashResult(HashKey("command template"), HashValue("2259B15D9120F50C1BD4B2A3E2CE5A0E")),
+      // md5 of backendName
+      HashResult(HashKey("backend name"), HashValue("DC3D1A5AB4B8064660ADE07FFDECBFFE")),
+      // md5 of 2
+      HashResult(HashKey("input count"), HashValue("C81E728D9D4C2F636F067F89CC14862C")),
+      // md5 of 0
+      HashResult(HashKey("output count"), HashValue("CFCD208495D565EF66E7DFF9F98764DA")),
+      HashResult(HashKey("runtime attribute", "failOnStderr"), HashValue("N/A")),
+      // md5 of 1
+      HashResult(HashKey(checkForHitOrMiss = false, "runtime attribute", "cpu"), HashValue("C4CA4238A0B923820DCC509A6F75849B")),
+      // md5 of 0
+      HashResult(HashKey("runtime attribute", "continueOnReturnCode"), HashValue("CFCD208495D565EF66E7DFF9F98764DA")),
+      // md5 of "hello" (with quotes)
+      HashResult(HashKey("input", "String stringInput"), HashValue("5DEAEE1C1332199E5B5BC7C5E4F7F0C2")),
+      // md5 of ubuntu@sha256:blablablba - make sure we use the dockerWithHash and not the docker runtime attribute
+      HashResult(HashKey("runtime attribute", "docker"), HashValue("C811916EA68009B0EFE0A3A86D73280E"))
+    )
+    val expectedAggregatedInitialHash = "F1A7118BED69B5A976A17C83FBA0D9D6"
+    val expectedInitialHashResult = InitialHashingResult(expectedInitialHashes, expectedAggregatedInitialHash)
+    parent.expectMsg(expectedInitialHashResult)
+    callCacheRead.expectMsg(expectedInitialHashResult)
+    actorUnderTest.stateName shouldBe WaitingForHashFileRequest
+    actorUnderTest.stateData shouldBe CallCacheHashingJobActorData(
+      List(SingleFileHashRequest(jobDescriptor.key, HashKey("input", "File fileInput"), WdlFile("world"), None)),
+      Option(callCacheRead.ref)
+    )
+  }
+  
+  def makeCCHJA(callCacheReader: Option[ActorRef],
+                testFileHashingActor: ActorRef,
+                parent: ActorRef,
+                writeToCache: Boolean = true,
+                addFileHashMockResult: Option[(CallCacheHashingJobActorData, Option[CCHJAFileHashResponse])] = None) = {
+    TestFSMRef(new CallCacheHashingJobActor(
+      templateJobDescriptor(),
+      callCacheReader,
+      None,
+      Set.empty,
+      "backend",
+      Props.empty,
+      writeToCache = writeToCache,
+      DockerWithHash("ubuntu@256:blablabla")
+    ) {
+      override def makeFileHashingActor() = testFileHashingActor
+      override def addFileHash(hashResult: HashResult, data: CallCacheHashingJobActorData) = {
+        addFileHashMockResult.getOrElse(super.addFileHash(hashResult, data))
+      }
+    }, parent)
+  }
+  
+  it should "send hash file requests when receiving a NextBatchOfFileHashesRequest" in {
+    val callCacheReadProbe = TestProbe()
+    val fileHashingActor = TestProbe()
+    
+    val cchja = makeCCHJA(Option(callCacheReadProbe.ref), fileHashingActor.ref, TestProbe().ref)
+
+    val fileHashRequest1 = SingleFileHashRequest(null, null, null, null)
+    val fileHashRequest2 = SingleFileHashRequest(null, null, null, null)
+    cchja.setState(
+      WaitingForHashFileRequest,
+      CallCacheHashingJobActorData(List(List(fileHashRequest1, fileHashRequest2)), List.empty, None)
+    )
+
+    cchja ! NextBatchOfFileHashesRequest
+
+    fileHashingActor.expectMsg(fileHashRequest1)
+    fileHashingActor.expectMsg(fileHashRequest2)
+    cchja.stateName shouldBe HashingFiles
+  }
+
+  it should "send NoFileHashesResult and stop if there are no input files" in {
+    val parent = TestProbe()
+    val callCacheReadProbe = TestProbe()
+    
+    val cchja = makeCCHJA(Option(callCacheReadProbe.ref), TestProbe().ref, parent.ref)
+    parent.watch(cchja)
+    
+    cchja.setState(
+      WaitingForHashFileRequest,
+      CallCacheHashingJobActorData(List.empty, List.empty, Option(callCacheReadProbe.ref))
+    )
+
+    cchja ! NextBatchOfFileHashesRequest
+
+    callCacheReadProbe.expectMsgClass(classOf[InitialHashingResult])
+    parent.expectMsgClass(classOf[InitialHashingResult])
+    
+    callCacheReadProbe.expectMsg(NoFileHashesResult)
+    parent.expectMsg(NoFileHashesResult)
+    parent.expectTerminated(cchja)
+  }
+
+  it should "send the PartialFileHashingResult when a batch is complete" in {
+    val callCacheReadProbe = TestProbe()
+    val hashResults = NonEmptyList.of(mock[HashResult])
+    
+    val result: PartialFileHashingResult = PartialFileHashingResult(hashResults)
+    val newData: CallCacheHashingJobActorData = CallCacheHashingJobActorData(List.empty, List.empty, Option(callCacheReadProbe.ref))
+    
+    val cchja = makeCCHJA(Option(callCacheReadProbe.ref), TestProbe().ref, TestProbe().ref, writeToCache = true, Option(newData -> Option(result)))
+
+    cchja.setState(HashingFiles)
+
+    cchja ! FileHashResponse(mock[HashResult])
+
+    callCacheReadProbe.expectMsgClass(classOf[InitialHashingResult])
+    callCacheReadProbe.expectMsg(result)
+    cchja.stateName shouldBe WaitingForHashFileRequest
+    cchja.stateData shouldBe newData
+  }
+
+  it should "send itself a NextBatchOfFileHashesRequest when a batch is complete and there is no CCReader" in {
+    val fileHashingActor = TestProbe()
+    val result: PartialFileHashingResult = PartialFileHashingResult(NonEmptyList.of(mock[HashResult]))
+    val fileHashRequest = SingleFileHashRequest(null, null, null, null)
+    val newData = CallCacheHashingJobActorData(List(List(fileHashRequest)), List.empty, None)
+    // still gives a CCReader when instantiating the actor, but not in the data (above)
+    // This ensures the check is done with the data and not the actor attribute, as the data will change if the ccreader dies but the actor attribute
+    // will stay Some(...)
+    val cchja = makeCCHJA(Option(TestProbe().ref), fileHashingActor.ref, TestProbe().ref, writeToCache = true, Option(newData -> Option(result)))
+    watch(cchja)
+    cchja.setState(HashingFiles)
+
+    cchja ! FileHashResponse(mock[HashResult])
+
+    // This proves that the ccjha keeps hashing files even though there is no ccreader requesting more hashes
+    fileHashingActor.expectMsg(fileHashRequest)
+    cchja.stateName shouldBe HashingFiles
+  }
+
+  it should "send FinalFileHashingResult to parent and CCReader and die" in {
+    val parent = TestProbe()
+    val callCacheReadProbe = TestProbe()
+    List(CompleteFileHashingResult(Set(mock[HashResult]), "AggregatedFileHash"), NoFileHashesResult) foreach { result =>
+      val newData = CallCacheHashingJobActorData(List.empty, List.empty, Option(callCacheReadProbe.ref))
+      val cchja = makeCCHJA(Option(callCacheReadProbe.ref), TestProbe().ref, parent.ref, writeToCache = true, Option(newData -> Option(result)))
+
+      parent.expectMsgClass(classOf[InitialHashingResult])
+      callCacheReadProbe.expectMsgClass(classOf[InitialHashingResult])
+
+      parent.watch(cchja)
+      cchja.setState(HashingFiles)
+
+      cchja ! FileHashResponse(mock[HashResult])
+
+      callCacheReadProbe.expectMsg(result)
+      parent.expectMsg(result)
+      parent.expectTerminated(cchja)
+    }
+  }
+
+  it should "wait for next file hash if the batch is not complete yet" in {
+    val callCacheReadProbe = TestProbe()
+    val parent = TestProbe()
+    val newData: CallCacheHashingJobActorData = CallCacheHashingJobActorData(List.empty, List.empty, Option(callCacheReadProbe.ref))
+    val cchja = makeCCHJA(Option(callCacheReadProbe.ref), TestProbe().ref, parent.ref, writeToCache = true, Option(newData -> None))
+
+    parent.expectMsgClass(classOf[InitialHashingResult])
+    callCacheReadProbe.expectMsgClass(classOf[InitialHashingResult])
+
+    cchja.setState(HashingFiles)
+
+    cchja ! FileHashResponse(mock[HashResult])
+
+    callCacheReadProbe.expectNoMsg()
+    parent.expectNoMsg()
+    cchja.stateName shouldBe HashingFiles
+  }
+
+  it should "stop if the read actor dies and writeToCache is off" in {
+    val callCacheReadProbe = TestProbe()
+    val parent = TestProbe()
+    val cchja = makeCCHJA(Option(callCacheReadProbe.ref), TestProbe().ref, parent.ref, writeToCache = false)
+    parent.watch(cchja)
+    system stop callCacheReadProbe.ref
+    parent.expectMsgClass(classOf[InitialHashingResult])
+    parent.expectTerminated(cchja)
+  }
+
+  it should "keep going if the read actor dies and writeToCache is on" in {
+    val callCacheReadProbe = TestProbe()
+    val parent = TestProbe()
+    val fileHasher = TestProbe()
+    val cchja = makeCCHJA(Option(callCacheReadProbe.ref), fileHasher.ref, parent.ref)
+    parent.expectMsgClass(classOf[InitialHashingResult])
+
+    val hashKey = HashKey("file")
+    val fileHashRequest: SingleFileHashRequest = SingleFileHashRequest(null, hashKey, null, null)
+    val data = CallCacheHashingJobActorData(List(List(fileHashRequest)), List.empty, Option(callCacheReadProbe.ref))
+    
+    cchja.setState(WaitingForHashFileRequest, data)
+    
+    system stop callCacheReadProbe.ref
+    fileHasher.expectMsg(fileHashRequest)
+    val result: HashResult = HashResult(hashKey, HashValue("fileHash"))
+    fileHasher.reply(FileHashResponse(result))
+
+    parent.expectMsg(CompleteFileHashingResult(Set(result), "45F27DD26834DBACBB05BBB1D651F5D1"))
+  }
+  
+  it should "propagate HashingFailedMessage errors and die" in {
+    val callCacheReadProbe = TestProbe()
+    val parent = TestProbe()
+    val cchja = makeCCHJA(Option(callCacheReadProbe.ref), TestProbe().ref, parent.ref)
+    parent.watch(cchja)
+    cchja.setState(WaitingForHashFileRequest)
+    parent.expectMsgClass(classOf[InitialHashingResult])
+    callCacheReadProbe.expectMsgClass(classOf[InitialHashingResult])
+    
+    val hashFailed = HashingFailedMessage("fileName", new Exception("Hashing failed ! - part of test flow"))
+    cchja ! hashFailed
+    parent.expectMsg(hashFailed)
+    callCacheReadProbe.expectMsg(hashFailed)
+    parent.expectTerminated(cchja)
+  }
+  
+  it should "run properly when writeToCache is ON and there is no CCRead actor" in {
+    val parent = TestProbe()
+    val cchja = makeCCHJA(None, TestProbe().ref, parent.ref)
+    parent.watch(cchja)
+    
+    parent.expectMsgClass(classOf[InitialHashingResult])
+    parent.expectMsg(NoFileHashesResult)
+    parent.expectTerminated(cchja)
+  }
+}
diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheReadingJobActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheReadingJobActorSpec.scala
new file mode 100644
index 000000000..195e8a802
--- /dev/null
+++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCacheReadingJobActorSpec.scala
@@ -0,0 +1,219 @@
+package cromwell.engine.workflow.lifecycle.execution.callcaching
+
+import akka.testkit.{TestFSMRef, TestProbe}
+import cats.data.NonEmptyList
+import cromwell.core.TestKitSuite
+import cromwell.core.callcaching.{HashKey, HashResult, HashValue, HashingFailedMessage}
+import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheHashingJobActor.{CompleteFileHashingResult, InitialHashingResult, NextBatchOfFileHashesRequest, NoFileHashesResult, PartialFileHashingResult}
+import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheReadActor._
+import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheReadingJobActor.{CCRJAWithData, WaitingForCacheHitOrMiss, _}
+import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.{CacheHit, CacheMiss, HashError}
+import org.scalatest.concurrent.Eventually
+import org.scalatest.{FlatSpecLike, Matchers}
+
+class CallCacheReadingJobActorSpec extends TestKitSuite with FlatSpecLike with Matchers with Eventually {
+  behavior of "CallCacheReadingJobActor"
+  
+  it should "try to match initial hashes against DB" in {
+    val callCacheReadProbe = TestProbe()
+    val callCacheHasingActor = TestProbe()
+    val actorUnderTest = TestFSMRef(new CallCacheReadingJobActor(callCacheReadProbe.ref))
+    actorUnderTest.stateName shouldBe WaitingForInitialHash
+
+    // The actual hashes don't matter here, we only care about the aggregated hash
+    val aggregatedInitialhash: String = "AggregatedInitialHash"
+    callCacheHasingActor.send(actorUnderTest, InitialHashingResult(Set.empty, aggregatedInitialhash))
+    callCacheReadProbe.expectMsg(HasMatchingInitialHashLookup(aggregatedInitialhash))
+    eventually {
+      actorUnderTest.stateName shouldBe WaitingForHashCheck
+      actorUnderTest.stateData shouldBe CCRJAWithData(callCacheHasingActor.ref, aggregatedInitialhash, None, 1)
+    }
+  }
+
+  it should "ask for file hashes if it found matching entries for initial aggregated hash" in {
+    val callCacheReadProbe = TestProbe()
+    val callCacheHashingActor = TestProbe()
+    val actorUnderTest = TestFSMRef(new CallCacheReadingJobActor(callCacheReadProbe.ref))
+    actorUnderTest.setState(WaitingForHashCheck, CCRJAWithData(callCacheHashingActor.ref, "AggregatedInitialHash", None, 1))
+
+    callCacheReadProbe.send(actorUnderTest, HasMatchingEntries)
+    callCacheHashingActor.expectMsg(NextBatchOfFileHashesRequest)
+    eventually {
+      actorUnderTest.stateName shouldBe WaitingForFileHashes
+    }
+  }
+
+  it should "cache miss and die if it didn't find any matching entries for initial aggregated hash" in {
+    val callCacheReadProbe = TestProbe()
+    val callCacheHashingActor = TestProbe()
+    val parent = TestProbe()
+    
+    val actorUnderTest = TestFSMRef(new CallCacheReadingJobActor(callCacheReadProbe.ref), parent.ref)
+    parent.watch(actorUnderTest)
+    
+    actorUnderTest.setState(WaitingForHashCheck, CCRJAWithData(callCacheHashingActor.ref, "AggregatedInitialHash", None, 1))
+
+    callCacheReadProbe.send(actorUnderTest, NoMatchingEntries)
+    parent.expectMsg(CacheMiss)
+    parent.expectTerminated(actorUnderTest)
+  }
+
+  it should "try to match partial file hashes against DB" in {
+    val callCacheReadProbe = TestProbe()
+    val callCacheHashingActor = TestProbe()
+    
+    val actorUnderTest = TestFSMRef(new CallCacheReadingJobActor(callCacheReadProbe.ref), TestProbe().ref)
+
+    actorUnderTest.setState(WaitingForFileHashes)
+
+    val fileHashes = NonEmptyList.of(HashResult(HashKey("f1"), HashValue("h1")), HashResult(HashKey("f2"), HashValue("h2")))
+    callCacheHashingActor.send(actorUnderTest, PartialFileHashingResult(fileHashes))
+    callCacheReadProbe.expectMsg(HasMatchingInputFilesHashLookup(fileHashes))
+    
+    eventually {
+      actorUnderTest.stateName shouldBe WaitingForHashCheck
+    }
+  }
+
+  it should "ask for matching cache entries for both aggregated hashes when got both" in {
+    val callCacheReadProbe = TestProbe()
+    val callCacheHashingActor = TestProbe()
+    
+    val actorUnderTest = TestFSMRef(new CallCacheReadingJobActor(callCacheReadProbe.ref), TestProbe().ref)
+
+    val aggregatedInitialHash: String = "AggregatedInitialHash"
+    val aggregatedFileHash: String = "AggregatedFileHash"
+    actorUnderTest.setState(WaitingForFileHashes, CCRJAWithData(callCacheHashingActor.ref, aggregatedInitialHash, None, 1))
+
+    val fileHashes = Set(HashResult(HashKey("f1"), HashValue("h1")), HashResult(HashKey("f2"), HashValue("h2")))
+    callCacheHashingActor.send(actorUnderTest, CompleteFileHashingResult(fileHashes, aggregatedFileHash))
+    callCacheReadProbe.expectMsg(CacheLookupRequest(AggregatedCallHashes(aggregatedInitialHash, aggregatedFileHash), 1))
+
+    eventually {
+      actorUnderTest.stateName shouldBe WaitingForCacheHitOrMiss
+      actorUnderTest.stateData shouldBe CCRJAWithData(callCacheHashingActor.ref, aggregatedInitialHash, Some(aggregatedFileHash), 1)
+    }
+  }
+
+  it should "ask for matching cache entries for initial hashes when there is no file input" in {
+    val callCacheReadProbe = TestProbe()
+    val callCacheHashingActor = TestProbe()
+    
+    val actorUnderTest = TestFSMRef(new CallCacheReadingJobActor(callCacheReadProbe.ref), TestProbe().ref)
+
+    val aggregatedInitialHash: String = "AggregatedInitialHash"
+    actorUnderTest.setState(WaitingForFileHashes, CCRJAWithData(callCacheHashingActor.ref, aggregatedInitialHash, None, 1))
+
+    callCacheHashingActor.send(actorUnderTest, NoFileHashesResult)
+    callCacheReadProbe.expectMsg(CacheLookupRequest(AggregatedCallHashes(aggregatedInitialHash, None), 1))
+
+    eventually {
+      actorUnderTest.stateName shouldBe WaitingForCacheHitOrMiss
+      actorUnderTest.stateData shouldBe CCRJAWithData(callCacheHashingActor.ref, aggregatedInitialHash, None, 1)
+    }
+  }
+
+  it should "reply with next hit when cache hit is successful" in {
+    val callCacheReadProbe = TestProbe()
+    val callCacheHashingActor = TestProbe()
+    val parent = TestProbe()
+    
+    val actorUnderTest = TestFSMRef(new CallCacheReadingJobActor(callCacheReadProbe.ref), parent.ref)
+
+    val aggregatedInitialHash: String = "AggregatedInitialHash"
+    actorUnderTest.setState(WaitingForCacheHitOrMiss, CCRJAWithData(callCacheHashingActor.ref, aggregatedInitialHash, None, 1))
+
+    val id: CallCachingEntryId = CallCachingEntryId(8)
+    callCacheReadProbe.send(actorUnderTest, CacheLookupNextHit(id))
+    parent.expectMsg(CacheHit(id))
+
+    eventually {
+      actorUnderTest.stateName shouldBe WaitingForCacheHitOrMiss
+      actorUnderTest.stateData shouldBe CCRJAWithData(callCacheHashingActor.ref, aggregatedInitialHash, None, 2)
+    }
+  }
+
+  it should "reply with cache miss if there's no hit" in {
+    val callCacheReadProbe = TestProbe()
+    val callCacheHashingActor = TestProbe()
+    val parent = TestProbe()
+    
+    val actorUnderTest = TestFSMRef(new CallCacheReadingJobActor(callCacheReadProbe.ref), parent.ref)
+    parent.watch(actorUnderTest)
+    
+    val aggregatedInitialHash: String = "AggregatedInitialHash"
+    actorUnderTest.setState(WaitingForCacheHitOrMiss, CCRJAWithData(callCacheHashingActor.ref, aggregatedInitialHash, None, 1))
+
+    callCacheReadProbe.send(actorUnderTest, CacheLookupNoHit)
+    parent.expectMsg(CacheMiss)
+
+    parent.expectTerminated(actorUnderTest)
+  }
+
+  it should "ask callCacheReadActor for next hit when requested (initial hash only)" in {
+    val callCacheReadProbe = TestProbe()
+    val callCacheHashingActor = TestProbe()
+    
+    val actorUnderTest = TestFSMRef(new CallCacheReadingJobActor(callCacheReadProbe.ref), TestProbe().ref)
+
+    val aggregatedInitialHash: String = "AggregatedInitialHash"
+    actorUnderTest.setState(WaitingForCacheHitOrMiss, CCRJAWithData(callCacheHashingActor.ref, aggregatedInitialHash, None, 2))
+
+    actorUnderTest ! NextHit
+    callCacheReadProbe.expectMsg(CacheLookupRequest(AggregatedCallHashes(aggregatedInitialHash, None), 2))
+
+    actorUnderTest.stateName shouldBe WaitingForCacheHitOrMiss
+  }
+
+  it should "ask callCacheReadActor for next hit when requested (with file hash)" in {
+    val callCacheReadProbe = TestProbe()
+    val callCacheHashingActor = TestProbe()
+    
+    val actorUnderTest = TestFSMRef(new CallCacheReadingJobActor(callCacheReadProbe.ref), TestProbe().ref)
+
+    val aggregatedInitialHash: String = "AggregatedInitialHash"
+    val aggregatedFileHash: String = "AggregatedFileHash"
+    actorUnderTest.setState(WaitingForCacheHitOrMiss, CCRJAWithData(callCacheHashingActor.ref, aggregatedInitialHash, Option(aggregatedFileHash), 2))
+
+    actorUnderTest ! NextHit
+    callCacheReadProbe.expectMsg(CacheLookupRequest(AggregatedCallHashes(aggregatedInitialHash, Option(aggregatedFileHash)), 2))
+
+    actorUnderTest.stateName shouldBe WaitingForCacheHitOrMiss
+  }
+
+  it should "reply with cache miss if there's a hash failure" in {
+    val callCacheReadProbe = TestProbe()
+    val callCacheHashingActor = TestProbe()
+    val parent = TestProbe()
+    
+    val actorUnderTest = TestFSMRef(new CallCacheReadingJobActor(callCacheReadProbe.ref), parent.ref)
+    parent.watch(actorUnderTest)
+
+    val aggregatedInitialHash: String = "AggregatedInitialHash"
+    actorUnderTest.setState(WaitingForCacheHitOrMiss, CCRJAWithData(callCacheHashingActor.ref, aggregatedInitialHash, None, 1))
+
+    callCacheHashingActor.send(actorUnderTest, HashingFailedMessage("file", new Exception("Hashing failed")))
+    parent.expectMsg(CacheMiss)
+
+    parent.expectTerminated(actorUnderTest)
+  }
+
+  it should "reply with cache miss if there's a lookup failure" in {
+    val callCacheReadProbe = TestProbe()
+    val callCacheHashingActor = TestProbe()
+    val parent = TestProbe()
+    
+    val actorUnderTest = TestFSMRef(new CallCacheReadingJobActor(callCacheReadProbe.ref), parent.ref)
+    parent.watch(actorUnderTest)
+
+    val aggregatedInitialHash: String = "AggregatedInitialHash"
+    actorUnderTest.setState(WaitingForCacheHitOrMiss, CCRJAWithData(callCacheHashingActor.ref, aggregatedInitialHash, None, 1))
+
+    val reason: Exception = new Exception("Lookup failed")
+    callCacheHashingActor.send(actorUnderTest, CacheResultLookupFailure(reason))
+    parent.expectMsg(HashError(reason))
+    parent.expectMsg(CacheMiss)
+
+    parent.expectTerminated(actorUnderTest)
+  }
+}
diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCachingSlickDatabaseSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCachingSlickDatabaseSpec.scala
new file mode 100644
index 000000000..72ea41700
--- /dev/null
+++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCachingSlickDatabaseSpec.scala
@@ -0,0 +1,82 @@
+package cromwell.engine.workflow.lifecycle.execution.callcaching
+
+import cats.data.NonEmptyList
+import com.typesafe.config.ConfigFactory
+import cromwell.core.Tags.DbmsTest
+import cromwell.core.WorkflowId
+import cromwell.database.slick.SlickDatabase
+import cromwell.database.sql.joins.CallCachingJoin
+import cromwell.database.sql.tables.{CallCachingAggregationEntry, CallCachingEntry, CallCachingHashEntry}
+import cromwell.services.ServicesStore
+import org.scalatest.concurrent.ScalaFutures
+import org.scalatest.time.{Millis, Seconds, Span}
+import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers}
+import org.specs2.mock.Mockito
+
+import scala.concurrent.ExecutionContext
+
+class CallCachingSlickDatabaseSpec extends FlatSpec with Matchers with ScalaFutures with BeforeAndAfterAll with Mockito {
+
+  implicit val ec = ExecutionContext.global
+  implicit val defaultPatience = PatienceConfig(scaled(Span(5, Seconds)), scaled(Span(100, Millis)))
+
+  "SlickDatabase (hsqldb)" should behave like testWith("database")
+
+  "SlickDatabase (mysql)" should behave like testWith("database-test-mysql")
+
+  def testWith(configPath: String): Unit = {
+    import ServicesStore.EnhancedSqlDatabase
+
+    lazy val databaseConfig = ConfigFactory.load.getConfig(configPath)
+    lazy val dataAccess = new SlickDatabase(databaseConfig).initialized
+
+    val callCachingEntryA = CallCachingEntry(
+      WorkflowId.randomId().toString,
+      "AwesomeWorkflow.GoodJob",
+      1,
+      None,
+      None,
+      allowResultReuse = false
+    )
+
+    val callCachingHashEntriesA = Seq(
+      CallCachingHashEntry(
+        hashKey = "input: String s1",
+        hashValue = "HASH_S1"
+      ),
+      CallCachingHashEntry(
+        hashKey = "input: String s2",
+        hashValue = "HASH_S2"
+      ),
+      CallCachingHashEntry(
+        hashKey = "input: String s4",
+        hashValue = "HASH_S4"
+      )
+    )
+
+    val aggregation = Option(CallCachingAggregationEntry("BASE_AGGREGATION", Option("FILE_AGGREGATION")))
+
+    it should "honor allowResultReuse" taggedAs DbmsTest in {
+      (for {
+        _ <- dataAccess.addCallCaching(Seq(
+          CallCachingJoin(
+            callCachingEntryA,
+            callCachingHashEntriesA,
+            aggregation,
+            Seq.empty, Seq.empty
+          )
+        ),
+          100
+        )
+        hasBaseAggregation <- dataAccess.hasMatchingCallCachingEntriesForBaseAggregation("BASE_AGGREGATION")
+        _ = hasBaseAggregation shouldBe false
+        hasHashPairMatch <- dataAccess.hasMatchingCallCachingEntriesForHashKeyValues(
+          NonEmptyList.of("input: String s1" -> "HASH_S1")
+        )
+        _ = hasHashPairMatch shouldBe false
+        hit <- dataAccess.findCacheHitForAggregation("BASE_AGGREGATION", Option("FILE_AGGREGATION"), 1)
+        _ = hit shouldBe empty
+      } yield ()).futureValue
+    }
+  }
+}
diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EJHADataSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EJHADataSpec.scala
deleted file mode 100644
index 9c3d8d547..000000000
--- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EJHADataSpec.scala
+++ /dev/null
@@ -1,96 +0,0 @@
-package cromwell.engine.workflow.lifecycle.execution.callcaching
-
-import cromwell.core.callcaching._
-import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheReadActor.CacheResultMatchesForHashes
-import org.scalatest.{FlatSpec, Matchers}
-
-class EJHADataSpec extends FlatSpec with Matchers {
-
-  behavior of "EJHA data"
-
-  val hashKey1 = HashKey("Peter Piper picked a peck of pickled peppers")
-  val hashKey2 = HashKey("I saw Susie sitting in a shoe shine shop: Where she sits she shines, and where she shines she sits")
-  val hashKey3 = HashKey("How many boards could the Mongols hoard if the Mongol hordes got bored?")
-  val hashKey4 = HashKey("The sixth sick Sikh's sixth sheep is sick.")
-  val hashKey5 = HashKey("The Doge did what a Doge does, when a Doge does his duty to a Duke, that is. When the Doge did his duty and the Duke didn't, that's when the Duchess did the dirt to the Duke with the Doge. There they were in the dark: The Duke with his dagger, the Doge with his dart and the Duchess with her dirk. The Duchess dug at the Duke just when the Duke dove at the Doge. Now the Duke ducked, the Doge dodged, and the Duchess didn't. So the Duke got the Duchess, the Duchess got the Doge, and the Doge got the Duke.")
-
-  val allHashKeys = Set(hashKey1, hashKey2, hashKey3, hashKey4, hashKey5)
-
-  it should "create lists appropriately in the apply method" in {
-    val readWriteData = EJHAData(allHashKeys, CallCachingActivity(ReadAndWriteCache))
-    readWriteData.remainingCacheChecks should be(allHashKeys)
-    readWriteData.remainingHashesNeeded should be(allHashKeys)
-
-    val readOnlyData = EJHAData(allHashKeys, CallCachingActivity(ReadCache))
-    readOnlyData.remainingCacheChecks should be(allHashKeys)
-    readOnlyData.remainingHashesNeeded should be(Set.empty[HashKey])
-
-    val writeOnlyData = EJHAData(allHashKeys, CallCachingActivity(WriteCache))
-    writeOnlyData.remainingCacheChecks should be(Set.empty[HashKey])
-    writeOnlyData.remainingHashesNeeded should be(allHashKeys)
-  }
-
-
-  it should "accumulate new hashes" in {
-    val data = EJHAData(allHashKeys, CallCachingActivity(WriteCache))
-    data.hashesKnown should be(Set.empty)
-    data.remainingHashesNeeded should be(allHashKeys)
-    data.allHashesKnown should be(false)
-
-    val mostHashKeys = Set(hashKey1, hashKey2, hashKey4, hashKey5)
-    val hashResults = mostHashKeys map { x => Set(makeHashResult(x)) }
-    val newData = hashResults.foldLeft(data)( (d, h) => d.withNewKnownHashes(h) )
-    newData.hashesKnown.map(_.hashKey) should be(mostHashKeys)
-    newData.remainingHashesNeeded should be(Set(hashKey3))
-    newData.allHashesKnown should be(false)
-
-    val newerData = newData.withNewKnownHashes(Set(makeHashResult(hashKey3)))
-    newerData.hashesKnown.map(_.hashKey) should be(allHashKeys)
-    newerData.remainingHashesNeeded should be(Set.empty)
-    newerData.allHashesKnown should be(true)
-  }
-
-  it should "intersect new cache meta info result IDs for cache hits" in {
-    val data = EJHAData(allHashKeys, CallCachingActivity(ReadCache))
-    data.possibleCacheResults should be(None)
-    data.allCacheResultsIntersected should be(false)
-    data.isDefinitelyCacheHit should be(false)
-    data.isDefinitelyCacheMiss should be(false)
-
-    // To save you time I'll just tell you: the intersection of all these sets is Set(5)
-    val cacheLookupResults: List[CacheResultMatchesForHashes] = List(
-      CacheResultMatchesForHashes(Set(makeHashResult(hashKey1)), Set(1, 2, 3, 4, 5, 6, 7, 8, 9, 10) map CallCachingEntryId),
-      CacheResultMatchesForHashes(Set(makeHashResult(hashKey2)), Set(1, 2, 3, 4, 5, 6) map CallCachingEntryId),
-      CacheResultMatchesForHashes(Set(makeHashResult(hashKey3)), Set(1, 2, 3, 5, 7, 8, 9, 10) map CallCachingEntryId),
-      CacheResultMatchesForHashes(Set(makeHashResult(hashKey4)), Set(4, 5, 6, 7, 8, 9, 10) map CallCachingEntryId),
-      CacheResultMatchesForHashes(Set(makeHashResult(hashKey5)), Set(1, 2, 5, 6, 7, 10) map CallCachingEntryId))
-    val newData = cacheLookupResults.foldLeft(data)( (d, c) => d.intersectCacheResults(c) )
-    newData.possibleCacheResults match{
-      case Some(set) => set should be(Set(CallCachingEntryId(5)))
-      case None => fail("There should be a cache result set")
-    }
-    newData.allCacheResultsIntersected should be(true)
-    newData.isDefinitelyCacheHit should be(true)
-    newData.isDefinitelyCacheMiss should be(false)
-  }
-
-  it should "intersect new cache meta info result IDs for cache misses" in {
-    val data = EJHAData(allHashKeys, CallCachingActivity(ReadCache))
-
-    // To save you time I'll just tell you: the intersection of all these sets is empty Set()
-    val cacheLookupResults: List[CacheResultMatchesForHashes] = List(
-      CacheResultMatchesForHashes(Set(makeHashResult(hashKey1)), Set(1, 2, 3, 4, 5, 6) map CallCachingEntryId),
-      CacheResultMatchesForHashes(Set(makeHashResult(hashKey2)), Set(1, 2, 3, 7, 8, 9) map CallCachingEntryId),
-      CacheResultMatchesForHashes(Set(makeHashResult(hashKey3)), Set(5, 7, 8, 9, 10) map CallCachingEntryId))
-    val newData = cacheLookupResults.foldLeft(data)( (d, c) => d.intersectCacheResults(c) )
-    newData.possibleCacheResults match{
-      case Some(set) => set should be(Set.empty)
-      case None => fail("There should be a cache result set")
-    }
-    newData.allCacheResultsIntersected should be(false)
-    newData.isDefinitelyCacheHit should be(false)
-    newData.isDefinitelyCacheMiss should be(true)
-  }
-
-  private def makeHashResult(hashKey: HashKey): HashResult = HashResult(hashKey, HashValue("whatever"))
-}
diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActorSpec.scala
index 9c718c5d6..1b744532c 100644
--- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActorSpec.scala
+++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/EngineJobHashingActorSpec.scala
@@ -1,221 +1,222 @@
 package cromwell.engine.workflow.lifecycle.execution.callcaching
 
-import akka.actor.{ActorRef, ActorSystem, Props}
-import akka.testkit.{ImplicitSender, TestProbe}
-import cats.data.NonEmptyList
-import cromwell.CromwellTestKitSpec
+import akka.actor.{Actor, ActorRef, Props}
+import akka.testkit.{TestActorRef, TestProbe}
 import cromwell.backend._
-import cromwell.backend.callcaching.FileHashingActor.{FileHashResponse, SingleFileHashRequest}
+import cromwell.core._
 import cromwell.core.callcaching._
-import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.{CacheHit, CacheMiss, CallCacheHashes}
-import org.scalatest.mockito.MockitoSugar
-import org.scalatest.{Matchers, WordSpecLike}
-import wdl4s._
-import wdl4s.values.{WdlFile, WdlValue}
+import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheHashingJobActor.{CompleteFileHashingResult, InitialHashingResult, NoFileHashesResult}
+import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheReadingJobActor.NextHit
+import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor._
+import cromwell.services.metadata.MetadataService.PutMetadataAction
+import org.scalatest.concurrent.Eventually
+import org.scalatest.prop.TableDrivenPropertyChecks
+import org.scalatest.{FlatSpecLike, Matchers}
+import wdl4s.wdl.values.WdlValue
+import wdl4s.wdl.{WdlTask, WdlTaskCall}
+
+class EngineJobHashingActorSpec extends TestKitSuite with FlatSpecLike with Matchers with BackendSpec with TableDrivenPropertyChecks with Eventually {
+  behavior of "EngineJobHashingActor"
 
-import scala.concurrent.duration._
-import scala.language.postfixOps
-
-class EngineJobHashingActorSpec extends CromwellTestKitSpec
-  with ImplicitSender with WordSpecLike with Matchers with MockitoSugar {
-
-  import EngineJobHashingActorSpec._
+  def templateJobDescriptor(inputs: Map[LocallyQualifiedName, WdlValue] = Map.empty) = {
+    val task = mock[WdlTask]
+    val call = mock[WdlTaskCall]
+    task.commandTemplateString returns "Do the stuff... now!!"
+    task.outputs returns List.empty
+    task.fullyQualifiedName returns "workflow.hello"
+    call.task returns task
+    val workflowDescriptor = mock[BackendWorkflowDescriptor]
+    workflowDescriptor.id returns WorkflowId.randomId()
+    val jobDescriptor = BackendJobDescriptor(workflowDescriptor, BackendJobDescriptorKey(call, None, 1), Map.empty, fqnMapToDeclarationMap(inputs), NoDocker, Map.empty)
+    jobDescriptor
+  }
+  
+  val serviceRegistryActorProbe = TestProbe()
+
+  def makeEJHA(receiver: ActorRef, activity: CallCachingActivity, ccReaderProps: Props = Props.empty) = {
+    TestActorRef[EngineJobHashingActor](
+      EngineJobHashingActorTest.props(
+        receiver,
+        serviceRegistryActorProbe.ref,
+        templateJobDescriptor(),
+        None,
+        Props.empty,
+        ccReaderProps,
+        Set.empty,
+        "backend",
+        activity,
+        DockerWithHash("ubuntu@sha256:blablabla")
+      )
+    )
+  }
 
-  implicit val actorSystem: ActorSystem = system
+  it should "record initial hashes" in {
+    val receiver = TestProbe()
+    val activity = CallCachingActivity(ReadAndWriteCache)
+    val actorUnderTest = makeEJHA(receiver.ref, activity)
 
-  val readModes = List(CallCachingActivity(ReadCache), CallCachingActivity(ReadAndWriteCache))
-  val writeModes = List(CallCachingActivity(WriteCache), CallCachingActivity(ReadAndWriteCache))
-  val allModes = List(CallCachingActivity(ReadCache), CallCachingActivity(WriteCache), CallCachingActivity(ReadAndWriteCache))
+    val initialResult: InitialHashingResult = mock[InitialHashingResult]
+    actorUnderTest ! initialResult
+    eventually {
+      actorUnderTest.underlyingActor.initialHash shouldBe Some(initialResult)
+    }
+  }
 
-  "Engine job hashing actor" must {
-    allModes foreach { activity =>
-      val expectation = activity.readWriteMode match {
-        case ReadCache => "cache hit"
-        case WriteCache => "hashes"
-        case ReadAndWriteCache => "cache hit and hashes"
-      }
+  it should "create a CCReader actor or not depending on CC activity" in {
+    val activities = Table(
+      ("readWriteMode", "hasCCReadActor"),
+      (ReadCache, true),
+      (WriteCache, false),
+      (ReadAndWriteCache, true)
+    )
+    forAll(activities) { case ((readWriteMode, hasCCReadActor)) =>
+      val receiver = TestProbe()
+      val actorUnderTest = makeEJHA(receiver.ref, CallCachingActivity(readWriteMode))
+      actorUnderTest.underlyingActor.callCacheReadingJobActor.isDefined shouldBe hasCCReadActor
+    }
+  }
 
-      s"Respect the CallCachingMode and report back $expectation for the ${activity.readWriteMode} activity" in {
-        val singleCallCachingEntryIdSet = Set(CallCachingEntryId(1))
-        val replyTo = TestProbe()
-        val deathWatch = TestProbe()
+  it should "send hashes to receiver when receiving a NoFileHashesResult" in {
+    val receiver = TestProbe()
+    val actorUnderTest = makeEJHA(receiver.ref, CallCachingActivity(ReadAndWriteCache))
+    val initialHashes = Set(HashResult(HashKey("key"), HashValue("value")))
+    val initialAggregatedHash = "aggregatedHash"
+    val initialResult = InitialHashingResult(initialHashes, initialAggregatedHash)
+    actorUnderTest ! initialResult
+    actorUnderTest ! NoFileHashesResult
+    receiver.expectMsg(CallCacheHashes(initialHashes, initialAggregatedHash, None))
+  }
 
-        val cacheLookupResponses: Map[String, Set[CallCachingEntryId]] = if (activity.readFromCache) standardCacheLookupResponses(singleCallCachingEntryIdSet, singleCallCachingEntryIdSet, singleCallCachingEntryIdSet, singleCallCachingEntryIdSet) else Map.empty
-        val ejha = createEngineJobHashingActor(
-          replyTo = replyTo.ref,
-          activity = activity,
-          cacheLookupResponses = cacheLookupResponses)
+  it should "send hashes to receiver when receiving a CompleteFileHashingResult" in {
+    val receiver = TestProbe()
+    val actorUnderTest = makeEJHA(receiver.ref, CallCachingActivity(ReadAndWriteCache))
+    
+    val initialHashes = Set(HashResult(HashKey("key"), HashValue("value")))
+    val initialAggregatedHash = "aggregatedHash"
+    val initialResult = InitialHashingResult(initialHashes, initialAggregatedHash)
+    val fileHashes = Set(HashResult(HashKey("file key"), HashValue("value")))
+    val fileAggregatedHash = "aggregatedFileHash"
+    val fileResult = CompleteFileHashingResult(fileHashes, fileAggregatedHash)
+    
+    actorUnderTest ! initialResult
+    actorUnderTest ! fileResult
+    receiver.expectMsg(CallCacheHashes(initialHashes, initialAggregatedHash, Option(FileHashes(fileHashes, fileAggregatedHash))))
+  }
 
-        deathWatch watch ejha
+  it should "forward CacheMiss to receiver" in {
+    val receiver = TestProbe()
+    val activity = CallCachingActivity(ReadAndWriteCache)
+    val actorUnderTest = makeEJHA(receiver.ref, activity)
 
-        if (activity.readFromCache) replyTo.expectMsg(CacheHit(NonEmptyList.of(CallCachingEntryId(1))))
-        if (activity.writeToCache) replyTo.expectMsgPF(max = 5 seconds, hint = "awaiting cache hit message") {
-          case CallCacheHashes(hashes) => hashes.size should be(4)
-          case x => fail(s"Cache hit anticipated! Instead got a ${x.getClass.getSimpleName}")
-        }
+    actorUnderTest ! CacheMiss
+    receiver.expectMsg(CacheMiss)
+  }
 
-        deathWatch.expectTerminated(ejha, 5 seconds)
-      }
+  it should "forward CacheHit to receiver" in {
+    val receiver = TestProbe()
+    val activity = CallCachingActivity(ReadAndWriteCache)
+    val actorUnderTest = makeEJHA(receiver.ref, activity)
 
-      s"Wait for requests to the FileHashingActor for the ${activity.readWriteMode} activity" in {
-        val singleCallCachingEntryIdSet = Set(CallCachingEntryId(1))
-        val replyTo = TestProbe()
-        val fileHashingActor = TestProbe()
-        val deathWatch = TestProbe()
-
-        val initialCacheLookupResponses: Map[String, Set[CallCachingEntryId]] = if (activity.readFromCache) standardCacheLookupResponses(singleCallCachingEntryIdSet, singleCallCachingEntryIdSet, singleCallCachingEntryIdSet, singleCallCachingEntryIdSet) else Map.empty
-        val fileCacheLookupResponses = Map("input: File inputFile1" -> singleCallCachingEntryIdSet, "input: File inputFile2" -> singleCallCachingEntryIdSet)
-
-        val jobDescriptor = templateJobDescriptor(inputs = Map(
-            "inputFile1" -> WdlFile("path"),
-            "inputFile2" -> WdlFile("path")))
-        val ejha = createEngineJobHashingActor(
-          replyTo = replyTo.ref,
-          activity = activity,
-          jobDescriptor = jobDescriptor,
-          fileHashingActor = Option(fileHashingActor.ref),
-          cacheLookupResponses = initialCacheLookupResponses ++ fileCacheLookupResponses)
-
-        deathWatch watch ejha
-
-        twice { iteration =>
-          fileHashingActor.expectMsgPF(max = 5 seconds, hint = s"awaiting file hash request #$iteration") {
-            case SingleFileHashRequest(jobKey, hashKey, file, initializationData) =>
-              file should be(WdlFile("path"))
-              fileHashingActor.send(ejha, FileHashResponse(HashResult(hashKey, HashValue("blah di blah"))))
-            case x => fail(s"SingleFileHashRequest anticipated! Instead got a ${x.getClass.getSimpleName}")
-          }
-        }
-
-        if (activity.readFromCache) replyTo.expectMsg(CacheHit(NonEmptyList.of(CallCachingEntryId(1))))
-        if (activity.writeToCache) replyTo.expectMsgPF(max = 5 seconds, hint = "awaiting cache hit message") {
-          case CallCacheHashes(hashes) => hashes.size should be(6)
-          case x => fail(s"Cache hit anticipated! Instead got a ${x.getClass.getSimpleName}")
-        }
-
-        deathWatch.expectTerminated(ejha, 5 seconds)
-      }
+    val cacheHit = mock[CacheHit]
+    actorUnderTest ! cacheHit
+    receiver.expectMsg(cacheHit)
+  }
 
-      s"Cache miss for bad FileHashingActor results but still return hashes in the ${activity.readWriteMode} activity" in {
-        val singleCallCachingEntryIdSet = Set(CallCachingEntryId(1))
-        val replyTo = TestProbe()
-        val fileHashingActor = TestProbe()
-        val deathWatch = TestProbe()
-
-        val initialCacheLookupResponses: Map[String, Set[CallCachingEntryId]] = if (activity.readFromCache) standardCacheLookupResponses(singleCallCachingEntryIdSet, singleCallCachingEntryIdSet, singleCallCachingEntryIdSet, singleCallCachingEntryIdSet) else Map.empty
-        val fileCacheLookupResponses = Map("input: File inputFile1" -> Set(CallCachingEntryId(2)), "input: File inputFile2" -> singleCallCachingEntryIdSet)
-
-        val jobDescriptor = templateJobDescriptor(inputs = Map(
-          "inputFile1" -> WdlFile("path"),
-          "inputFile2" -> WdlFile("path")))
-        val ejha = createEngineJobHashingActor(
-          replyTo = replyTo.ref,
-          activity = activity,
-          jobDescriptor = jobDescriptor,
-          fileHashingActor = Option(fileHashingActor.ref),
-          cacheLookupResponses = initialCacheLookupResponses ++ fileCacheLookupResponses)
-
-        deathWatch watch ejha
-
-        // Hello, future Cromwellian! I imagine you're reading this because you've just introduced file hash short-circuiting on cache miss and,
-        // depending on timings, you may not get the second file hash request in read-only mode. You might want to refactor this test to reply
-        // only to the "cache miss" file and then check that the test probe receives the appropriate "cancellation" message.
-        // ... or not! Don't just blindly allow a ghost of the past to tell you what to do! Live your own life and excel!
-        twice { iteration =>
-          fileHashingActor.expectMsgPF(max = 5 seconds, hint = s"awaiting file hash request #$iteration") {
-            case SingleFileHashRequest(jobKey, hashKey, file, initializationData) =>
-              file should be(WdlFile("path"))
-              fileHashingActor.send(ejha, FileHashResponse(HashResult(hashKey, HashValue("blah di blah"))))
-            case x => fail(s"SingleFileHashRequest anticipated! Instead got a ${x.getClass.getSimpleName}")
-          }
-        }
-
-        if (activity.readFromCache) replyTo.expectMsg(CacheMiss)
-        if (activity.writeToCache) replyTo.expectMsgPF(max = 5 seconds, hint = "awaiting cache hit message") {
-          case CallCacheHashes(hashes) => hashes.size should be(6)
-          case x => fail(s"Cache hit anticipated! Instead got a ${x.getClass.getSimpleName}")
-        }
-
-        deathWatch.expectTerminated(ejha, 5 seconds)
+  it should "forward NextHit to CCRead actor" in {
+    val receiver = TestProbe()
+    val activity = CallCachingActivity(ReadAndWriteCache)
+    val monitorProbe = TestProbe()
+    val ccReadActorProps = Props(new Actor {
+      override def receive: Receive = {
+        case NextHit => monitorProbe.ref forward NextHit
       }
+    })
+    
+    val actorUnderTest = makeEJHA(receiver.ref, activity, ccReadActorProps)
 
-      s"Detect call cache misses for the ${activity.readWriteMode} activity" in {
-        val singleCallCachingEntryIdSet = Set(CallCachingEntryId(1))
-        val replyTo = TestProbe()
-        val deathWatch = TestProbe()
-
-        val cacheLookupResponses: Map[String, Set[CallCachingEntryId]] = if (activity.readFromCache) standardCacheLookupResponses(singleCallCachingEntryIdSet, singleCallCachingEntryIdSet, Set(CallCachingEntryId(2)), singleCallCachingEntryIdSet) else Map.empty
-        val ejha = createEngineJobHashingActor(
-          replyTo = replyTo.ref,
-          activity = activity,
-          cacheLookupResponses = cacheLookupResponses)
-
-        deathWatch watch ejha
+    actorUnderTest ! NextHit
+    monitorProbe.expectMsg(NextHit)
+  }
 
-        if (activity.readFromCache) replyTo.expectMsg(CacheMiss)
-        if (activity.writeToCache) replyTo.expectMsgPF(max = 5 seconds, hint = "awaiting cache hit message") {
-          case CallCacheHashes(hashes) => hashes.size should be(4)
-          case x => fail(s"Cache hit anticipated! Instead got a ${x.getClass.getSimpleName}")
-        }
+  it should "fail if it receives NextHit and doesn't have a CCRead actor" in {
+    val receiver = TestProbe()
+    val activity = CallCachingActivity(WriteCache)
+    val actorUnderTest = makeEJHA(receiver.ref, activity)
+    receiver.watch(actorUnderTest)
+    actorUnderTest ! NextHit
+    receiver.expectMsgClass(classOf[HashError])
+    serviceRegistryActorProbe.expectMsgClass(classOf[PutMetadataAction])
+    receiver.expectTerminated(actorUnderTest)
+  }
 
-        deathWatch.expectTerminated(ejha, 5 seconds)
-      }
-    }
+  it should "fail if it receives a HashingFailedMessage" in {
+    val receiver = TestProbe()
+    val activity = CallCachingActivity(ReadAndWriteCache)
+    val actorUnderTest = makeEJHA(receiver.ref, activity)
+    receiver.watch(actorUnderTest)
+    actorUnderTest ! HashingFailedMessage("someFile", new Exception("[TEST] Some exception"))
+    receiver.expectMsgClass(classOf[HashError])
+    serviceRegistryActorProbe.expectMsgClass(classOf[PutMetadataAction])
+    receiver.expectTerminated(actorUnderTest)
   }
-}
 
-object EngineJobHashingActorSpec extends BackendSpec {
-  import org.mockito.Mockito._
-
-  def createEngineJobHashingActor
-  (
-    replyTo: ActorRef,
-    activity: CallCachingActivity,
-    jobDescriptor: BackendJobDescriptor = templateJobDescriptor(),
-    initializationData: Option[BackendInitializationData] = None,
-    fileHashingActor: Option[ActorRef] = None,
-    cacheLookupResponses: Map[String, Set[CallCachingEntryId]] = Map.empty,
-    runtimeAttributeDefinitions: Set[RuntimeAttributeDefinition] = Set.empty,
-    backendName: String = "whatever"
-  )(implicit system: ActorSystem) = {
-    val callCacheReadActor = system.actorOf(Props(new PredictableCallCacheReadActor(cacheLookupResponses)))
-    
-    system.actorOf(EngineJobHashingActor.props(
-      receiver = replyTo,
+  it should "fail if it receives a FinalFileHashingResult but has no InitialHashingResult" in {
+    val receiver = TestProbe()
+    val activity = CallCachingActivity(ReadAndWriteCache)
+    val actorUnderTest = makeEJHA(receiver.ref, activity)
+    receiver.watch(actorUnderTest)
+    actorUnderTest ! NoFileHashesResult
+    receiver.expectMsgClass(classOf[HashError])
+    serviceRegistryActorProbe.expectMsgClass(classOf[PutMetadataAction])
+    receiver.expectTerminated(actorUnderTest)
+  }
+  
+  object EngineJobHashingActorTest {
+    def props(receiver: ActorRef,
+              serviceRegistryActor: ActorRef,
+              jobDescriptor: BackendJobDescriptor,
+              initializationData: Option[BackendInitializationData],
+              fileHashingActorProps: Props,
+              callCacheReadingJobActorProps: Props,
+              runtimeAttributeDefinitions: Set[RuntimeAttributeDefinition],
+              backendName: String,
+              activity: CallCachingActivity,
+              callCachingEligible: CallCachingEligible): Props = Props(new EngineJobHashingActorTest(
+      receiver = receiver,
+      serviceRegistryActor = serviceRegistryActor,
       jobDescriptor = jobDescriptor,
       initializationData = initializationData,
-      fileHashingActor = fileHashingActor.getOrElse(emptyActor),
-      callCacheReadActor = callCacheReadActor,
+      fileHashingActorProps = fileHashingActorProps,
+      callCacheReadingJobActorProps = callCacheReadingJobActorProps,
       runtimeAttributeDefinitions = runtimeAttributeDefinitions,
       backendName = backendName,
-      activity = activity))
+      activity = activity,
+      callCachingEligible = callCachingEligible))
   }
-
-  def emptyActor(implicit actorSystem: ActorSystem) = actorSystem.actorOf(Props.empty)
-
-  def templateJobDescriptor(inputs: Map[LocallyQualifiedName, WdlValue] = Map.empty) = {
-    val task = mock[Task]
-    val call = mock[TaskCall]
-    when(task.commandTemplateString).thenReturn("Do the stuff... now!!")
-    when(task.outputs).thenReturn(List.empty)
-    when(call.task).thenReturn(task)
-    val workflowDescriptor = mock[BackendWorkflowDescriptor]
-    val jobDescriptor = BackendJobDescriptor(workflowDescriptor, BackendJobDescriptorKey(call, None, 1), Map.empty, fqnMapToDeclarationMap(inputs), CallCachingEligible)
-    jobDescriptor
+  
+  class EngineJobHashingActorTest(receiver: ActorRef,
+                                  serviceRegistryActor: ActorRef,
+                                  jobDescriptor: BackendJobDescriptor,
+                                  initializationData: Option[BackendInitializationData],
+                                  fileHashingActorProps: Props,
+                                  callCacheReadingJobActorProps: Props,
+                                  runtimeAttributeDefinitions: Set[RuntimeAttributeDefinition],
+                                  backendName: String,
+                                  activity: CallCachingActivity,
+                                  callCachingEligible: CallCachingEligible) extends EngineJobHashingActor(
+    receiver = receiver,
+    serviceRegistryActor = serviceRegistryActor,
+    jobDescriptor = jobDescriptor,
+    initializationData = initializationData,
+    fileHashingActorProps = fileHashingActorProps,
+    callCacheReadingJobActorProps = callCacheReadingJobActorProps,
+    runtimeAttributeDefinitions = runtimeAttributeDefinitions,
+    backendName = backendName,
+    activity = activity,
+    callCachingEligible = callCachingEligible) {
+    // override preStart to nothing to prevent the creation of the CCHJA.
+    // This way it doesn't interfere with the tests and we can manually inject the messages we want
+    override def preStart() =  ()
   }
 
-  def standardCacheLookupResponses(commandTemplate: Set[CallCachingEntryId],
-                                   inputCount: Set[CallCachingEntryId],
-                                   backendName: Set[CallCachingEntryId],
-                                   outputCount: Set[CallCachingEntryId]) = Map(
-    "command template" -> commandTemplate,
-    "input count" -> inputCount,
-    "backend name" -> backendName,
-    "output count" -> outputCount
-  )
-
-  def twice[A](block: Int => A) = {
-    block(1)
-    block(2)
-  }
 }
diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/PredictableCallCacheReadActor.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/PredictableCallCacheReadActor.scala
deleted file mode 100644
index c98587840..000000000
--- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/PredictableCallCacheReadActor.scala
+++ /dev/null
@@ -1,42 +0,0 @@
-package cromwell.engine.workflow.lifecycle.execution.callcaching
-
-import akka.actor.{Actor, ActorLogging, ActorRef}
-import cromwell.core.callcaching.HashResult
-import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheReadActor.{CacheLookupRequest, CacheResultLookupFailure, CacheResultMatchesForHashes}
-import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.CallCacheHashes
-
-import scala.util.{Failure, Success, Try}
-
-/**
-  * Has a set of responses which it will respond with. If it gets a request for anything that it's not expecting to respond to will generate a failure.
-  */
-class PredictableCallCacheReadActor(responses: Map[String, Set[CallCachingEntryId]]) extends Actor with ActorLogging {
-
-  var responsesRemaining = responses
-
-  override def receive = {
-    case CacheLookupRequest(callCacheHashes: CallCacheHashes) =>
-      callCacheHashes.hashes.toList match {
-        case Nil =>
-          throw new Exception("Should never be looking up 0 hash keys!?")
-        case head :: tail =>
-          val startSet = toTry(head.hashKey.key, responses.get(head.hashKey.key))
-          respond(sender, callCacheHashes.hashes, tail.foldLeft(startSet)(resultLookupFolder))
-      }
-  }
-
-  private def respond(sndr: ActorRef, hashes: Set[HashResult], result: Try[Set[CallCachingEntryId]]) = result match {
-    case Success(cacheMatches) => sndr ! CacheResultMatchesForHashes(hashes, cacheMatches)
-    case Failure(t) => sndr ! CacheResultLookupFailure(t)
-  }
-
-  private def toTry[A](name: String, option: Option[A]): Try[A] = option match {
-    case Some(x) => Success(x)
-    case None => Failure(new Exception(s"Error looking up response $name!"))
-  }
-
-  private def resultLookupFolder(current: Try[Set[CallCachingEntryId]], next: HashResult): Try[Set[CallCachingEntryId]] = current flatMap { c =>
-    val lookedUp = toTry(next.hashKey.key, responses.get(next.hashKey.key))
-    lookedUp map { l => c.intersect(l) }
-  }
-}
diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaBackendIsCopyingCachedOutputsSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaBackendIsCopyingCachedOutputsSpec.scala
index b39711759..b58485f3f 100644
--- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaBackendIsCopyingCachedOutputsSpec.scala
+++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaBackendIsCopyingCachedOutputsSpec.scala
@@ -1,14 +1,14 @@
 package cromwell.engine.workflow.lifecycle.execution.ejea
 
-import cats.data.NonEmptyList
-import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor._
-import EngineJobExecutionActorSpec._
 import cromwell.core.callcaching._
-import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.{CacheHit, CallCacheHashes, EJHAResponse, HashError}
+import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor._
+import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheReadingJobActor.NextHit
 import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCachingEntryId
+import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.{CacheHit, CallCacheHashes, EJHAResponse, HashError}
+import cromwell.engine.workflow.lifecycle.execution.ejea.EngineJobExecutionActorSpec._
+import cromwell.engine.workflow.lifecycle.execution.ejea.HasJobSuccessResponse.SuccessfulCallCacheHashes
 
 import scala.util.{Failure, Success, Try}
-import cromwell.engine.workflow.lifecycle.execution.ejea.HasJobSuccessResponse.SuccessfulCallCacheHashes
 
 class EjeaBackendIsCopyingCachedOutputsSpec extends EngineJobExecutionActorSpec with HasJobSuccessResponse with HasJobFailureResponses with CanExpectJobStoreWrites with CanExpectCacheWrites with CanExpectCacheInvalidation {
 
@@ -20,7 +20,7 @@ class EjeaBackendIsCopyingCachedOutputsSpec extends EngineJobExecutionActorSpec
     val hashResultsDataValue = Some(Success(SuccessfulCallCacheHashes))
     val hashErrorDataValue = Some(Failure(hashErrorCause))
 
-    val hashResultsEjhaResponse = Some(SuccessfulCallCacheHashes)
+    val hashResultsEjhaResponse: Option[CallCacheHashes] = Some(SuccessfulCallCacheHashes)
     val hashErrorEjhaResponse = Some(HashError(hashErrorCause))
 
     case class InitialHashDataAndEjhaResponseCombination(name: String,
@@ -53,7 +53,6 @@ class EjeaBackendIsCopyingCachedOutputsSpec extends EngineJobExecutionActorSpec
 
           // Nothing should happen here:
           helper.jobStoreProbe.expectNoMsg(awaitAlmostNothing)
-          helper.callCacheWriteActorCreations should be(NothingYet) // Rely on the await timeout from the previous step to allow time to pass
 
           // Send the response from the copying actor
           ejea ! successResponse
@@ -79,7 +78,6 @@ class EjeaBackendIsCopyingCachedOutputsSpec extends EngineJobExecutionActorSpec
           ejhaResponse foreach { resp =>
             // Nothing should have happened yet:
             helper.jobStoreProbe.expectNoMsg(awaitAlmostNothing)
-            helper.callCacheWriteActorCreations should be(NothingYet) // Rely on the await timeout from the previous step to allow time to pass
 
             // Ok, now send the response from the EJHA (if there was one!):
             ejea ! resp
@@ -108,28 +106,30 @@ class EjeaBackendIsCopyingCachedOutputsSpec extends EngineJobExecutionActorSpec
             eventually {
               ejea.stateName should be(InvalidatingCacheEntry)
             }
-            ejea.stateData should be(ResponsePendingData(helper.backendJobDescriptor, helper.bjeaProps, initialHashData, cacheHit))
+            ejea.stateData should be(ResponsePendingData(helper.backendJobDescriptor, helper.bjeaProps, initialHashData, Option(helper.ejhaProbe.ref), cacheHit))
           }
 
           s"not invalidate a call for caching if backend coping failed when invalidation is disabled, when it was going to receive $hashComboName, if call caching is $mode" in {
             val invalidationDisabledOptions = CallCachingOptions(invalidateBadCacheResults = false)
             val cacheInvalidationDisabledMode = mode match {
-              case CallCachingActivity(rw, options) => CallCachingActivity(rw, invalidationDisabledOptions)
+              case CallCachingActivity(rw, _) => CallCachingActivity(rw, invalidationDisabledOptions)
               case _ => fail(s"Mode $mode not appropriate for cache invalidation tests")
             }
             ejea = ejeaInBackendIsCopyingCachedOutputsState(initialHashData, cacheInvalidationDisabledMode)
             // Send the response from the copying actor
             ejea ! failureNonRetryableResponse
 
+            helper.ejhaProbe.expectMsg(NextHit)
+            
             eventually {
-              ejea.stateName should be(RunningJob)
+              ejea.stateName should be(CheckingCallCache)
             }
             // Make sure we didn't start invalidating anything:
             helper.invalidateCacheActorCreations.hasExactlyOne should be(false)
-            ejea.stateData should be(ResponsePendingData(helper.backendJobDescriptor, helper.bjeaProps, initialHashData, None))
+            ejea.stateData should be(ResponsePendingData(helper.backendJobDescriptor, helper.bjeaProps, initialHashData, Option(helper.ejhaProbe.ref), cacheHit))
           }
 
-          s"invalidate a call for caching if backend coping failed (preserving and received hashes) when call caching is $mode, the EJEA has $hashComboName and then gets a success result" in {
+          s"invalidate a call for caching if backend copying failed (preserving and received hashes) when call caching is $mode, the EJEA has $hashComboName and then gets a success result" in {
             ejea = ejeaInBackendIsCopyingCachedOutputsState(initialHashData, mode)
             // Send the response from the EJHA (if there was one!):
             ejhaResponse foreach {
@@ -138,7 +138,6 @@ class EjeaBackendIsCopyingCachedOutputsSpec extends EngineJobExecutionActorSpec
 
             // Nothing should happen here:
             helper.jobStoreProbe.expectNoMsg(awaitAlmostNothing)
-            helper.callCacheWriteActorCreations should be(NothingYet) // Rely on the await timeout from the previous step to allow time to pass
 
             // Send the response from the copying actor
             ejea ! failureNonRetryableResponse
@@ -147,7 +146,7 @@ class EjeaBackendIsCopyingCachedOutputsSpec extends EngineJobExecutionActorSpec
             eventually {
               ejea.stateName should be(InvalidatingCacheEntry)
             }
-            ejea.stateData should be(ResponsePendingData(helper.backendJobDescriptor, helper.bjeaProps, finalHashData, cacheHit))
+            ejea.stateData should be(ResponsePendingData(helper.backendJobDescriptor, helper.bjeaProps, finalHashData, Option(helper.ejhaProbe.ref), cacheHit))
           }
         }
       }
@@ -155,7 +154,7 @@ class EjeaBackendIsCopyingCachedOutputsSpec extends EngineJobExecutionActorSpec
   }
 
   private val cacheId: CallCachingEntryId = CallCachingEntryId(74)
-  private val cacheHit = Option(CacheHit(NonEmptyList.of(cacheId)))
-  def standardResponsePendingData(hashes: Option[Try[CallCacheHashes]]) = ResponsePendingData(helper.backendJobDescriptor, helper.bjeaProps, hashes, cacheHit)
+  private val cacheHit = Option(EJEACacheHit(CacheHit(cacheId), 0, None))
+  def standardResponsePendingData(hashes: Option[Try[CallCacheHashes]]) = ResponsePendingData(helper.backendJobDescriptor, helper.bjeaProps, hashes, Option(helper.ejhaProbe.ref), cacheHit)
   def ejeaInBackendIsCopyingCachedOutputsState(initialHashes: Option[Try[CallCacheHashes]], callCachingMode: CallCachingMode, restarting: Boolean = false) = helper.buildEJEA(restarting = restarting, callCachingMode = callCachingMode).setStateInline(data = standardResponsePendingData(initialHashes))
 }
diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaCheckingCacheEntryExistenceSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaCheckingCacheEntryExistenceSpec.scala
new file mode 100644
index 000000000..683a551e2
--- /dev/null
+++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaCheckingCacheEntryExistenceSpec.scala
@@ -0,0 +1,44 @@
+package cromwell.engine.workflow.lifecycle.execution.ejea
+
+import cromwell.core.callcaching.CallCachingOff
+import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor.{CheckingCacheEntryExistence, CheckingJobStore, NoData, PreparingJob}
+import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheReadActor._
+import cromwell.engine.workflow.lifecycle.execution.ejea.EngineJobExecutionActorSpec.EnhancedTestEJEA
+import cromwell.engine.workflow.lifecycle.execution.preparation.CallPreparation
+
+class EjeaCheckingCacheEntryExistenceSpec extends EngineJobExecutionActorSpec {
+
+  override implicit val stateUnderTest = CheckingJobStore
+
+  "An EJEA in EjeaCheckingCacheEntryExistence state should" should {
+    "disable call caching and prepare job if a cache entry already exists for this job" in {
+      createCheckingCacheEntryExistenceEjea()
+      
+      ejea ! HasCallCacheEntry(CallCacheEntryForCall(helper.workflowId, helper.jobDescriptorKey))
+      helper.jobPreparationProbe.expectMsg(awaitTimeout, "expecting CallPreparation Start", CallPreparation.Start)
+      ejea.stateName should be(PreparingJob)
+      
+      ejea.underlyingActor.effectiveCallCachingMode shouldBe CallCachingOff
+    }
+
+    "prepare job if no cache entry already exists" in {
+      createCheckingCacheEntryExistenceEjea()
+
+      ejea ! NoCallCacheEntry(CallCacheEntryForCall(helper.workflowId, helper.jobDescriptorKey))
+
+      helper.jobPreparationProbe.expectMsg(awaitTimeout, "expecting CallPreparation Start", CallPreparation.Start)
+      ejea.stateName should be(PreparingJob)
+    }
+
+    "prepare job if cache entry existence lookup fails" in {
+      createCheckingCacheEntryExistenceEjea()
+
+      ejea ! CacheResultLookupFailure(new Exception("[TEST] Failed to lookup cache entry existence"))
+
+      helper.jobPreparationProbe.expectMsg(awaitTimeout, "expecting CallPreparation Start", CallPreparation.Start)
+      ejea.stateName should be(PreparingJob)
+    }
+  }
+
+  private def createCheckingCacheEntryExistenceEjea(): Unit = { ejea = helper.buildEJEA(restarting = true).setStateInline(state = CheckingCacheEntryExistence, data = NoData) }
+}
diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaCheckingCallCacheSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaCheckingCallCacheSpec.scala
index 930133d10..74f379964 100644
--- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaCheckingCallCacheSpec.scala
+++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaCheckingCallCacheSpec.scala
@@ -1,11 +1,10 @@
 package cromwell.engine.workflow.lifecycle.execution.ejea
 
-import cats.data.NonEmptyList
-import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor.{CheckingCallCache, FetchingCachedOutputsFromDatabase, ResponsePendingData, RunningJob}
-import EngineJobExecutionActorSpec.EnhancedTestEJEA
 import cromwell.core.callcaching.{CallCachingActivity, CallCachingOff, ReadCache}
-import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.{CacheHit, CacheMiss, HashError}
+import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor.{CheckingCallCache, FetchingCachedOutputsFromDatabase, ResponsePendingData, RunningJob}
 import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCachingEntryId
+import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.{CacheHit, CacheMiss, HashError}
+import cromwell.engine.workflow.lifecycle.execution.ejea.EngineJobExecutionActorSpec.EnhancedTestEJEA
 import org.scalatest.concurrent.Eventually
 
 class EjeaCheckingCallCacheSpec extends EngineJobExecutionActorSpec with Eventually with CanExpectFetchCachedResults {
@@ -15,7 +14,7 @@ class EjeaCheckingCallCacheSpec extends EngineJobExecutionActorSpec with Eventua
   "An EJEA in CheckingCallCache mode" should {
     "Try to fetch the call cache outputs if it gets a CacheHit" in {
       createCheckingCallCacheEjea()
-      ejea ! CacheHit(NonEmptyList.of(CallCachingEntryId(75)))
+      ejea ! CacheHit(CallCachingEntryId(75))
       expectFetchCachedResultsActor(CallCachingEntryId(75))
       ejea.stateName should be(FetchingCachedOutputsFromDatabase)
     }
diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaCheckingJobStoreSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaCheckingJobStoreSpec.scala
index 433f87223..859dbaafe 100644
--- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaCheckingJobStoreSpec.scala
+++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaCheckingJobStoreSpec.scala
@@ -2,9 +2,9 @@ package cromwell.engine.workflow.lifecycle.execution.ejea
 
 import cromwell.backend.BackendJobExecutionActor.{JobFailedNonRetryableResponse, JobFailedRetryableResponse, JobSucceededResponse}
 import cromwell.core._
-import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor.{CheckingJobStore, NoData, PreparingJob}
+import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor.{CheckingCacheEntryExistence, CheckingJobStore, NoData}
+import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheReadActor.CallCacheEntryForCall
 import cromwell.engine.workflow.lifecycle.execution.ejea.EngineJobExecutionActorSpec.EnhancedTestEJEA
-import cromwell.engine.workflow.lifecycle.execution.preparation.CallPreparation
 import cromwell.jobstore.JobStoreActor.{JobComplete, JobNotComplete}
 import cromwell.jobstore.{JobResultFailure, JobResultSuccess}
 
@@ -54,13 +54,13 @@ class EjeaCheckingJobStoreSpec extends EngineJobExecutionActorSpec {
       }
     }
 
-    "begin preparing the job if it's not already complete" in {
+    "check for cache entry existence if it's not already complete" in {
       createCheckingJobStoreEjea()
       ejea.setState(CheckingJobStore)
       ejea ! JobNotComplete
 
-      helper.jobPreparationProbe.expectMsg(awaitTimeout, "expecting RecoverJobCommand", CallPreparation.Start)
-      ejea.stateName should be(PreparingJob)
+      helper.callCacheReadActorProbe.expectMsg(awaitTimeout, "expecting CallCacheEntryForCall", CallCacheEntryForCall(helper.workflowId, helper.jobDescriptorKey))
+      ejea.stateName should be(CheckingCacheEntryExistence)
 
       ejea.stop()
     }
diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaFetchingCachedOutputsFromDatabaseSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaFetchingCachedOutputsFromDatabaseSpec.scala
index 4acd2efab..ba8a84b91 100644
--- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaFetchingCachedOutputsFromDatabaseSpec.scala
+++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaFetchingCachedOutputsFromDatabaseSpec.scala
@@ -10,7 +10,7 @@ import cromwell.engine.workflow.lifecycle.execution.callcaching.FetchCachedResul
 import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCachingEntryId
 import cromwell.engine.workflow.lifecycle.execution.ejea.EngineJobExecutionActorSpec._
 import cromwell.engine.workflow.lifecycle.execution.ejea.HasJobSuccessResponse.SuccessfulCallCacheHashes
-import wdl4s.values.WdlString
+import wdl4s.wdl.values.WdlString
 
 import scala.util.{Failure, Success}
 
diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaInvalidatingCacheEntrySpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaInvalidatingCacheEntrySpec.scala
index ce6537824..165e030c8 100644
--- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaInvalidatingCacheEntrySpec.scala
+++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaInvalidatingCacheEntrySpec.scala
@@ -1,13 +1,13 @@
 package cromwell.engine.workflow.lifecycle.execution.ejea
 
-import cats.data.NonEmptyList
+import akka.actor.ActorRef
 import cromwell.core.callcaching.{CallCachingActivity, ReadCache}
 import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor._
-import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.CacheHit
-import cromwell.engine.workflow.lifecycle.execution.callcaching.{CallCacheInvalidatedFailure, CallCacheInvalidatedSuccess, CallCachingEntryId}
+import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCacheReadingJobActor.NextHit
+import cromwell.engine.workflow.lifecycle.execution.callcaching.{CallCacheInvalidatedFailure, CallCacheInvalidatedSuccess}
 import cromwell.engine.workflow.lifecycle.execution.ejea.EngineJobExecutionActorSpec._
 
-class EjeaInvalidatingCacheEntrySpec extends EngineJobExecutionActorSpec with CanExpectFetchCachedResults {
+class EjeaInvalidatingCacheEntrySpec extends EngineJobExecutionActorSpec {
 
   override implicit val stateUnderTest = InvalidatingCacheEntry
 
@@ -17,26 +17,21 @@ class EjeaInvalidatingCacheEntrySpec extends EngineJobExecutionActorSpec with Ca
     val invalidateSuccess = CallCacheInvalidatedSuccess(None)
     val invalidateFailure = CallCacheInvalidatedFailure(invalidationErrorCause)
 
-    val metaInfo31: CallCachingEntryId = CallCachingEntryId(31)
-    val metaInfo32: CallCachingEntryId = CallCachingEntryId(32)
-    val cacheHitWithTwoIds = CacheHit(NonEmptyList.of(metaInfo32, metaInfo31))
-    val cacheHitWithSingleId = CacheHit(NonEmptyList.of(metaInfo31))
-
     List(invalidateSuccess, invalidateFailure) foreach { invalidateActorResponse =>
-      s"try the next available hit when response is $invalidateActorResponse" in {
-        ejea = ejeaInvalidatingCacheEntryState(Option(cacheHitWithTwoIds), restarting = false)
+      s"ask the ejha for the next hit when response is $invalidateActorResponse" in {
+        ejea = ejeaInvalidatingCacheEntryState(Option(helper.ejhaProbe.ref))
         // Send the response from the invalidate actor
         ejea ! invalidateActorResponse
 
         helper.bjeaProbe.expectNoMsg(awaitAlmostNothing)
-        expectFetchCachedResultsActor(cacheHitWithSingleId.cacheResultIds.head)
-        eventually { ejea.stateName should be(FetchingCachedOutputsFromDatabase) }
-        ejea.stateData should be(ResponsePendingData(helper.backendJobDescriptor, helper.bjeaProps, None, Option(cacheHitWithSingleId)))
+        helper.ejhaProbe.expectMsg(NextHit)
+        eventually { ejea.stateName should be(CheckingCallCache) }
+        ejea.stateData should be(ResponsePendingData(helper.backendJobDescriptor, helper.bjeaProps, None, Option(helper.ejhaProbe.ref), None))
       }
 
       RestartOrExecuteCommandTuples foreach { case RestartOrExecuteCommandTuple(operationName, restarting, expectedMessage) =>
-        s"$operationName a job if cache invalidation succeeds and there are no other cache hits to try when invalidate response is $invalidateActorResponse" in {
-          ejea = ejeaInvalidatingCacheEntryState(Option(cacheHitWithSingleId), restarting = restarting)
+        s"$operationName a job if there is no ejha when invalidate response is $invalidateActorResponse" in {
+          ejea = ejeaInvalidatingCacheEntryState(None, restarting = restarting)
           // Send the response from the invalidate actor
           ejea ! invalidateActorResponse
 
@@ -48,6 +43,6 @@ class EjeaInvalidatingCacheEntrySpec extends EngineJobExecutionActorSpec with Ca
     }
   }
 
-  def standardResponsePendingData(hit: Option[CacheHit]) = ResponsePendingData(helper.backendJobDescriptor, helper.bjeaProps, None, hit)
-  def ejeaInvalidatingCacheEntryState(hit: Option[CacheHit], restarting: Boolean = false) = helper.buildEJEA(restarting = restarting, callCachingMode = CallCachingActivity(ReadCache)).setStateInline(data = standardResponsePendingData(hit))
+  def standardResponsePendingData(ejha: Option[ActorRef]) = ResponsePendingData(helper.backendJobDescriptor, helper.bjeaProps, None, ejha)
+  def ejeaInvalidatingCacheEntryState(ejha: Option[ActorRef], restarting: Boolean = false) = helper.buildEJEA(restarting = restarting, callCachingMode = CallCachingActivity(ReadCache)).setStateInline(state = InvalidatingCacheEntry, data = standardResponsePendingData(ejha))
 }
diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaPreparingJobSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaPreparingJobSpec.scala
index c4b14eec0..8deb32b2e 100644
--- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaPreparingJobSpec.scala
+++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaPreparingJobSpec.scala
@@ -1,7 +1,8 @@
 package cromwell.engine.workflow.lifecycle.execution.ejea
 
+import cromwell.backend.BackendJobDescriptor
 import cromwell.backend.BackendJobExecutionActor.JobFailedNonRetryableResponse
-import cromwell.core.callcaching.CallCachingMode
+import cromwell.core.callcaching.{CallCachingMode, DockerWithHash}
 import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor._
 import cromwell.engine.workflow.lifecycle.execution.ejea.EngineJobExecutionActorSpec._
 import cromwell.engine.workflow.lifecycle.execution.preparation.CallPreparation.{BackendJobPreparationSucceeded, CallPreparationFailed}
@@ -15,18 +16,26 @@ class EjeaPreparingJobSpec extends EngineJobExecutionActorSpec with CanExpectHas
 
     CallCachingModes foreach { mode =>
       if (mode.readFromCache) {
-        s"Start checking for a cache hit when job preparation succeeds ($mode)" in {
+        s"Start checking for a cache hit when job preparation succeeds and a docker hash is available ($mode)" in {
+          val jobDescriptor = helper.backendJobDescriptor.copy(maybeCallCachingEligible = DockerWithHash("hello"))
           ejea = ejeaInPreparingState(mode)
-          ejea ! jobPrepSuccessResponse
-          expectHashingActorInitialization(mode)
+          ejea ! jobPrepSuccessResponse(jobDescriptor)
+          expectHashingActorInitialization(mode, jobDescriptor)
           ejea.stateName should be(CheckingCallCache)
+          ejea.stateData should be(ResponsePendingData(jobDescriptor, helper.bjeaProps, None, Option(helper.ejhaProbe.ref)))
+        }
+
+        s"Not check for a cache hit when job preparation succeeds and no docker hash is available ($mode)" in {
+          ejea = ejeaInPreparingState(mode)
+          ejea ! jobPrepSuccessResponse(helper.backendJobDescriptor)
+          ejea.stateName should be(RunningJob)
           ejea.stateData should be(ResponsePendingData(helper.backendJobDescriptor, helper.bjeaProps, None))
         }
       } else {
         RestartOrExecuteCommandTuples foreach { case RestartOrExecuteCommandTuple(operationName, restarting, expectedMessage) =>
           s"Send BJEA '$operationName' when job preparation succeeds ($mode)" in {
             ejea = ejeaInPreparingState(mode = mode, restarting = restarting)
-            ejea ! jobPrepSuccessResponse
+            ejea ! jobPrepSuccessResponse(helper.backendJobDescriptor)
             helper.bjeaProbe.expectMsg(awaitTimeout, "job preparation", expectedMessage)
             ejea.stateName should be(RunningJob)
             ejea.stateData should be(ResponsePendingData(helper.backendJobDescriptor, helper.bjeaProps, None))
@@ -45,7 +54,7 @@ class EjeaPreparingJobSpec extends EngineJobExecutionActorSpec with CanExpectHas
     }
   }
 
-  def jobPrepSuccessResponse = BackendJobPreparationSucceeded(helper.backendJobDescriptor, helper.bjeaProps)
+  def jobPrepSuccessResponse(jobDescriptor: BackendJobDescriptor) = BackendJobPreparationSucceeded(jobDescriptor, helper.bjeaProps)
 
   def ejeaInPreparingState(mode: CallCachingMode, restarting: Boolean = false) = helper.buildEJEA(restarting = restarting, callCachingMode = mode).setStateInline(state = PreparingJob, data = NoData)
 
diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaRunningJobSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaRunningJobSpec.scala
index 5d2ccd4ce..2239a2843 100644
--- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaRunningJobSpec.scala
+++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaRunningJobSpec.scala
@@ -18,7 +18,7 @@ class EjeaRunningJobSpec extends EngineJobExecutionActorSpec with Eventually wit
   val hashError = HashError(new Exception("ARGH!!!"))
 
   "A 'RunningJob' EJEA" should {
-    CallCachingModes foreach { case mode =>
+    CallCachingModes foreach { mode =>
       val andMaybeCallCacheHashes = if (mode.writeToCache) "then CallCacheHashes " else ""
       s"Handle receiving a SucceededResponse ${andMaybeCallCacheHashes}correctly in $mode mode" in {
         ejea = ejeaInRunningState(mode)
@@ -53,16 +53,16 @@ class EjeaRunningJobSpec extends EngineJobExecutionActorSpec with Eventually wit
           eventually { ejea.stateData should be(SucceededResponseData(successResponse, None)) }
           ejea.stateName should be(RunningJob)
           ejea ! hashError
-          expectJobStoreWrite(SucceededResponseData(successResponse, Some(Failure(hashError.t))))
+          expectJobStoreWrite(SucceededResponseData(successResponse, Some(Failure(hashError.reason))))
         }
 
         s"Handle receiving HashError then SuccessResponse correctly in $mode mode" in {
           ejea = ejeaInRunningState(mode)
           ejea ! hashError
-          eventually { ejea.stateData should be(ResponsePendingData(helper.backendJobDescriptor, helper.bjeaProps, Some(Failure(hashError.t)))) }
+          eventually { ejea.stateData should be(ResponsePendingData(helper.backendJobDescriptor, helper.bjeaProps, Some(Failure(hashError.reason)))) }
           ejea.stateName should be(RunningJob)
           ejea ! successResponse
-          expectJobStoreWrite(SucceededResponseData(successResponse, Some(Failure(hashError.t))))
+          expectJobStoreWrite(SucceededResponseData(successResponse, Some(Failure(hashError.reason))))
         }
       }
     }
@@ -94,7 +94,6 @@ class EjeaRunningJobSpec extends EngineJobExecutionActorSpec with Eventually wit
       ejea ! abortedResponse
 
       helper.replyToProbe.expectMsg(max = awaitTimeout, hint = "parent wants the response", abortedResponse)
-      helper.callCacheWriteActorCreations should be(NothingYet)
 
       helper.deathwatch.expectTerminated(ejea, awaitTimeout)
       // Make sure nothing was sent to the JobStore or CacheResultSaver in the meanwhile:
diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaUpdatingJobStoreSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaUpdatingJobStoreSpec.scala
index 73a224a00..41e3f548e 100644
--- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaUpdatingJobStoreSpec.scala
+++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EjeaUpdatingJobStoreSpec.scala
@@ -17,7 +17,7 @@ class EjeaUpdatingJobStoreSpec extends EngineJobExecutionActorSpec with HasJobSu
       ("SucceededResponse", successResponse _, true),
       ("FailedRetryableResponse", failureRetryableResponse _, true),
       ("FailedNonRetryableResponse", failureNonRetryableResponse _, false)
-    ) foreach { case (name, responseMaker, retryable) =>
+    ) foreach { case (name, responseMaker, retryable @ _) =>
       s"Forward a saved $name response on and shut down, if the JobStore write is successful" in {
         val response = responseMaker.apply
         ejea = ejeaInUpdatingJobStoreState(response)
diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EngineJobExecutionActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EngineJobExecutionActorSpec.scala
index 105776781..ac2a1143b 100644
--- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EngineJobExecutionActorSpec.scala
+++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EngineJobExecutionActorSpec.scala
@@ -3,8 +3,7 @@ package cromwell.engine.workflow.lifecycle.execution.ejea
 import akka.actor.Actor
 import akka.testkit.TestFSMRef
 import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor._
-import cromwell.jobstore.{Pending => _}
-import cromwell.CromwellTestKitSpec
+import cromwell.CromwellTestKitWordSpec
 import cromwell.backend.BackendJobExecutionActor
 import cromwell.backend.BackendJobExecutionActor.BackendJobExecutionActorCommand
 import cromwell.core.callcaching._
@@ -15,7 +14,7 @@ import scala.concurrent.duration._
 import scala.language.postfixOps
 
 
-trait EngineJobExecutionActorSpec extends CromwellTestKitSpec
+trait EngineJobExecutionActorSpec extends CromwellTestKitWordSpec
   with Matchers with Mockito with BeforeAndAfterAll with BeforeAndAfter {
 
   // If we WANT something to happen, make sure it happens within this window:
@@ -40,8 +39,7 @@ trait EngineJobExecutionActorSpec extends CromwellTestKitSpec
     List(
       ("FetchCachedResultsActor", helper.fetchCachedResultsActorCreations),
       ("JobHashingActor", helper.jobHashingInitializations),
-      ("CallCacheInvalidateActor", helper.invalidateCacheActorCreations),
-      ("CallCacheWriteActor", helper.callCacheWriteActorCreations)) foreach {
+      ("CallCacheInvalidateActor", helper.invalidateCacheActorCreations)) foreach {
       case (name, GotTooMany(list)) => fail(s"Too many $name creations (${list.size})")
       case _ => // Fine.
     }
@@ -56,11 +54,6 @@ trait EngineJobExecutionActorSpec extends CromwellTestKitSpec
     * var callCacheWriteActorCreations: ExpectOne[(CallCacheHashes, SucceededResponse)] = NothingYet
     */
 
-  override def afterAll(): Unit = {
-    system.terminate()
-    super.afterAll()
-  }
-
   // Some helper lists
   val CallCachingModes = List(CallCachingOff, CallCachingActivity(ReadCache), CallCachingActivity(WriteCache), CallCachingActivity(ReadAndWriteCache))
   case class RestartOrExecuteCommandTuple(operationName: String, restarting: Boolean, expectedMessageToBjea: BackendJobExecutionActorCommand)
diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EngineJobExecutionActorSpecUtil.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EngineJobExecutionActorSpecUtil.scala
index b6ddb5601..88163d352 100644
--- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EngineJobExecutionActorSpecUtil.scala
+++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/EngineJobExecutionActorSpecUtil.scala
@@ -1,15 +1,16 @@
 package cromwell.engine.workflow.lifecycle.execution.ejea
 
+import cromwell.backend.BackendJobDescriptor
 import cromwell.backend.BackendJobExecutionActor.{AbortedResponse, JobFailedNonRetryableResponse, JobFailedRetryableResponse, JobSucceededResponse}
 import cromwell.core.JobOutput
 import cromwell.core.callcaching._
 import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor.{EJEAData, SucceededResponseData, UpdatingCallCache, UpdatingJobStore}
-import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.CallCacheHashes
 import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCachingEntryId
+import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.{CallCacheHashes, FileHashes}
 import cromwell.jobstore.JobStoreActor.RegisterJobCompleted
 import cromwell.jobstore.{JobResultSuccess, JobStoreKey}
 import org.scalatest.concurrent.Eventually
-import wdl4s.values.{WdlInteger, WdlString}
+import wdl4s.wdl.values.{WdlInteger, WdlString}
 
 import scala.util.Success
 
@@ -26,12 +27,6 @@ private[ejea] trait CanExpectCacheWrites extends Eventually { self: EngineJobExe
   def expectCacheWrite(expectedResponse: JobSucceededResponse, expectedCallCacheHashes: CallCacheHashes): Unit = {
     eventually { ejea.stateName should be(UpdatingCallCache) }
     ejea.stateData should be(SucceededResponseData(expectedResponse, Some(Success(expectedCallCacheHashes))))
-    helper.callCacheWriteActorCreations match {
-      case GotOne(creation) =>
-        creation._1 should be(expectedCallCacheHashes)
-        creation._2 should be(expectedResponse)
-      case _ => fail("Expected exactly one cache write actor creation.")
-    }
     ()
   }
 }
@@ -52,10 +47,10 @@ private[ejea] trait CanExpectJobStoreWrites extends CanValidateJobStoreKey { sel
 }
 
 private[ejea] trait CanExpectHashingInitialization extends Eventually { self: EngineJobExecutionActorSpec =>
-  def expectHashingActorInitialization(mode: CallCachingMode): Unit = {
+  def expectHashingActorInitialization(mode: CallCachingMode, jobDescriptor: BackendJobDescriptor): Unit = {
     eventually { helper.jobHashingInitializations.hasExactlyOne should be(true) }
     helper.jobHashingInitializations.checkIt { initialization =>
-      initialization._1 should be(helper.backendJobDescriptor)
+      initialization._1 should be(jobDescriptor)
       initialization._2 should be(mode)
     }
   }
@@ -83,10 +78,17 @@ private[ejea] trait CanExpectCacheInvalidation extends Eventually { self: Engine
 private[ejea] trait HasJobSuccessResponse { self: EngineJobExecutionActorSpec =>
   val successRc = Option(171)
   val successOutputs = Map("a" -> JobOutput(WdlInteger(3)), "b" -> JobOutput(WdlString("bee")))
-  def successResponse = JobSucceededResponse(helper.jobDescriptorKey, successRc, successOutputs, None, Seq.empty)
+  def successResponse = JobSucceededResponse(helper.jobDescriptorKey, successRc, successOutputs, None, Seq.empty, None)
 }
 private[ejea] object HasJobSuccessResponse {
-  val SuccessfulCallCacheHashes = CallCacheHashes(Set(HashResult(HashKey("whatever you want"), HashValue("whatever you need"))))
+  val SuccessfulCallCacheHashes = CallCacheHashes(
+    Set(HashResult(HashKey("whatever you want"), HashValue("whatever you need"))),
+    "initialHash",
+    Option(FileHashes(
+      Set(HashResult(HashKey("whatever file you want"), HashValue("whatever file you need"))),
+      "fileHash"
+    ))
+  )
 }
 
 private[ejea] trait HasJobFailureResponses { self: EngineJobExecutionActorSpec =>
diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala
index 1ceef04c4..29278c556 100644
--- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala
+++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ejea/PerTestHelper.scala
@@ -4,24 +4,25 @@ import java.util.UUID
 
 import akka.actor.{ActorRef, ActorSystem, Props}
 import akka.testkit.{TestFSMRef, TestProbe}
-import cromwell.backend.BackendJobExecutionActor.JobSucceededResponse
 import cromwell.backend._
+import cromwell.backend.standard.callcaching._
 import cromwell.core.JobExecutionToken.JobExecutionTokenType
-import cromwell.core.callcaching.{CallCachingActivity, CallCachingEligible, CallCachingMode, CallCachingOff}
+import cromwell.core.callcaching._
 import cromwell.core.{CallOutputs, JobExecutionToken, WorkflowId}
 import cromwell.engine.EngineWorkflowDescriptor
 import cromwell.engine.workflow.lifecycle.execution.EngineJobExecutionActor.{EJEAData, EngineJobExecutionActorState}
 import cromwell.engine.workflow.lifecycle.execution.callcaching.CallCachingEntryId
-import cromwell.engine.workflow.lifecycle.execution.callcaching.EngineJobHashingActor.CallCacheHashes
 import cromwell.engine.workflow.lifecycle.execution.ejea.EngineJobExecutionActorSpec._
 import cromwell.engine.workflow.lifecycle.execution.{EngineJobExecutionActor, WorkflowExecutionActorData}
 import cromwell.engine.workflow.mocks.{DeclarationMock, TaskMock, WdlExpressionMock}
 import cromwell.util.AkkaTestUtil._
 import org.specs2.mock.Mockito
-import wdl4s._
-import wdl4s.expression.{NoFunctions, WdlStandardLibraryFunctions}
+import wdl4s.wdl._
+import wdl4s.wdl.expression.{NoFunctions, WdlStandardLibraryFunctions}
 import wdl4s.parser.WdlParser.Ast
-import wdl4s.types.{WdlIntegerType, WdlStringType}
+import wdl4s.wdl.types.{WdlIntegerType, WdlStringType}
+
+import scala.util.Success
 
 
 private[ejea] class PerTestHelper(implicit val system: ActorSystem) extends Mockito with TaskMock with WdlExpressionMock with DeclarationMock {
@@ -41,23 +42,22 @@ private[ejea] class PerTestHelper(implicit val system: ActorSystem) extends Mock
     outputs = Seq(("outString", WdlStringType, mockStringExpression("hello")))
   )
 
-  val workflow = new Workflow(
+  val workflow = new WdlWorkflow(
     unqualifiedName = workflowName,
     workflowOutputWildcards = Seq.empty,
     wdlSyntaxErrorFormatter = mock[WdlSyntaxErrorFormatter],
     meta = Map.empty,
     parameterMeta = Map.empty,
     ast = mock[Ast])
-  val call: TaskCall = TaskCall(None, task, Map.empty, mock[Ast])
+  val call: WdlTaskCall = WdlTaskCall(None, task, Map.empty, mock[Ast])
   call.parent_=(workflow)
   val jobDescriptorKey = BackendJobDescriptorKey(call, jobIndex, jobAttempt)
 
   val backendWorkflowDescriptor = BackendWorkflowDescriptor(workflowId, null, null, null, null)
-  val backendJobDescriptor = BackendJobDescriptor(backendWorkflowDescriptor, jobDescriptorKey, runtimeAttributes = Map.empty, inputDeclarations = Map.empty, CallCachingEligible)
+  val backendJobDescriptor = BackendJobDescriptor(backendWorkflowDescriptor, jobDescriptorKey, runtimeAttributes = Map.empty, inputDeclarations = Map.empty, FloatingDockerTagWithoutHash("ubuntu:latest"), Map.empty)
 
   var fetchCachedResultsActorCreations: ExpectOne[(CallCachingEntryId, Seq[TaskOutput])] = NothingYet
   var jobHashingInitializations: ExpectOne[(BackendJobDescriptor, CallCachingActivity)] = NothingYet
-  var callCacheWriteActorCreations: ExpectOne[(CallCacheHashes, JobSucceededResponse)] = NothingYet
   var invalidateCacheActorCreations: ExpectOne[CallCachingEntryId] = NothingYet
 
   val deathwatch = TestProbe()
@@ -66,35 +66,55 @@ private[ejea] class PerTestHelper(implicit val system: ActorSystem) extends Mock
   val replyToProbe = TestProbe()
   val parentProbe = TestProbe()
   val serviceRegistryProbe = TestProbe()
+  val ioActorProbe = TestProbe()
   val jobStoreProbe = TestProbe()
   val callCacheReadActorProbe = TestProbe()
+  val callCacheWriteActorProbe = TestProbe()
   val dockerHashActorProbe = TestProbe()
   val callCacheHitCopyingProbe = TestProbe()
   val jobPreparationProbe = TestProbe()
   val jobTokenDispenserProbe = TestProbe()
+  val ejhaProbe = TestProbe()
 
   def buildFactory() = new BackendLifecycleActorFactory {
 
     override def jobExecutionActorProps(jobDescriptor: BackendJobDescriptor,
                                         initializationData: Option[BackendInitializationData],
                                         serviceRegistryActor: ActorRef,
+                                        ioActor: ActorRef,
                                         backendSingletonActor: Option[ActorRef]): Props = bjeaProps
 
-    override def cacheHitCopyingActorProps: Option[(BackendJobDescriptor, Option[BackendInitializationData], ActorRef) => Props] = Option((_, _, _) => callCacheHitCopyingProbe.props)
+    override def cacheHitCopyingActorProps: Option[(BackendJobDescriptor, Option[BackendInitializationData], ActorRef, ActorRef) => Props] = Option((_, _, _, _) => callCacheHitCopyingProbe.props)
 
     override def expressionLanguageFunctions(workflowDescriptor: BackendWorkflowDescriptor, jobKey: BackendJobDescriptorKey, initializationData: Option[BackendInitializationData]): WdlStandardLibraryFunctions = {
       NoFunctions
     }
 
+    override def fileHashingActorProps:
+    Option[(BackendJobDescriptor, Option[BackendInitializationData], ActorRef, ActorRef) => Props] = {
+      Option(fileHashingActorInner(classOf[DefaultStandardFileHashingActor]))
+    }
+
+    def fileHashingActorInner(standardFileHashingActor: Class[_ <: StandardFileHashingActor])
+                             (jobDescriptor: BackendJobDescriptor,
+                              initializationDataOption: Option[BackendInitializationData],
+                              serviceRegistryActor: ActorRef,
+                              ioActor: ActorRef): Props = {
+      Props.empty
+    }
+
     // These two factory methods should never be called from EJEA or any of its descendants:
     override def workflowFinalizationActorProps(workflowDescriptor: BackendWorkflowDescriptor,
-                                                calls: Set[TaskCall],
+                                                ioActor: ActorRef,
+                                                calls: Set[WdlTaskCall],
                                                 jobExecutionMap: JobExecutionMap,
                                                 workflowOutputs: CallOutputs,
                                                 initializationData: Option[BackendInitializationData]): Option[Props] = throw new UnsupportedOperationException("Unexpected finalization actor creation!")
     override def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor,
-                                                  calls: Set[TaskCall],
-                                                  serviceRegistryActor: ActorRef): Option[Props] = throw new UnsupportedOperationException("Unexpected finalization actor creation!")
+                                                  ioActor: ActorRef,
+                                                  calls: Set[WdlTaskCall],
+                                                  serviceRegistryActor: ActorRef,
+                                                  restarting: Boolean): Option[Props] = throw new UnsupportedOperationException("Unexpected finalization actor creation!")
   }
 
   def buildEJEA(restarting: Boolean = true,
@@ -114,8 +134,10 @@ private[ejea] class PerTestHelper(implicit val system: ActorSystem) extends Mock
       initializationData = None,
       restarting = restarting,
       serviceRegistryActor = serviceRegistryProbe.ref,
+      ioActor = ioActorProbe.ref,
       jobStoreActor = jobStoreProbe.ref,
       callCacheReadActor = callCacheReadActorProbe.ref,
+      callCacheWriteActor = callCacheWriteActorProbe.ref,
       dockerHashActor = dockerHashActorProbe.ref,
       jobTokenDispenserActor = jobTokenDispenserProbe.ref,
       backendName = "NOT USED",
@@ -136,16 +158,21 @@ private[ejea] class MockEjea(helper: PerTestHelper,
                              initializationData: Option[BackendInitializationData],
                              restarting: Boolean,
                              serviceRegistryActor: ActorRef,
+                             ioActor: ActorRef,
                              jobStoreActor: ActorRef,
                              callCacheReadActor: ActorRef,
+                             callCacheWriteActor: ActorRef,
                              dockerHashActor: ActorRef,
                              jobTokenDispenserActor: ActorRef,
                              backendName: String,
-                             callCachingMode: CallCachingMode) extends EngineJobExecutionActor(replyTo, jobDescriptorKey, executionData, factory, initializationData, restarting, serviceRegistryActor, jobStoreActor, callCacheReadActor, dockerHashActor, jobTokenDispenserActor, None, backendName, callCachingMode) {
+                             callCachingMode: CallCachingMode) extends EngineJobExecutionActor(replyTo, jobDescriptorKey, executionData, factory, initializationData, restarting, serviceRegistryActor, ioActor, jobStoreActor, callCacheReadActor, callCacheWriteActor, dockerHashActor, jobTokenDispenserActor, None, backendName, callCachingMode) {
 
+  implicit val system = context.system
   override def makeFetchCachedResultsActor(cacheId: CallCachingEntryId, taskOutputs: Seq[TaskOutput]) = helper.fetchCachedResultsActorCreations = helper.fetchCachedResultsActorCreations.foundOne((cacheId, taskOutputs))
-  override def initializeJobHashing(jobDescriptor: BackendJobDescriptor, activity: CallCachingActivity) = helper.jobHashingInitializations = helper.jobHashingInitializations.foundOne((jobDescriptor, activity))
-  override def createSaveCacheResultsActor(hashes: CallCacheHashes, success: JobSucceededResponse) = helper.callCacheWriteActorCreations = helper.callCacheWriteActorCreations.foundOne((hashes, success))
+  override def initializeJobHashing(jobDescriptor: BackendJobDescriptor, activity: CallCachingActivity, callCachingEligible: CallCachingEligible) = {
+    helper.jobHashingInitializations = helper.jobHashingInitializations.foundOne((jobDescriptor, activity))
+    Success(helper.ejhaProbe.ref)
+  }
   override def invalidateCacheHit(cacheId: CallCachingEntryId): Unit = { helper.invalidateCacheActorCreations = helper.invalidateCacheActorCreations.foundOne(cacheId) }
   override def createJobPreparationActor(jobPrepProps: Props, name: String) = jobPreparationProbe.ref
 }
diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/preparation/JobPreparationActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/preparation/JobPreparationActorSpec.scala
index 837f87e96..cea38cd81 100644
--- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/preparation/JobPreparationActorSpec.scala
+++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/preparation/JobPreparationActorSpec.scala
@@ -1,164 +1,142 @@
 package cromwell.engine.workflow.lifecycle.execution.preparation
 
-import akka.testkit.{ImplicitSender, TestActorRef, TestProbe}
-import cromwell.core.callcaching.{CallCachingEligible, CallCachingIneligible}
-import cromwell.core.callcaching.docker.DockerHashActor.{DockerHashBackPressure, DockerHashFailedResponse, DockerHashResponseSuccess}
-import cromwell.core.callcaching.docker.{DockerHashRequest, DockerHashResult, DockerImageIdentifier, DockerImageIdentifierWithoutHash}
+import akka.testkit.{ImplicitSender, TestActorRef}
+import cromwell.core.callcaching.{DockerWithHash, FloatingDockerTagWithoutHash}
 import cromwell.core.{LocallyQualifiedName, TestKitSuite}
+import cromwell.docker.DockerHashActor.DockerHashSuccessResponse
+import cromwell.docker.{DockerHashRequest, DockerHashResult, DockerImageIdentifier, DockerImageIdentifierWithoutHash}
+import cromwell.engine.workflow.WorkflowDockerLookupActor.WorkflowDockerLookupFailure
 import cromwell.engine.workflow.lifecycle.execution.preparation.CallPreparation.{BackendJobPreparationSucceeded, CallPreparationFailed, Start}
-import org.scalatest.{FlatSpecLike, Matchers}
-import wdl4s.Declaration
-import wdl4s.values.{WdlString, WdlValue}
+import cromwell.services.keyvalue.KeyValueServiceActor.{KvGet, KvKeyLookupFailed, KvPair}
+import org.scalatest.{BeforeAndAfter, FlatSpecLike, Matchers}
+import org.specs2.mock.Mockito
+import wdl4s.wdl.Declaration
+import wdl4s.wdl.values.{WdlString, WdlValue}
 
-import scala.util.{Failure, Success}
 import scala.concurrent.duration._
 import scala.language.postfixOps
+import scala.util.{Failure, Success}
 
-class JobPreparationActorSpec extends TestKitSuite with FlatSpecLike with Matchers with ImplicitSender {
+class JobPreparationActorSpec extends TestKitSuite("JobPrepActorSpecSystem") with FlatSpecLike with Matchers with ImplicitSender with BeforeAndAfter with Mockito {
 
   behavior of "JobPreparationActor"
-  
-  val helper = new JobPreparationTestHelper()
-  val inputs = Map.empty[Declaration, WdlValue]
-  
+
+  // Generated fresh for each test case in 'before'
+  var helper: JobPreparationTestHelper = _
+  var inputs: Map[Declaration, WdlValue] = _
+
+  before {
+    helper = new JobPreparationTestHelper()
+    inputs = Map.empty[Declaration, WdlValue]
+  }
+
   it should "fail preparation if it can't evaluate inputs or runtime attributes" in {
     val exception = new Exception("Failed to prepare inputs/attributes - part of test flow")
     val failure = Failure(exception)
     val expectedResponse = CallPreparationFailed(helper.jobKey, exception)
-    val actor = TestActorRef(helper.buildJobPreparationMock(null, null, null, null, failure), self)
+    val actor = TestActorRef(helper.buildTestJobPreparationActor(null, null, null, failure, List.empty), self)
     actor ! Start
     expectMsg(expectedResponse)
+    helper.workflowDockerLookupActor.expectNoMsg(100 millis)
   }
 
   it should "prepare successfully a job without docker attribute" in {
     val attributes = Map.empty[LocallyQualifiedName, WdlValue]
     val inputsAndAttributes = Success((inputs, attributes))
-    val dockerHashingActor = TestProbe()
-    val actor = TestActorRef(helper.buildJobPreparationMock(null, null, null, dockerHashingActor.ref, inputsAndAttributes), self)
+    val actor = TestActorRef(helper.buildTestJobPreparationActor(null, null, null, inputsAndAttributes, List.empty), self)
     actor ! Start
     expectMsgPF(5 seconds) {
       case success: BackendJobPreparationSucceeded =>
-        success.jobDescriptor.callCachingEligibility shouldBe CallCachingEligible
+        success.jobDescriptor.maybeCallCachingEligible.dockerHash shouldBe None
     }
-    dockerHashingActor.expectNoMsg(1 second)
+    helper.workflowDockerLookupActor.expectNoMsg(1 second)
   }
 
   it should "not ask for the docker hash if the attribute already contains a hash" in {
     val dockerValue = "ubuntu@sha256:71cd81252a3563a03ad8daee81047b62ab5d892ebbfbf71cf53415f29c130950"
-    val attributes = Map (
+    val attributes = Map(
       "docker" -> WdlString(dockerValue)
     )
     val inputsAndAttributes = Success((inputs, attributes))
-    val dockerHashingActor = TestProbe()
-    val actor = TestActorRef(helper.buildJobPreparationMock(null, null, null, dockerHashingActor.ref, inputsAndAttributes), self)
+    val actor = TestActorRef(helper.buildTestJobPreparationActor(null, null, null, inputsAndAttributes, List.empty), self)
     actor ! Start
     expectMsgPF(5 seconds) {
       case success: BackendJobPreparationSucceeded =>
         success.jobDescriptor.runtimeAttributes("docker").valueString shouldBe dockerValue
-        success.jobDescriptor.callCachingEligibility shouldBe CallCachingEligible
+        success.jobDescriptor.maybeCallCachingEligible shouldBe DockerWithHash("ubuntu@sha256:71cd81252a3563a03ad8daee81047b62ab5d892ebbfbf71cf53415f29c130950")
     }
-    dockerHashingActor.expectNoMsg(1 second)
+    helper.workflowDockerLookupActor.expectNoMsg(1 second)
   }
 
-  it should "replace the tag with a docker hash if the attribute has a floating tag, and make it ineligible for caching" in {
+  it should "lookup any requested key/value prefetches before (not) performing a docker hash lookup" in {
     val dockerValue = "ubuntu:latest"
     val attributes = Map (
       "docker" -> WdlString(dockerValue)
     )
     val hashResult = DockerHashResult("sha256", "71cd81252a3563a03ad8daee81047b62ab5d892ebbfbf71cf53415f29c130950")
     val inputsAndAttributes = Success((inputs, attributes))
-    val finalValue = "library/ubuntu@sha256:71cd81252a3563a03ad8daee81047b62ab5d892ebbfbf71cf53415f29c130950"
-    val dockerHashingActor = TestProbe()
-    val actor = TestActorRef(helper.buildJobPreparationMock(1 minute, 1 minutes, List.empty, dockerHashingActor.ref, inputsAndAttributes), self)
+    val prefetchedKey1 = "hello"
+    val prefetchedVal1 = KvPair(helper.scopedKeyMaker(prefetchedKey1), Some("world"))
+    val prefetchedKey2 = "bonjour"
+    val prefetchedVal2 = KvKeyLookupFailed(KvGet(helper.scopedKeyMaker(prefetchedKey2)))
+    val prefetchedValues = Map(prefetchedKey1 -> prefetchedVal1, prefetchedKey2 -> prefetchedVal2)
+    var keysToPrefetch = List(prefetchedKey1, prefetchedKey2)
+    val actor = TestActorRef(helper.buildTestJobPreparationActor(1 minute, 1 minutes, List.empty, inputsAndAttributes, List(prefetchedKey1, prefetchedKey2)), self)
     actor ! Start
-    dockerHashingActor.expectMsgClass(classOf[DockerHashRequest])
-    dockerHashingActor.reply(DockerHashResponseSuccess(hashResult))
+
+    def respondFromKv() = {
+      helper.serviceRegistryProbe.expectMsgPF(max = 100 milliseconds) {
+        case KvGet(k) if keysToPrefetch.contains(k.key) =>
+          actor.tell(msg = prefetchedValues(k.key), sender = helper.serviceRegistryProbe.ref)
+          keysToPrefetch = keysToPrefetch diff List(k.key)
+      }
+    }
+    respondFromKv()
+    helper.workflowDockerLookupActor.expectNoMsg(max = 100 milliseconds)
+    respondFromKv()
+
+    val req = helper.workflowDockerLookupActor.expectMsgClass(classOf[DockerHashRequest])
+    helper.workflowDockerLookupActor.reply(DockerHashSuccessResponse(hashResult, req))
     expectMsgPF(5 seconds) {
       case success: BackendJobPreparationSucceeded =>
-        success.jobDescriptor.runtimeAttributes("docker").valueString shouldBe finalValue
-        success.jobDescriptor.callCachingEligibility.isInstanceOf[CallCachingIneligible] shouldBe true
-        success.jobDescriptor.callCachingEligibility.asInstanceOf[CallCachingIneligible].message shouldBe
-          s"""You are using a floating docker tag in this task. Cromwell does not consider tasks with floating tags to be eligible for call caching.
-              |If you want this task to be eligible for call caching in the future, use a docker runtime attribute with a digest instead.
-              |This is the exact docker image that was used for this job: $finalValue
-              |You can replace the docker runtime attribute in your task with the above value to make this task eligible for call caching.""".stripMargin
+        success.jobDescriptor.prefetchedKvStoreEntries should be(Map(prefetchedKey1 -> prefetchedVal1, prefetchedKey2 -> prefetchedVal2))
     }
   }
 
-  it should "make a job ineligible for caching if it can't get the docker hash" in {
+  it should "leave the docker attribute as is and provide a DockerWithHash value" in {
     val dockerValue = "ubuntu:latest"
-    val dockerId = DockerImageIdentifier.fromString(dockerValue).get.asInstanceOf[DockerImageIdentifierWithoutHash]
     val attributes = Map (
       "docker" -> WdlString(dockerValue)
     )
+    val hashResult = DockerHashResult("sha256", "71cd81252a3563a03ad8daee81047b62ab5d892ebbfbf71cf53415f29c130950")
     val inputsAndAttributes = Success((inputs, attributes))
-    val dockerHashingActor = TestProbe()
-    val actor = TestActorRef(helper.buildJobPreparationMock(1 minute, 1 minutes, List.empty, dockerHashingActor.ref, inputsAndAttributes), self)
+    val finalValue = "ubuntu@sha256:71cd81252a3563a03ad8daee81047b62ab5d892ebbfbf71cf53415f29c130950"
+    val actor = TestActorRef(helper.buildTestJobPreparationActor(1 minute, 1 minutes, List.empty, inputsAndAttributes, List.empty), self)
     actor ! Start
-    dockerHashingActor.expectMsgClass(classOf[DockerHashRequest])
-    dockerHashingActor.reply(DockerHashFailedResponse(new Exception("Failed to get docker hash - part of test flow"), dockerId))
+    helper.workflowDockerLookupActor.expectMsgClass(classOf[DockerHashRequest])
+    helper.workflowDockerLookupActor.reply(DockerHashSuccessResponse(hashResult, mock[DockerHashRequest]))
     expectMsgPF(5 seconds) {
       case success: BackendJobPreparationSucceeded =>
         success.jobDescriptor.runtimeAttributes("docker").valueString shouldBe dockerValue
-        success.jobDescriptor.callCachingEligibility.isInstanceOf[CallCachingIneligible] shouldBe true
-        success.jobDescriptor.callCachingEligibility.asInstanceOf[CallCachingIneligible].message shouldBe
-          s"""You are using a floating docker tag in this task. Cromwell does not consider tasks with floating tags to be eligible for call caching.
-              |If you want this task to be eligible for call caching in the future, use a docker runtime attribute with a digest instead.
-              |Cromwell attempted to retrieve the current hash for this docker image but failed.
-              |This is not necessarily a cause for concern as Cromwell is currently only able to retrieve hashes for Dockerhub and GCR images.
-              |The job will be dispatched to the appropriate backend that will attempt to run it.""".stripMargin
+        success.jobDescriptor.maybeCallCachingEligible shouldBe DockerWithHash(finalValue)
     }
   }
-  
-  it should "wait and resubmit the docker request when it gets a backpressure message" in {
-    val dockerValue = "ubuntu:latest"
-    val dockerId = DockerImageIdentifier.fromString(dockerValue).get.asInstanceOf[DockerImageIdentifierWithoutHash]
-    val attributes = Map (
-      "docker" -> WdlString(dockerValue)
-    )
-    val inputsAndAttributes = Success((inputs, attributes))
-    val dockerHashingActor = TestProbe()
-    val backpresureWaitTime = 2 seconds
-    val actor = TestActorRef(helper.buildJobPreparationMock(backpresureWaitTime, 1 minute, List.empty, dockerHashingActor.ref, inputsAndAttributes), self)
-    val request = DockerHashRequest(dockerId)
-    actor ! Start
-    dockerHashingActor.expectMsg(request)
-    dockerHashingActor.reply(DockerHashBackPressure(request))
-    // Give a couple of seconds of margin to account for test latency etc...
-    dockerHashingActor.expectMsg(backpresureWaitTime.+(2 seconds), request)
-  }
 
-  it should "automatically re submit the docker request if no answer and finally give up" in {
+  it should "not provide a DockerWithHash value if it can't get the docker hash" in {
     val dockerValue = "ubuntu:latest"
-    val dockerId = DockerImageIdentifier.fromString(dockerValue).get.asInstanceOf[DockerImageIdentifierWithoutHash]
+    val request = DockerHashRequest(DockerImageIdentifier.fromString(dockerValue).get.asInstanceOf[DockerImageIdentifierWithoutHash])
     val attributes = Map (
       "docker" -> WdlString(dockerValue)
     )
     val inputsAndAttributes = Success((inputs, attributes))
-    val dockerHashingActor = TestProbe()
-    val requestTimeout = 1 second
-    val actor = TestActorRef(helper.buildJobPreparationMock(1 minute, requestTimeout, List.empty, dockerHashingActor.ref, inputsAndAttributes), self)
-    val request = DockerHashRequest(dockerId)
+    val actor = TestActorRef(helper.buildTestJobPreparationActor(1 minute, 1 minutes, List.empty, inputsAndAttributes, List.empty), self)
     actor ! Start
-    
-    within(10 seconds) {
-      dockerHashingActor.expectMsg(request)
-      dockerHashingActor.expectMsg(request)
-      dockerHashingActor.expectMsg(request)
-      dockerHashingActor.expectMsg(request)
-    }
-
+    helper.workflowDockerLookupActor.expectMsgClass(classOf[DockerHashRequest])
+    helper.workflowDockerLookupActor.reply(WorkflowDockerLookupFailure(new Exception("Failed to get docker hash - part of test flow"), request))
     expectMsgPF(5 seconds) {
       case success: BackendJobPreparationSucceeded =>
         success.jobDescriptor.runtimeAttributes("docker").valueString shouldBe dockerValue
-        success.jobDescriptor.callCachingEligibility.isInstanceOf[CallCachingIneligible] shouldBe true
-        success.jobDescriptor.callCachingEligibility.asInstanceOf[CallCachingIneligible].message shouldBe
-          s"""You are using a floating docker tag in this task. Cromwell does not consider tasks with floating tags to be eligible for call caching.
-              |If you want this task to be eligible for call caching in the future, use a docker runtime attribute with a digest instead.
-              |Cromwell attempted to retrieve the current hash for this docker image but failed.
-              |This is not necessarily a cause for concern as Cromwell is currently only able to retrieve hashes for Dockerhub and GCR images.
-              |The job will be dispatched to the appropriate backend that will attempt to run it.""".stripMargin
+        success.jobDescriptor.maybeCallCachingEligible shouldBe FloatingDockerTagWithoutHash("ubuntu:latest")
     }
   }
-  
 }
diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/preparation/JobPreparationTestHelper.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/preparation/JobPreparationTestHelper.scala
index 6c985a2c0..4d129ba00 100644
--- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/preparation/JobPreparationTestHelper.scala
+++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/preparation/JobPreparationTestHelper.scala
@@ -3,53 +3,91 @@ package cromwell.engine.workflow.lifecycle.execution.preparation
 import akka.actor.{ActorRef, ActorSystem, Props}
 import akka.testkit.TestProbe
 import cromwell.backend._
+import cromwell.core.WorkflowId
 import cromwell.engine.EngineWorkflowDescriptor
 import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActorData
-import org.mockito.Mockito._
+import cromwell.services.keyvalue.KeyValueServiceActor.{KvJobKey, ScopedKey}
 import org.specs2.mock.Mockito
-import wdl4s.Declaration
-import wdl4s.values.WdlValue
+import wdl4s.wdl._
+import wdl4s.wdl.values.WdlValue
 
 import scala.concurrent.duration.FiniteDuration
 import scala.util.Try
+import JobPreparationTestHelper._
+import wdl4s.wdl.expression.NoFunctions
 
 class JobPreparationTestHelper(implicit val system: ActorSystem) extends Mockito {
   val executionData = mock[WorkflowExecutionActorData]
   val workflowDescriptor = mock[EngineWorkflowDescriptor]
   val backendDescriptor = mock[BackendWorkflowDescriptor]
   workflowDescriptor.backendDescriptor returns backendDescriptor
+  workflowDescriptor.id returns WorkflowId.randomId()
   executionData.workflowDescriptor returns workflowDescriptor
   val jobKey = mock[BackendJobDescriptorKey]
   val serviceRegistryProbe = TestProbe()
+  val ioActor = TestProbe()
+  val workflowDockerLookupActor = TestProbe()
 
-  def buildJobPreparationMock(
-                               backpressureTimeout: FiniteDuration,
-                               noResponseTimeout: FiniteDuration,
-                               dockerHashCredentials: List[Any],
-                               dockerHashingActor: ActorRef,
-                               inputsAndAttributes: Try[(Map[Declaration, WdlValue], Map[wdl4s.LocallyQualifiedName, WdlValue])]
-                             ) = {
-    val factory = mock[BackendLifecycleActorFactory]
-    when(factory.dockerHashCredentials(None)).thenReturn(dockerHashCredentials)
-    when(factory.jobExecutionActorProps(
-      any[BackendJobDescriptor],
-      any[Option[BackendInitializationData]],
-      any[ActorRef],
-      any[Option[ActorRef]]
-    )).thenReturn(Props.empty)
-
-    Props(new JobPreparationActor(
-      executionData,
-      jobKey,
-      factory,
-      dockerHashingActor,
-      None,
-      serviceRegistryProbe.ref,
-      None
-    ) {
-      override lazy val dockerNoResponseTimeout = noResponseTimeout
-      override lazy val backpressureWaitTime = backpressureTimeout
-      override def evaluateInputsAndAttributes = inputsAndAttributes
-    })
+  val workflowId = WorkflowId.randomId()
+  val scopedKeyMaker: ScopedKeyMaker = key => ScopedKey(workflowId, KvJobKey("correct.horse.battery.staple", None, 1), key)
+
+  def buildTestJobPreparationActor(backpressureTimeout: FiniteDuration,
+                                   noResponseTimeout: FiniteDuration,
+                                   dockerHashCredentials: List[Any],
+                                   inputsAndAttributes: Try[(Map[Declaration, WdlValue], Map[wdl4s.wdl.LocallyQualifiedName, WdlValue])],
+                                   kvStoreKeysForPrefetch: List[String]) = {
+
+    Props(new TestJobPreparationActor(
+      kvStoreKeysForPrefetch = kvStoreKeysForPrefetch,
+      dockerHashCredentialsInput = dockerHashCredentials,
+      backpressureWaitTimeInput = backpressureTimeout,
+      dockerNoResponseTimeoutInput = noResponseTimeout,
+      inputsAndAttributes = inputsAndAttributes,
+      executionData = executionData,
+      jobKey = jobKey,
+      workflowDockerLookupActor = workflowDockerLookupActor.ref,
+      serviceRegistryActor = serviceRegistryProbe.ref,
+      ioActor = ioActor.ref,
+      scopedKeyMaker))
   }
 }
+
+private[preparation] class TestJobPreparationActor(kvStoreKeysForPrefetch: List[String],
+                                                   dockerHashCredentialsInput: List[Any],
+                                                   backpressureWaitTimeInput: FiniteDuration,
+                                                   dockerNoResponseTimeoutInput: FiniteDuration,
+                                                   inputsAndAttributes: Try[(Map[Declaration, WdlValue], Map[wdl4s.wdl.LocallyQualifiedName, WdlValue])],
+                                                   executionData: WorkflowExecutionActorData,
+                                                   jobKey: BackendJobDescriptorKey,
+                                                   workflowDockerLookupActor: ActorRef,
+                                                   serviceRegistryActor: ActorRef,
+                                                   ioActor: ActorRef,
+                                                   scopedKeyMaker: ScopedKeyMaker) extends JobPreparationActor(executionData = executionData,
+                                                                                                  jobKey = jobKey,
+                                                                                                  factory = null,
+                                                                                                  workflowDockerLookupActor = workflowDockerLookupActor,
+                                                                                                  initializationData = None,
+                                                                                                  serviceRegistryActor = serviceRegistryActor,
+                                                                                                  ioActor = ioActor,
+                                                                                                  backendSingletonActor = None) {
+
+  override lazy val kvStoreKeysToPrefetch = kvStoreKeysForPrefetch
+
+  override private[preparation] lazy val expressionLanguageFunctions = NoFunctions
+  override private[preparation] lazy val dockerHashCredentials = dockerHashCredentialsInput
+  override private[preparation] lazy val noResponseTimeout = dockerNoResponseTimeoutInput
+  override private[preparation] lazy val hasDockerDefinition = true
+
+  override def scopedKey(key: String) = scopedKeyMaker.apply(key)
+  override def evaluateInputsAndAttributes = inputsAndAttributes
+
+  override private[preparation] def jobExecutionProps(jobDescriptor: BackendJobDescriptor,
+                                                      initializationData: Option[BackendInitializationData],
+                                                      serviceRegistryActor: ActorRef,
+                                                      ioActor: ActorRef,
+                                                      backendSingletonActor: Option[ActorRef]) = Props.empty
+}
+
+object JobPreparationTestHelper {
+  type ScopedKeyMaker = String => ScopedKey
+}
diff --git a/engine/src/test/scala/cromwell/engine/workflow/mocks/DeclarationMock.scala b/engine/src/test/scala/cromwell/engine/workflow/mocks/DeclarationMock.scala
index 471d00777..337bb0820 100644
--- a/engine/src/test/scala/cromwell/engine/workflow/mocks/DeclarationMock.scala
+++ b/engine/src/test/scala/cromwell/engine/workflow/mocks/DeclarationMock.scala
@@ -1,8 +1,8 @@
 package cromwell.engine.workflow.mocks
 
 import org.specs2.mock.Mockito
-import wdl4s.{Declaration, WdlExpression}
-import wdl4s.types.WdlType
+import wdl4s.wdl.{Declaration, WdlExpression}
+import wdl4s.wdl.types.WdlType
 
 object DeclarationMock {
   type DeclarationMockType = (String, WdlType, WdlExpression)
diff --git a/engine/src/test/scala/cromwell/engine/workflow/mocks/TaskMock.scala b/engine/src/test/scala/cromwell/engine/workflow/mocks/TaskMock.scala
index 4d8ef9c1d..45d5e076c 100644
--- a/engine/src/test/scala/cromwell/engine/workflow/mocks/TaskMock.scala
+++ b/engine/src/test/scala/cromwell/engine/workflow/mocks/TaskMock.scala
@@ -2,7 +2,7 @@ package cromwell.engine.workflow.mocks
 
 import cromwell.engine.workflow.mocks.DeclarationMock.DeclarationMockType
 import org.specs2.mock.Mockito
-import wdl4s._
+import wdl4s.wdl._
 import wdl4s.parser.WdlParser.Ast
 
 trait TaskMock extends Mockito {
@@ -13,7 +13,7 @@ trait TaskMock extends Mockito {
                commandTemplateString: String = "!!shazam!!",
                outputs: Seq[DeclarationMockType] = Seq.empty
               ) = {
-    val task = mock[Task]
+    val task = mock[WdlTask]
     task.declarations returns declarations
     task.runtimeAttributes returns runtimeAttributes
     task.commandTemplateString returns commandTemplateString
diff --git a/engine/src/test/scala/cromwell/engine/workflow/mocks/WdlExpressionMock.scala b/engine/src/test/scala/cromwell/engine/workflow/mocks/WdlExpressionMock.scala
index 6bb15b306..735a8b852 100644
--- a/engine/src/test/scala/cromwell/engine/workflow/mocks/WdlExpressionMock.scala
+++ b/engine/src/test/scala/cromwell/engine/workflow/mocks/WdlExpressionMock.scala
@@ -1,10 +1,10 @@
 package cromwell.engine.workflow.mocks
 
 import org.specs2.mock.Mockito
-import wdl4s.WdlExpression
-import wdl4s.WdlExpression._
-import wdl4s.expression.WdlFunctions
-import wdl4s.values.{WdlInteger, WdlString, WdlValue}
+import wdl4s.wdl.WdlExpression
+import wdl4s.wdl.WdlExpression._
+import wdl4s.wdl.expression.WdlFunctions
+import wdl4s.wdl.values.{WdlInteger, WdlString, WdlValue}
 
 import scala.util.Success
 
diff --git a/engine/src/test/scala/cromwell/engine/workflow/tokens/TestTokenGrabbingActor.scala b/engine/src/test/scala/cromwell/engine/workflow/tokens/TestTokenGrabbingActor.scala
index 669e6783f..5677a3eb6 100644
--- a/engine/src/test/scala/cromwell/engine/workflow/tokens/TestTokenGrabbingActor.scala
+++ b/engine/src/test/scala/cromwell/engine/workflow/tokens/TestTokenGrabbingActor.scala
@@ -16,7 +16,7 @@ class TestTokenGrabbingActor(tokenDispenser: ActorRef, tokenType: JobExecutionTo
 
   def receive = {
     case JobExecutionTokenDispensed(dispensedToken) => token = Option(dispensedToken)
-    case JobExecutionTokenDenied(positionInQueue) => rejections += 1
+    case JobExecutionTokenDenied(_) => rejections += 1
     case AkkaTestUtil.ThrowException => throw new RuntimeException("Test exception (don't be scared by the stack trace, it's deliberate!)")
     case AkkaTestUtil.InternalStop => context.stop(self)
   }
diff --git a/engine/src/test/scala/cromwell/jobstore/JobResultSpec.scala b/engine/src/test/scala/cromwell/jobstore/JobResultSpec.scala
index d997fb56c..d84c1d5df 100644
--- a/engine/src/test/scala/cromwell/jobstore/JobResultSpec.scala
+++ b/engine/src/test/scala/cromwell/jobstore/JobResultSpec.scala
@@ -3,10 +3,10 @@ package cromwell.jobstore
 import cromwell.core.JobOutput
 import cromwell.jobstore.JobResultJsonFormatter._
 import org.scalatest.{FlatSpec, Matchers}
-import wdl4s.values.WdlString
+import wdl4s.wdl.values.WdlString
 import spray.json._
-import wdl4s.types.{WdlIntegerType, WdlMapType, WdlStringType}
-import wdl4s.values._
+import wdl4s.wdl.types.{WdlIntegerType, WdlMapType, WdlStringType}
+import wdl4s.wdl.values._
 
 class JobResultSpec extends FlatSpec with Matchers {
 
diff --git a/engine/src/test/scala/cromwell/jobstore/JobStoreServiceSpec.scala b/engine/src/test/scala/cromwell/jobstore/JobStoreServiceSpec.scala
index ac84a6c45..37e04579e 100644
--- a/engine/src/test/scala/cromwell/jobstore/JobStoreServiceSpec.scala
+++ b/engine/src/test/scala/cromwell/jobstore/JobStoreServiceSpec.scala
@@ -1,6 +1,6 @@
 package cromwell.jobstore
 
-import cromwell.CromwellTestKitSpec
+import cromwell.CromwellTestKitWordSpec
 import cromwell.backend.BackendJobDescriptorKey
 import cromwell.core.{JobOutput, WorkflowId}
 import cromwell.jobstore.JobStoreActor._
@@ -9,9 +9,9 @@ import cromwell.services.SingletonServicesStore
 import org.scalatest.Matchers
 import org.specs2.mock.Mockito
 import wdl4s.parser.WdlParser.Ast
-import wdl4s.types.WdlStringType
-import wdl4s.values.WdlString
-import wdl4s._
+import wdl4s.wdl.types.WdlStringType
+import wdl4s.wdl.values.WdlString
+import wdl4s.wdl._
 
 import scala.concurrent.duration._
 import scala.language.postfixOps
@@ -21,7 +21,7 @@ object JobStoreServiceSpec {
   val EmptyExpression = WdlExpression.fromString(""" "" """)
 }
 
-class JobStoreServiceSpec extends CromwellTestKitSpec with Matchers with Mockito {
+class JobStoreServiceSpec extends CromwellTestKitWordSpec with Matchers with Mockito {
 
   "JobStoreService" should {
     "work" in {
@@ -29,9 +29,9 @@ class JobStoreServiceSpec extends CromwellTestKitSpec with Matchers with Mockito
       val jobStoreService = system.actorOf(JobStoreActor.props(jobStore))
 
       val workflowId = WorkflowId.randomId()
-      val successCall = mock[TaskCall]
+      val successCall = mock[WdlTaskCall]
       successCall.fullyQualifiedName returns "foo.bar"
-      val mockTask = mock[Task]
+      val mockTask = mock[WdlTask]
       mockTask.outputs returns Seq(TaskOutput("baz", WdlStringType, EmptyExpression, mock[Ast], Option(mockTask)))
       successCall.task returns mockTask
 
@@ -50,7 +50,7 @@ class JobStoreServiceSpec extends CromwellTestKitSpec with Matchers with Mockito
         case JobComplete(JobResultSuccess(Some(0), os)) if os == outputs =>
       }
 
-      val failureCall = mock[TaskCall]
+      val failureCall = mock[WdlTaskCall]
       failureCall.fullyQualifiedName returns "baz.qux"
       val failureKey = BackendJobDescriptorKey(failureCall, None, 1).toJobStoreKey(workflowId)
 
diff --git a/engine/src/test/scala/cromwell/jobstore/JobStoreWriterSpec.scala b/engine/src/test/scala/cromwell/jobstore/JobStoreWriterSpec.scala
index eabc86a5e..3ffd3cfec 100644
--- a/engine/src/test/scala/cromwell/jobstore/JobStoreWriterSpec.scala
+++ b/engine/src/test/scala/cromwell/jobstore/JobStoreWriterSpec.scala
@@ -1,31 +1,36 @@
 package cromwell.jobstore
 
 import akka.testkit.TestFSMRef
-import cromwell.CromwellTestKitSpec
+import cromwell.CromwellTestKitWordSpec
 import cromwell.core.WorkflowId
+import cromwell.core.actor.BatchingDbWriter
+import cromwell.core.actor.BatchingDbWriter.{BatchingDbWriterState, WritingToDb}
+import cromwell.jobstore.JobStore.{JobCompletion, WorkflowCompletion}
 import cromwell.jobstore.JobStoreActor.{JobStoreWriteSuccess, RegisterJobCompleted, RegisterWorkflowCompleted}
 import org.scalatest.{BeforeAndAfter, Matchers}
-import wdl4s.TaskOutput
+import wdl4s.wdl.TaskOutput
 
 import scala.concurrent.duration._
 import scala.concurrent.{ExecutionContext, Future, Promise}
 import scala.language.postfixOps
 
-class JobStoreWriterSpec extends CromwellTestKitSpec with Matchers with BeforeAndAfter {
+class JobStoreWriterSpec extends CromwellTestKitWordSpec with Matchers with BeforeAndAfter {
   
   var database: WriteCountingJobStore = _
-  var jobStoreWriter: TestFSMRef[JobStoreWriterState, JobStoreWriterData, JobStoreWriterActor] = _
+  var jobStoreWriter: TestFSMRef[BatchingDbWriterState, BatchingDbWriter.BatchingDbWriterData, JobStoreWriterActor] = _
   var workflowId: WorkflowId = _
   val successResult: JobResult = JobResultSuccess(Some(0), Map.empty)
+  val flushFrequency = 0.5 second
+  val sleepMillis = flushFrequency.toMillis * 5
 
   before {
     database = WriteCountingJobStore.makeNew
-    jobStoreWriter = TestFSMRef(new JobStoreWriterActor(database))
+    jobStoreWriter = TestFSMRef(new JobStoreWriterActor(database, 5, flushFrequency))
     workflowId = WorkflowId.randomId()
   }
 
-  private def sendRegisterCompletions(attempts: Int): Unit = {
-    0 until attempts foreach { a => jobStoreWriter ! RegisterJobCompleted(jobKey(attempt = a), successResult) }
+  private def sendRegisterCompletion(attempt: Int): Unit = {
+    jobStoreWriter ! RegisterJobCompleted(jobKey(attempt), successResult)
   }
 
   private def jobKey(attempt: Int): JobStoreKey = JobStoreKey(workflowId, s"call.fqn", None, attempt)
@@ -52,26 +57,26 @@ class JobStoreWriterSpec extends CromwellTestKitSpec with Matchers with BeforeAn
       case JobStoreWriteSuccess(RegisterWorkflowCompleted(id)) => id shouldBe workflowId
       case message => fail(s"Unexpected response message: $message")
     }
-    jobStoreWriter.underlyingActor.stateName shouldBe Pending
     ()
   }
 
   "JobStoreWriter" should {
     "be able to collapse writes together if they arrive while a database access is ongoing" in {
 
-      sendRegisterCompletions(attempts = 3)
-
+      // Send a job completion. The database will hang.
+      sendRegisterCompletion(1)
       val writer = jobStoreWriter.underlyingActor
-      writer.stateName shouldBe WritingToDatabase
-      writer.stateData.currentOperation should have size 1
-      writer.stateData.nextOperation should have size 2
+      eventually {
+        writer.stateName shouldBe WritingToDb
+      }
 
-      // The testing DB intentionally blocks after the first write until `continue` is called.
+      // Send some more completions.  These should pile up in the state data.
+      List(2, 3) foreach sendRegisterCompletion
+
+      writer.stateData.length should equal(2)
       database.continue()
 
       assertReceived(expectedJobStoreWriteAcks = 3)
-      writer.stateData.currentOperation shouldBe empty
-      writer.stateData.nextOperation shouldBe empty
 
       assertDb(
         totalWritesCalled = 2,
@@ -82,20 +87,22 @@ class JobStoreWriterSpec extends CromwellTestKitSpec with Matchers with BeforeAn
 
     "be able to skip job-completion writes if the workflow completes, but still respond appropriately" in {
 
-      sendRegisterCompletions(attempts = 2)
-      jobStoreWriter ! RegisterWorkflowCompleted(workflowId)
-
+      // Send a job completion. The database will hang.
+      sendRegisterCompletion(1)
       val writer = jobStoreWriter.underlyingActor
-      writer.stateName shouldBe WritingToDatabase
-      writer.stateData.currentOperation should have size 1
-      writer.stateData.nextOperation should have size 2
+      eventually {
+        writer.stateName shouldBe WritingToDb
+      }
+
+      // Send some more completions.  These should pile up in the state data.
+      List(2, 3) foreach sendRegisterCompletion
+      jobStoreWriter ! RegisterWorkflowCompleted(workflowId)
 
+      writer.stateData.length should equal(3)
       // The testing DB intentionally blocks after the first write until `continue` is called.
       database.continue()
 
       assertReceived(expectedJobStoreWriteAcks = 3)
-      writer.stateData.currentOperation shouldBe empty
-      writer.stateData.nextOperation shouldBe empty
 
       assertDb(
         totalWritesCalled = 2,
@@ -114,8 +121,9 @@ class WriteCountingJobStore(var totalWritesCalled: Int, var jobCompletionsRecord
 
   def continue() = writePromise.trySuccess(())
 
-  override def writeToDatabase(jobCompletions: Map[JobStoreKey, JobResult], workflowCompletions: List[WorkflowId])
+  override def writeToDatabase(workflowCompletions: Seq[WorkflowCompletion], jobCompletions: Seq[JobCompletion], batchSize: Int)
                               (implicit ec: ExecutionContext): Future[Unit] = {
+
     totalWritesCalled += 1
     jobCompletionsRecorded += jobCompletions.size
     workflowCompletionsRecorded += workflowCompletions.size
diff --git a/engine/src/test/scala/cromwell/server/CromwellServerSpec.scala b/engine/src/test/scala/cromwell/server/CromwellServerSpec.scala
deleted file mode 100644
index 01cd00236..000000000
--- a/engine/src/test/scala/cromwell/server/CromwellServerSpec.scala
+++ /dev/null
@@ -1,33 +0,0 @@
-package cromwell.server
-
-import akka.pattern.ask
-import akka.util.Timeout
-import com.typesafe.config.ConfigFactory
-import org.scalatest.concurrent.{PatienceConfiguration, ScalaFutures}
-import org.scalatest.{FlatSpec, Matchers}
-import org.specs2.mock.Mockito
-import spray.http.{ContentTypes, HttpRequest, HttpResponse, Timedout}
-
-import scala.concurrent.duration._
-
-class CromwellServerSpec extends FlatSpec with Matchers with Mockito with ScalaFutures {
-  implicit val timeout: Timeout = 5.seconds
-
-  it should "return 500 errors as Json" in {
-    val cromwellSystem = new CromwellSystem {}
-
-    val cromwellServerActor = cromwellSystem.actorSystem.actorOf(CromwellServerActor.props(ConfigFactory.empty()))
-    val response = cromwellServerActor.ask(Timedout(mock[HttpRequest]))
-
-    response.futureValue(PatienceConfiguration.Timeout(timeout.duration)) match {
-      case h: HttpResponse =>
-        h.entity.toOption match {
-          case Some(e) => e.contentType.toString() shouldBe ContentTypes.`application/json`.mediaType.value.toString
-          case None => fail()
-        }
-      case _ => fail()
-    }
-
-    cromwellSystem.shutdownActorSystem()
-  }
-}
diff --git a/engine/src/test/scala/cromwell/subworkflowstore/SubWorkflowStoreSpec.scala b/engine/src/test/scala/cromwell/subworkflowstore/SubWorkflowStoreSpec.scala
index 5101cee46..27c8b61b2 100644
--- a/engine/src/test/scala/cromwell/subworkflowstore/SubWorkflowStoreSpec.scala
+++ b/engine/src/test/scala/cromwell/subworkflowstore/SubWorkflowStoreSpec.scala
@@ -1,17 +1,18 @@
 package cromwell.subworkflowstore
 
-import cromwell.CromwellTestKitSpec
+import cromwell.CromwellTestKitWordSpec
 import cromwell.core.{JobKey, WorkflowId, WorkflowSourceFilesWithoutImports}
 import cromwell.services.SingletonServicesStore
 import cromwell.subworkflowstore.SubWorkflowStoreActor._
 import org.scalatest.Matchers
 import org.specs2.mock.Mockito
-import wdl4s.{TaskCall, WdlExpression}
+import wdl4s.wdl.{WdlTaskCall, WdlExpression}
 import cromwell.core.ExecutionIndex._
 
 import scala.concurrent.duration._
 import SubWorkflowStoreSpec._
 import akka.testkit.TestProbe
+import cromwell.database.sql.SqlDatabase
 import cromwell.database.sql.tables.SubWorkflowStoreEntry
 import cromwell.engine.workflow.workflowstore.WorkflowStoreActor.SubmitWorkflow
 import cromwell.engine.workflow.workflowstore.WorkflowStoreSubmitActor.WorkflowSubmittedToStore
@@ -24,19 +25,19 @@ object SubWorkflowStoreSpec {
   val EmptyExpression = WdlExpression.fromString(""" "" """)
 }
 
-class SubWorkflowStoreSpec extends CromwellTestKitSpec with Matchers with Mockito {
+class SubWorkflowStoreSpec extends CromwellTestKitWordSpec with Matchers with Mockito {
   "SubWorkflowStore" should {
     "work" in {
       lazy val subWorkflowStore = new SqlSubWorkflowStore(SingletonServicesStore.databaseInterface)
       val subWorkflowStoreService = system.actorOf(SubWorkflowStoreActor.props(subWorkflowStore))
 
       lazy val workflowStore = SqlWorkflowStore(SingletonServicesStore.databaseInterface)
-      val workflowStoreService = system.actorOf(WorkflowStoreActor.props(workflowStore, TestProbe().ref))
+      val workflowStoreService = system.actorOf(WorkflowStoreActor.props(workflowStore, TestProbe().ref, mock[SqlDatabase]))
 
       val parentWorkflowId = WorkflowId.randomId()
       val subWorkflowId = WorkflowId.randomId()
       val subSubWorkflowId = WorkflowId.randomId()
-      val call = mock[TaskCall]
+      val call = mock[WdlTaskCall]
       call.fullyQualifiedName returns "foo.bar"
       val jobKey = new JobKey {
         override def scope = call
@@ -45,7 +46,13 @@ class SubWorkflowStoreSpec extends CromwellTestKitSpec with Matchers with Mockit
         override def tag: String = "foobar"
       }
 
-      workflowStoreService ! SubmitWorkflow(WorkflowSourceFilesWithoutImports("", "{}", "{}", "{}"))
+      workflowStoreService ! SubmitWorkflow(WorkflowSourceFilesWithoutImports(
+        workflowSource = "",
+        workflowType = Option("WDL"),
+        workflowTypeVersion = None,
+        inputsJson = "{}",
+        workflowOptionsJson = "{}",
+        labelsJson = "{}"))
       val rootWorkflowId = expectMsgType[WorkflowSubmittedToStore](10 seconds).workflowId
 
       // Query for non existing sub workflow
@@ -72,17 +79,22 @@ class SubWorkflowStoreSpec extends CromwellTestKitSpec with Matchers with Mockit
       
       // Delete root workflow
       subWorkflowStoreService ! WorkflowComplete(rootWorkflowId)
-      expectMsgType[SubWorkflowStoreCompleteSuccess](MaxWait)
 
-      // Verify that everything is gone
-      subWorkflowStoreService ! QuerySubWorkflow(rootWorkflowId, jobKey)
-      expectMsgType[SubWorkflowNotFound](MaxWait)
+      // Verify that everything is gone (eventually!)
+      eventually {
+        subWorkflowStoreService ! QuerySubWorkflow(rootWorkflowId, jobKey)
+        expectMsgType[SubWorkflowNotFound](MaxWait)
+      }
 
-      subWorkflowStoreService ! QuerySubWorkflow(parentWorkflowId, jobKey)
-      expectMsgType[SubWorkflowNotFound](MaxWait)
+      eventually {
+        subWorkflowStoreService ! QuerySubWorkflow(parentWorkflowId, jobKey)
+        expectMsgType[SubWorkflowNotFound](MaxWait)
+      }
 
-      subWorkflowStoreService ! QuerySubWorkflow(subWorkflowId, jobKey)
-      expectMsgType[SubWorkflowNotFound](MaxWait)
+      eventually {
+        subWorkflowStoreService ! QuerySubWorkflow(subWorkflowId, jobKey)
+        expectMsgType[SubWorkflowNotFound](MaxWait)
+      }
     }
   }
 }
diff --git a/engine/src/test/scala/cromwell/webservice/CromwellApiServiceSpec.scala b/engine/src/test/scala/cromwell/webservice/CromwellApiServiceSpec.scala
index b360b8e09..558e3657a 100644
--- a/engine/src/test/scala/cromwell/webservice/CromwellApiServiceSpec.scala
+++ b/engine/src/test/scala/cromwell/webservice/CromwellApiServiceSpec.scala
@@ -1,46 +1,78 @@
 package cromwell.webservice
 
-import akka.actor.{Actor, ActorSystem, Props}
+import akka.actor.{Actor, ActorLogging, ActorSystem, Props}
 import cromwell.core.{WorkflowId, WorkflowMetadataKeys, WorkflowSubmitted, WorkflowSucceeded}
-
-import scala.util.{Failure, Success, Try}
+import akka.http.scaladsl.coding.{Decoder, Gzip}
+import akka.http.scaladsl.server.Route
+import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._
+import spray.json.DefaultJsonProtocol._
 import cromwell.engine.workflow.workflowstore.WorkflowStoreActor.{AbortWorkflow, BatchSubmitWorkflows, SubmitWorkflow}
 import cromwell.engine.workflow.workflowstore.WorkflowStoreEngineActor
 import cromwell.engine.workflow.workflowstore.WorkflowStoreEngineActor.WorkflowAbortFailed
-import cromwell.services.metadata.MetadataService._
-import org.scalatest.{FlatSpec, Matchers}
-import spray.http._
-import spray.json.DefaultJsonProtocol._
-import spray.json.{JsString, _}
-import spray.routing._
 import cromwell.engine.workflow.workflowstore.WorkflowStoreSubmitActor.{WorkflowSubmittedToStore, WorkflowsBatchSubmittedToStore}
+import cromwell.services.metadata.MetadataService._
+import akka.http.scaladsl.model._
+import akka.http.scaladsl.model.headers.{HttpEncodings, `Accept-Encoding`}
+import akka.http.scaladsl.testkit.{RouteTestTimeout, ScalatestRouteTest}
+import akka.http.scaladsl.unmarshalling.Unmarshal
+import akka.stream.ActorMaterializer
+import cromwell.engine.workflow.WorkflowManagerActor
+import cromwell.services.metadata._
 import cromwell.util.SampleWdl.HelloWorld
-import spray.httpx.SprayJsonSupport._
-import spray.testkit.ScalatestRouteTest
+import org.scalatest.{AsyncFlatSpec, Matchers}
+import spray.json._
 
-class CromwellApiServiceSpec extends FlatSpec with ScalatestRouteTest with Matchers {
+import scala.concurrent.duration._
+
+class CromwellApiServiceSpec extends AsyncFlatSpec with ScalatestRouteTest with Matchers {
   import CromwellApiServiceSpec._
 
-  val cromwellApiService = new MockApiService()
+  val akkaHttpService = new MockApiService()
   val version = "v1"
 
-  behavior of "REST API /status endpoint"
-  it should "return 200 for get of a known workflow id" in {
-    val workflowId = MockApiService.ExistingWorkflowId
-    Get(s"/workflows/$version/$workflowId/status") ~>
-      cromwellApiService.statusRoute ~>
+  implicit def default = RouteTestTimeout(5.seconds)
+
+  "REST ENGINE /stats endpoint" should "return 200 for stats" in {
+    Get(s"/engine/$version/stats") ~>
+      akkaHttpService.engineRoutes ~>
       check {
-          status should be(StatusCodes.OK)
-          val result = responseAs[JsObject]
-          result.fields(WorkflowMetadataKeys.Status) should be(JsString("Submitted"))
+        status should be(StatusCodes.OK)
+        val resp = responseAs[JsObject]
+        val workflows = resp.fields("workflows").asInstanceOf[JsNumber].value.toInt
+        val jobs = resp.fields("jobs").asInstanceOf[JsNumber].value.toInt
+        workflows should be(1)
+        jobs should be(23)
       }
   }
 
-    it should "return 404 for get of unknown workflow" in {
-      val workflowId = MockApiService.UnrecognizedWorkflowId
+  "REST ENGINE /version endpoint" should "return 200 for version" in {
+    Get(s"/engine/$version/version") ~>
+      akkaHttpService.engineRoutes ~>
+      check {
+        status should be(StatusCodes.OK)
+        val resp = responseAs[JsObject]
+        val cromwellVersion = resp.fields("cromwell").asInstanceOf[JsString].value
+        cromwellVersion should fullyMatch regex """\d+-([0-9a-f]){7}(-SNAP)?"""
+      }
+  }
+
+    behavior of "REST API /status endpoint"
+    it should "return 200 for get of a known workflow id" in {
+      val workflowId = CromwellApiServiceSpec.ExistingWorkflowId
 
       Get(s"/workflows/$version/$workflowId/status") ~>
-        cromwellApiService.statusRoute ~>
+        akkaHttpService.workflowRoutes ~>
+        check {
+            status should be(StatusCodes.OK)
+            // Along w/ checking value, ensure it is valid JSON despite the requested content type
+            responseAs[JsObject].fields(WorkflowMetadataKeys.Status) should be(JsString("Submitted"))
+        }
+    }
+
+    it should "return 404 for get of unknown workflow" in {
+      val workflowId = CromwellApiServiceSpec.UnrecognizedWorkflowId
+      Get(s"/workflows/$version/$workflowId/status") ~>
+        akkaHttpService.workflowRoutes ~>
         check {
           assertResult(StatusCodes.NotFound) {
             status
@@ -48,9 +80,9 @@ class CromwellApiServiceSpec extends FlatSpec with ScalatestRouteTest with Match
         }
     }
 
-     it should "return 400 for get of a malformed workflow id's status" in {
+    it should "return 400 for get of a malformed workflow id's status" in {
       Get(s"/workflows/$version/foobar/status") ~>
-        cromwellApiService.statusRoute ~>
+        akkaHttpService.workflowRoutes ~>
         check {
           assertResult(StatusCodes.BadRequest) {
             status
@@ -66,40 +98,40 @@ class CromwellApiServiceSpec extends FlatSpec with ScalatestRouteTest with Match
         }
     }
 
-  behavior of "REST API /abort endpoint"
-  it should "return 404 for abort of unknown workflow" in {
-    val workflowId = MockApiService.UnrecognizedWorkflowId
+    behavior of "REST API /abort endpoint"
+    it should "return 404 for abort of unknown workflow" in {
+      val workflowId = CromwellApiServiceSpec.UnrecognizedWorkflowId
 
-    Post(s"/workflows/$version/$workflowId/abort") ~>
-      cromwellApiService.abortRoute ~>
-      check {
-        assertResult(StatusCodes.NotFound) {
-          status
+      Post(s"/workflows/$version/$workflowId/abort") ~>
+        akkaHttpService.workflowRoutes ~>
+        check {
+          assertResult(StatusCodes.NotFound) {
+            status
+          }
         }
-      }
-  }
+    }
 
-  it should "return 400 for abort of a malformed workflow id" in {
-    Post(s"/workflows/$version/foobar/abort") ~>
-      cromwellApiService.abortRoute ~>
-      check {
-        assertResult(StatusCodes.BadRequest) {
-          status
-        }
-        assertResult(
-          """{
-            |  "status": "fail",
-            |  "message": "Invalid workflow ID: 'foobar'."
-            |}""".stripMargin
-        ) {
-          responseAs[String]
+    it should "return 400 for abort of a malformed workflow id" in {
+      Post(s"/workflows/$version/foobar/abort") ~>
+        akkaHttpService.workflowRoutes ~>
+        check {
+          assertResult(StatusCodes.BadRequest) {
+            status
+          }
+          assertResult(
+            """{
+              |  "status": "fail",
+              |  "message": "Invalid workflow ID: 'foobar'."
+              |}""".stripMargin
+          ) {
+            responseAs[String]
+          }
         }
-      }
-  }
+    }
 
     it should "return 403 for abort of a workflow in a terminal state" in {
-      Post(s"/workflows/$version/${MockApiService.AbortedWorkflowId}/abort") ~>
-      cromwellApiService.abortRoute ~>
+      Post(s"/workflows/$version/${CromwellApiServiceSpec.AbortedWorkflowId}/abort") ~>
+        akkaHttpService.workflowRoutes ~>
       check {
         assertResult(StatusCodes.Forbidden) {
           status
@@ -107,24 +139,19 @@ class CromwellApiServiceSpec extends FlatSpec with ScalatestRouteTest with Match
         assertResult(
           s"""{
               |  "status": "error",
-              |  "message": "Workflow ID '${MockApiService.AbortedWorkflowId}' is in terminal state 'Aborted' and cannot be aborted."
+              |  "message": "Workflow ID '${CromwellApiServiceSpec.AbortedWorkflowId}' is in terminal state 'Aborted' and cannot be aborted."
               |}""".stripMargin
         ) {
           responseAs[String]
         }
       }
     }
-
     it should "return 200 for abort of a known workflow id" in {
-      Post(s"/workflows/$version/${MockApiService.ExistingWorkflowId}/abort") ~>
-        cromwellApiService.abortRoute ~>
+      Post(s"/workflows/$version/${CromwellApiServiceSpec.ExistingWorkflowId}/abort") ~>
+        akkaHttpService.workflowRoutes ~>
         check {
           assertResult(
-            s"""{
-                |  "id": "${MockApiService.ExistingWorkflowId.toString}",
-                |  "status": "Aborted"
-                |}"""
-              .stripMargin) {
+            s"""{"id":"${CromwellApiServiceSpec.ExistingWorkflowId.toString}","status":"Aborted"}""") {
             responseAs[String]
           }
           assertResult(StatusCodes.OK) {
@@ -133,187 +160,232 @@ class CromwellApiServiceSpec extends FlatSpec with ScalatestRouteTest with Match
         }
     }
 
-  behavior of "REST API submission endpoint"
-  it should "return 201 for a successful workflow submission " in {
-    val bodyParts: Map[String, BodyPart] = Map("wdlSource" -> BodyPart(HelloWorld.wdlSource()), "workflowInputs" -> BodyPart(HelloWorld.rawInputs.toJson.toString()))
-    Post(s"/workflows/$version", MultipartFormData(bodyParts)) ~>
-      cromwellApiService.submitRoute ~>
-      check {
-        assertResult(
-          s"""{
-              |  "id": "${MockApiService.ExistingWorkflowId.toString}",
-              |  "status": "Submitted"
-              |}""".stripMargin) {
-          responseAs[String]
-        }
-        assertResult(StatusCodes.Created) {
-          status
+    behavior of "REST API submission endpoint"
+    it should "return 201 for a successful workflow submission " in {
+      val workflowSource = Multipart.FormData.BodyPart("workflowSource", HttpEntity(MediaTypes.`application/json`, HelloWorld.workflowSource()))
+      val workflowInputs = Multipart.FormData.BodyPart("workflowInputs", HttpEntity(MediaTypes.`application/json`, HelloWorld.rawInputs.toJson.toString()))
+      val formData = Multipart.FormData(workflowSource, workflowInputs).toEntity()
+      Post(s"/workflows/$version", formData) ~>
+        akkaHttpService.workflowRoutes ~>
+        check {
+          assertResult(
+            s"""{
+                |  "id": "${CromwellApiServiceSpec.ExistingWorkflowId.toString}",
+                |  "status": "Submitted"
+                |}""".stripMargin) {
+            responseAs[String].parseJson.prettyPrint
+          }
+          assertResult(StatusCodes.Created) {
+            status
+          }
         }
-      }
-  }
+    }
 
-  it should "return 400 for an unrecognized form data request parameter " in {
-    val bodyParts: Map[String, BodyPart] = Map("incorrectParameter" -> BodyPart(HelloWorld.wdlSource()))
-    Post(s"/workflows/$version", MultipartFormData(bodyParts)) ~>
-      cromwellApiService.submitRoute ~>
-      check {
-        assertResult(
-          s"""{
-              |  "status": "fail",
-              |  "message": "Unexpected body part name: incorrectParameter"
-              |}""".stripMargin) {
-          responseAs[String]
+    it should "return 400 for an unrecognized form data request parameter " in {
+      val formData = Multipart.FormData(Multipart.FormData.BodyPart("incorrectParameter", HttpEntity(MediaTypes.`application/json`, HelloWorld.workflowSource()))).toEntity()
+      Post(s"/workflows/$version", formData) ~>
+        akkaHttpService.workflowRoutes ~>
+        check {
+          assertResult(
+            s"""{
+                |  "status": "fail",
+                |  "message": "Error(s): Unexpected body part name: incorrectParameter"
+                |}""".stripMargin) {
+            responseAs[String]
+          }
+          assertResult(StatusCodes.BadRequest) {
+            status
+          }
         }
-        assertResult(StatusCodes.BadRequest) {
-          status
+    }
+
+    it should "return 400 for a workflow submission with unsupported workflow option keys" in {
+      val options = """
+                      |{
+                      |  "defaultRuntimeOptions": {
+                      |  "cpu":1
+                      |  }
+                      |}
+                      |""".stripMargin
+
+      val workflowSource = Multipart.FormData.BodyPart("workflowSource", HttpEntity(MediaTypes.`application/json`, HelloWorld.workflowSource()))
+      val workflowInputs = Multipart.FormData.BodyPart("workflowOptions", HttpEntity(MediaTypes.`application/json`, options))
+      val formData = Multipart.FormData(workflowSource, workflowInputs).toEntity()
+
+      Post(s"/workflows/$version", formData) ~>
+        akkaHttpService.workflowRoutes ~>
+        check {
+          assertResult(StatusCodes.BadRequest) {
+            status
+          }
         }
-      }
-  }
+    }
 
-  it should "succesfully merge and override multiple input files" in {
+    it should "return 400 for a workflow submission with malformed workflow options json" in {
+      val options = s"""
+                       |{"read_from_cache": "true"
+                       |""".stripMargin
 
-    val input1 = Map("wf.a1" -> "hello", "wf.a2" -> "world").toJson.toString
-    val input2 = Map.empty[String, String].toJson.toString
-    val overrideInput1 = Map("wf.a2" -> "universe").toJson.toString
-    val allInputs = cromwellApiService.mergeMaps(Seq(Option(input1), Option(input2), Option(overrideInput1)))
+      val workflowSource = Multipart.FormData.BodyPart("workflowSource", HttpEntity(MediaTypes.`application/json`, HelloWorld.workflowSource()))
+      val workflowInputs = Multipart.FormData.BodyPart("workflowOptions", HttpEntity(MediaTypes.`application/json`, options))
+      val formData = Multipart.FormData(workflowSource, workflowInputs).toEntity()
 
-    check {
-      allInputs.fields.keys should contain allOf("wf.a1", "wf.a2")
-      allInputs.fields("wf.a2") should be(JsString("universe"))
+      Post(s"/workflows/$version", formData) ~>
+        akkaHttpService.workflowRoutes ~>
+        check {
+          assertResult(StatusCodes.BadRequest) {
+            status
+          }
+        }
     }
-  }
 
-  behavior of "REST API batch submission endpoint"
-  it should "return 200 for a successful workflow submission " in {
-    val inputs = HelloWorld.rawInputs.toJson
-    val bodyParts = Map("wdlSource" -> BodyPart(HelloWorld.wdlSource()), "workflowInputs" -> BodyPart(s"[$inputs, $inputs]"))
+    behavior of "REST API batch submission endpoint"
+    it should "return 200 for a successful workflow submission " in {
+      val inputs = HelloWorld.rawInputs.toJson
+      val workflowSource = Multipart.FormData.BodyPart("workflowSource", HttpEntity(MediaTypes.`application/json`, HelloWorld.workflowSource()))
+      val workflowInputs = Multipart.FormData.BodyPart("workflowInputs", HttpEntity(MediaTypes.`application/json`, s"[$inputs, $inputs]"))
+      val formData = Multipart.FormData(workflowSource, workflowInputs).toEntity()
 
-    Post(s"/workflows/$version/batch", MultipartFormData(bodyParts)) ~>
-      cromwellApiService.submitBatchRoute ~>
-      check {
-        assertResult(
-          s"""[{
-              |  "id": "${MockApiService.ExistingWorkflowId.toString}",
-              |  "status": "Submitted"
-              |}, {
-              |  "id": "${MockApiService.ExistingWorkflowId.toString}",
-              |  "status": "Submitted"
-              |}]""".stripMargin) {
-          responseAs[String]
-        }
-        assertResult(StatusCodes.OK) {
-          status
+      Post(s"/workflows/$version/batch", formData) ~>
+        akkaHttpService.workflowRoutes ~>
+        check {
+          assertResult(
+            s"""[{
+                |  "id": "${CromwellApiServiceSpec.ExistingWorkflowId.toString}",
+                |  "status": "Submitted"
+                |}, {
+                |  "id": "${CromwellApiServiceSpec.ExistingWorkflowId.toString}",
+                |  "status": "Submitted"
+                |}]""".stripMargin) {
+            responseAs[String].parseJson.prettyPrint
+          }
+          assertResult(StatusCodes.Created) {
+            status
+          }
         }
-      }
-  }
+    }
 
-  it should "return 400 for an submission with no inputs" in {
-    val bodyParts = Map("wdlSource" -> BodyPart(HelloWorld.wdlSource()))
+    it should "return 400 for an submission with no inputs" in {
+      val formData = Multipart.FormData(Multipart.FormData.BodyPart("workflowSource", HttpEntity(MediaTypes.`application/json`, HelloWorld.workflowSource()))).toEntity()
 
-    Post(s"/workflows/$version/batch", MultipartFormData(bodyParts)) ~>
-      cromwellApiService.submitBatchRoute ~>
-      check {
-        assertResult(
-          s"""{
-              |  "status": "fail",
-              |  "message": "No inputs were provided"
-              |}""".stripMargin) {
-          responseAs[String]
+      Post(s"/workflows/$version/batch", formData) ~>
+        akkaHttpService.workflowRoutes ~>
+        check {
+          assertResult(
+            s"""{
+                |  "status": "fail",
+                |  "message": "Error(s): No inputs were provided"
+                |}""".stripMargin) {
+            responseAs[String]
+          }
+          assertResult(StatusCodes.BadRequest) {
+            status
+          }
         }
-        assertResult(StatusCodes.BadRequest) {
-          status
+    }
+
+    behavior of "REST API /outputs endpoint"
+    it should "return 200 with GET of outputs on successful execution of workflow" in {
+      Get(s"/workflows/$version/${CromwellApiServiceSpec.ExistingWorkflowId}/outputs") ~>
+        akkaHttpService.workflowRoutes ~>
+        check {
+          status should be(StatusCodes.OK)
+          responseAs[JsObject].fields.keys should contain allOf(WorkflowMetadataKeys.Id, WorkflowMetadataKeys.Outputs)
         }
-      }
-  }
+    }
 
-  behavior of "REST API /outputs endpoint"
-  it should "return 200 with GET of outputs on successful execution of workflow" in {
-    Get(s"/workflows/$version/${MockApiService.ExistingWorkflowId}/outputs") ~>
-      cromwellApiService.workflowOutputsRoute ~>
+    it should "return 404 with outputs on unknown workflow" in {
+      Get(s"/workflows/$version/${CromwellApiServiceSpec.UnrecognizedWorkflowId}/outputs") ~>
+        akkaHttpService.workflowRoutes ~>
       check {
-        status should be(StatusCodes.OK)
-        val result = responseAs[JsObject]
-        result.fields.keys should contain allOf(WorkflowMetadataKeys.Id, WorkflowMetadataKeys.Outputs)
+        assertResult(StatusCodes.NotFound) {
+          status
+        }
       }
-  }
+    }
 
-  it should "return 404 with outputs on unknown workflow" in {
-    Get(s"/workflows/$version/${MockApiService.UnrecognizedWorkflowId}/outputs") ~>
-    cromwellApiService.workflowOutputsRoute ~>
-    check {
-      assertResult(StatusCodes.NotFound) {
-        status
-      }
+    it should "return 405 with POST of outputs on successful execution of workflow" in {
+      Post(s"/workflows/$version/${CromwellApiServiceSpec.UnrecognizedWorkflowId}/outputs") ~>
+        Route.seal(akkaHttpService.workflowRoutes) ~>
+        check {
+          assertResult(StatusCodes.MethodNotAllowed) {
+            status
+          }
+        }
     }
-  }
 
-  it should "return 405 with POST of outputs on successful execution of workflow" in {
-    Post(s"/workflows/$version/${MockApiService.UnrecognizedWorkflowId}/outputs") ~>
-      cromwellApiService.sealRoute(cromwellApiService.workflowOutputsRoute) ~>
-      check {
-        assertResult(StatusCodes.MethodNotAllowed) {
-          status
+    behavior of "REST API /logs endpoint"
+    it should "return 200 with paths to stdout/stderr/backend log" in {
+      Get(s"/workflows/$version/${CromwellApiServiceSpec.ExistingWorkflowId}/logs") ~>
+        akkaHttpService.workflowRoutes ~>
+        check {
+          status should be(StatusCodes.OK)
+
+          val call = responseAs[JsObject].fields("calls").convertTo[JsObject].fields("mycall").convertTo[Seq[JsObject]].head
+          call.fields("stdout") should be(JsString("stdout.txt"))
+          call.fields("stderr") should be(JsString("stderr.txt"))
+          call.fields("stdout") should be(JsString("stdout.txt"))
+          call.fields("backendLogs").convertTo[JsObject].fields("log") should be (JsString("backend.log"))
         }
-      }
-  }
+    }
 
-  behavior of "REST API /logs endpoint"
-  it should "return 200 with paths to stdout/stderr/backend log" in {
-    Get(s"/workflows/$version/${MockApiService.ExistingWorkflowId}/logs") ~>
-      cromwellApiService.workflowLogsRoute ~>
-      check {
-        status should be(StatusCodes.OK)
-        val result = responseAs[JsObject]
+    it should "return 404 with logs on unknown workflow" in {
+      Get(s"/workflows/$version/${CromwellApiServiceSpec.UnrecognizedWorkflowId}/logs") ~>
+        akkaHttpService.workflowRoutes ~>
+        check {
+          assertResult(StatusCodes.NotFound) {
+            status
+          }
+        }
+    }
 
-        val call = result.fields("calls").convertTo[JsObject].fields("mycall").convertTo[Seq[JsObject]].head
-        call.fields("stdout") should be(JsString("stdout.txt"))
-        call.fields("stderr") should be(JsString("stderr.txt"))
-        call.fields("stdout") should be(JsString("stdout.txt"))
-        call.fields("backendLogs").convertTo[JsObject].fields("log") should be (JsString("backend.log"))
-      }
-  }
+    behavior of "REST API /metadata endpoint"
+    it should "return with full metadata from the metadata route" in {
+      Get(s"/workflows/$version/${CromwellApiServiceSpec.ExistingWorkflowId}/metadata") ~>
+        akkaHttpService.workflowRoutes ~>
+        check {
+          status should be(StatusCodes.OK)
+          val result = responseAs[JsObject]
+          result.fields.keys should contain allOf("testKey1", "testKey2")
+          result.fields.keys shouldNot contain("testKey3")
+          result.fields("testKey1") should be(JsString("myValue1"))
+          result.fields("testKey2") should be(JsString("myValue2"))
+        }
+    }
 
-  it should "return 404 with logs on unknown workflow" in {
-    Get(s"/workflows/$version/${MockApiService.UnrecognizedWorkflowId}/logs") ~>
-      cromwellApiService.workflowLogsRoute ~>
-      check {
-        assertResult(StatusCodes.NotFound) {
-          status
+    it should "return with gzip encoding when requested" in {
+      Get(s"/workflows/$version/${CromwellApiServiceSpec.ExistingWorkflowId}/metadata").addHeader(`Accept-Encoding`(HttpEncodings.gzip)) ~>
+        akkaHttpService.workflowRoutes ~>
+        check {
+          response.headers.find(_.name == "Content-Encoding").get.value should be("gzip")
         }
-      }
-  }
+    }
 
-  behavior of "REST API /metadata endpoint"
-  it should "return with full metadata from the metadata route" in {
-    Get(s"/workflows/$version/${MockApiService.ExistingWorkflowId}/metadata") ~>
-      cromwellApiService.metadataRoute ~>
-      check {
-        status should be(StatusCodes.OK)
-        val result = responseAs[JsObject]
-        result.fields.keys should contain allOf("testKey1", "testKey2")
-        result.fields.keys shouldNot contain("testKey3")
-        result.fields("testKey1") should be(JsString("myValue1"))
-        result.fields("testKey2") should be(JsString("myValue2"))
-      }
-  }
+    it should "not return with gzip encoding when not requested" in {
+      Get(s"/workflows/$version/${CromwellApiServiceSpec.ExistingWorkflowId}/metadata") ~>
+        akkaHttpService.workflowRoutes ~>
+        check {
+          response.headers.find(_.name == "Content-Encoding") shouldBe None
+        }
+    }
 
-  it should "return with included metadata from the metadata route" in {
-    Get(s"/workflows/$version/${MockApiService.ExistingWorkflowId}/metadata?includeKey=testKey1&includeKey=testKey2a") ~>
-      cromwellApiService.metadataRoute ~>
-      check {
-        status should be(StatusCodes.OK)
-        val result = responseAs[JsObject]
-        result.fields.keys should contain allOf("testKey1a", "testKey1b", "testKey2a")
-        result.fields.keys should contain noneOf("testKey2b", "testKey3")
-        result.fields("testKey1a") should be(JsString("myValue1a"))
-        result.fields("testKey1b") should be(JsString("myValue1b"))
-        result.fields("testKey2a") should be(JsString("myValue2a"))
-      }
-  }
+    it should "return with included metadata from the metadata route" in {
+      Get(s"/workflows/$version/${CromwellApiServiceSpec.ExistingWorkflowId}/metadata?includeKey=testKey1&includeKey=testKey2a") ~>
+        akkaHttpService.workflowRoutes ~>
+        check {
+          status should be(StatusCodes.OK)
+          val result = responseAs[JsObject]
+          result.fields.keys should contain allOf("testKey1a", "testKey1b", "testKey2a")
+          result.fields.keys should contain noneOf("testKey2b", "testKey3")
+          result.fields("testKey1a") should be(JsString("myValue1a"))
+          result.fields("testKey1b") should be(JsString("myValue1b"))
+          result.fields("testKey2a") should be(JsString("myValue2a"))
+        }
+    }
 
     it should "return with excluded metadata from the metadata route" in {
-     Get(s"/workflows/$version/${MockApiService.ExistingWorkflowId}/metadata?excludeKey=testKey2b&excludeKey=testKey3") ~>
-        cromwellApiService.metadataRoute ~>
+     Get(s"/workflows/$version/${CromwellApiServiceSpec.ExistingWorkflowId}/metadata?excludeKey=testKey2b&excludeKey=testKey3") ~>
+       akkaHttpService.workflowRoutes ~>
         check {
           status should be(StatusCodes.OK)
           val result = responseAs[JsObject]
@@ -325,163 +397,189 @@ class CromwellApiServiceSpec extends FlatSpec with ScalatestRouteTest with Match
         }
     }
 
-  it should "return an error when included and excluded metadata requested from the metadata route" in {
-    Get(s"/workflows/$version/${MockApiService.ExistingWorkflowId}/metadata?includeKey=testKey1&excludeKey=testKey2") ~>
-      cromwellApiService.metadataRoute ~>
-      check {
-        assertResult(StatusCodes.BadRequest) {
-          status
+    it should "return an error when included and excluded metadata requested from the metadata route" in {
+      Get(s"/workflows/$version/${CromwellApiServiceSpec.ExistingWorkflowId}/metadata?includeKey=testKey1&excludeKey=testKey2") ~>
+        akkaHttpService.workflowRoutes ~>
+        check {
+          assertResult(StatusCodes.BadRequest) {
+            status
+          }
+
+          val decoder: Decoder = Gzip
+          Unmarshal(decoder.decodeMessage(response)).to[String] map { r =>
+            assertResult(
+              s"""{
+                  |  "status": "fail",
+                  |  "message": "includeKey and excludeKey may not be specified together"
+                  |}""".stripMargin
+            ) { r }
+          }
         }
-        assertResult(
-          s"""{
-              |  "status": "fail",
-              |  "message": "includeKey and excludeKey may not be specified together"
-              |}""".stripMargin
-        ) {
-          responseAs[String]
+    }
+
+    behavior of "REST API /timing endpoint"
+    it should "return 200 with an HTML document for the timings route" in {
+      Get(s"/workflows/$version/${CromwellApiServiceSpec.ExistingWorkflowId}/timing") ~>
+        akkaHttpService.workflowRoutes ~>
+        check {
+          assertResult(StatusCodes.OK) { status }
+          assertResult("<html>") {
+            responseAs[String].substring(0, 6)
+          }
         }
-      }
-  }
+    }
 
-  behavior of "REST API /timing endpoint"
-  it should "return 200 with an HTML document for the timings route" in {
-    Get(s"/workflows/$version/${MockApiService.ExistingWorkflowId}/timing") ~>
-      cromwellApiService.timingRoute ~>
-      check {
-        assertResult(StatusCodes.OK) { status }
-        assertResult("<html>") {
-          responseAs[String].substring(0, 6)
+    behavior of "REST API /query GET endpoint"
+    it should "return good results for a good query" in {
+      Get(s"/workflows/$version/query?status=Succeeded&id=${CromwellApiServiceSpec.ExistingWorkflowId}") ~>
+        akkaHttpService.workflowRoutes ~>
+        check {
+          status should be(StatusCodes.OK)
+          contentType should be(ContentTypes.`application/json`)
+          val results = responseAs[JsObject].fields("results").convertTo[Seq[JsObject]]
+          results.head.fields("id") should be(JsString(CromwellApiServiceSpec.ExistingWorkflowId.toString))
+          results.head.fields("status") should be(JsString("Succeeded"))
         }
-      }
-  }
+    }
 
-  behavior of "REST API /query GET endpoint"
-  it should "return good results for a good query" in {
-    Get(s"/workflows/$version/query?status=Succeeded&id=${MockApiService.ExistingWorkflowId}") ~>
-      cromwellApiService.queryRoute ~>
-      check {
-        status should be(StatusCodes.OK)
-        val results = responseAs[JsObject].fields("results").convertTo[Seq[JsObject]]
+    behavior of "REST API /query POST endpoint"
+    it should "return good results for a good query map body" in {
+      Post(s"/workflows/$version/query", HttpEntity(ContentTypes.`application/json`, """[{"status":"Succeeded"}]""")) ~>
+        akkaHttpService.workflowRoutes ~>
+        check {
+          assertResult(StatusCodes.OK) {
+            status
+          }
+          assertResult(true) {
+            entityAs[String].contains("\"status\":\"Succeeded\"")
+          }
+        }
+    }
 
-        results.head.fields("id") should be(JsString(MockApiService.ExistingWorkflowId.toString))
-        results.head.fields("status") should be(JsString("Succeeded"))
-      }
-  }
+    behavior of "REST API /labels PATCH endpoint"
+    it should "return successful status response when assigning valid labels to an existing workflow ID" in {
 
-  behavior of "REST API /query POST endpoint"
-  it should "return good results for a good query map body" in {
-    Post(s"/workflows/$version/query", HttpEntity(ContentTypes.`application/json`, """[{"status":"Succeeded"}]""")) ~>
-      cromwellApiService.queryPostRoute ~>
-      check {
-        assertResult(StatusCodes.OK) {
-          status
-        }
-        assertResult(true) {
-          body.asString.contains("\"status\": \"Succeeded\"")
+      val validLabelsJson =
+        """
+          |{
+          |  "label-key-1":"label-value-1",
+          |  "label-key-2":"label-value-2"
+          |}
+        """.stripMargin
+
+      val workflowId = CromwellApiServiceSpec.ExistingWorkflowId
+
+      Patch(s"/workflows/$version/$workflowId/labels", HttpEntity(ContentTypes.`application/json`, validLabelsJson)) ~>
+        akkaHttpService.workflowRoutes ~>
+        check {
+          status shouldBe StatusCodes.OK
+          val actualResult = responseAs[JsObject]
+          val expectedResults =
+            s"""
+              |{
+              |  "id": "$workflowId",
+              |  "labels": {
+              |    "label-key-1":"label-value-1",
+              |    "label-key-2":"label-value-2"
+              |  }
+              |}
+            """.stripMargin.parseJson
+
+          actualResult shouldBe expectedResults
         }
-      }
-  }
+    }
 }
 
 object CromwellApiServiceSpec {
-  class MockApiService()(implicit val system: ActorSystem) extends CromwellApiService {
-    import MockApiService._
+  val ExistingWorkflowId = WorkflowId.fromString("c4c6339c-8cc9-47fb-acc5-b5cb8d2809f5")
+  val AbortedWorkflowId = WorkflowId.fromString("0574111c-c7d3-4145-8190-7a7ed8e8324a")
+  val UnrecognizedWorkflowId = WorkflowId.fromString("2bdd06cc-e794-46c8-a897-4c86cedb6a06")
+  val RecognizedWorkflowIds = Set(ExistingWorkflowId, AbortedWorkflowId)
 
+  class MockApiService()(implicit val system: ActorSystem) extends CromwellApiService {
     override def actorRefFactory = system
+
+    override val materializer = ActorMaterializer()
+    override val ec = system.dispatcher
     override val workflowStoreActor = actorRefFactory.actorOf(Props(new MockWorkflowStoreActor()))
-    override val serviceRegistryActor = actorRefFactory.actorOf(Props.empty)
-    override val workflowManagerActor = actorRefFactory.actorOf(Props.empty)
-
-    override def handleMetadataRequest(message: AnyRef): Route = {
-      message match {
-        case GetStatus(w) => complete(submittedStatusResponse(w))
-        case WorkflowOutputs(w) => complete(outputResponse(w))
-        case GetLogs(w) => complete(logsResponse(w))
-        case GetSingleWorkflowMetadataAction(w, None, None, _) => complete(fullMetadataResponse(w))
-        case GetSingleWorkflowMetadataAction(w, Some(i), None, _) => complete(filteredMetadataResponse(w))
-        case GetSingleWorkflowMetadataAction(w, None, Some(_), _) => complete(filteredMetadataResponse(w))
-        case _ => throw new IllegalArgumentException("Woopsie!")
-      }
-    }
+    override val serviceRegistryActor = actorRefFactory.actorOf(Props(new MockServiceRegistryActor()))
+    override val workflowManagerActor = actorRefFactory.actorOf(Props(new MockWorkflowManagerActor()))
+  }
 
-    override def handleQueryMetadataRequest(parameters: Seq[(String, String)]): Route = {
-      complete(QueryResponse)
+  object MockServiceRegistryActor {
+    def fullMetadataResponse(workflowId: WorkflowId) = {
+      List(MetadataEvent(MetadataKey(workflowId, None, "testKey1"), MetadataValue("myValue1", MetadataString)),
+        MetadataEvent(MetadataKey(workflowId, None, "testKey2"), MetadataValue("myValue2", MetadataString)))
+    }
+    def filteredMetadataResponse(workflowId: WorkflowId) = {
+      List(MetadataEvent(MetadataKey(workflowId, None, "testKey1a"), MetadataValue("myValue1a", MetadataString)),
+        MetadataEvent(MetadataKey(workflowId, None, "testKey1b"), MetadataValue("myValue1b", MetadataString)),
+        MetadataEvent(MetadataKey(workflowId, None, "testKey2a"), MetadataValue("myValue2a", MetadataString)))
     }
 
-    override def withRecognizedWorkflowId(possibleWorkflowId: String)(recognizedWorkflowId: WorkflowId => Route): Route = {
-      requestContext =>
-        Try(WorkflowId.fromString(possibleWorkflowId)) match {
-          case Success(workflowId) =>
-              if (RecognizedWorkflowIds.contains(workflowId)) recognizedWorkflowId(workflowId)(requestContext)
-              else failBadRequest(new NoSuchElementException("A hollow voice says 'fool'"), StatusCodes.NotFound)(requestContext)
-          case Failure(e) =>
-            failBadRequest(new RuntimeException(s"Invalid workflow ID: '$possibleWorkflowId'."))(requestContext)
-        }
+    def metadataQuery(workflowId: WorkflowId) = MetadataQuery(workflowId, None, None, None, None, false)
+
+    def logsEvents(id: WorkflowId) = {
+      val stdout = MetadataEvent(MetadataKey(id, Some(MetadataJobKey("mycall", None, 1)), CallMetadataKeys.Stdout), MetadataValue("stdout.txt", MetadataString))
+      val stderr = MetadataEvent(MetadataKey(id, Some(MetadataJobKey("mycall", None, 1)), CallMetadataKeys.Stderr), MetadataValue("stderr.txt", MetadataString))
+      val backend = MetadataEvent(MetadataKey(id, Some(MetadataJobKey("mycall", None, 1)), s"${CallMetadataKeys.BackendLogsPrefix}:log"), MetadataValue("backend.log", MetadataString))
+      Vector(stdout, stderr, backend)
     }
   }
 
-  object MockApiService {
-    val ExistingWorkflowId = WorkflowId.fromString("c4c6339c-8cc9-47fb-acc5-b5cb8d2809f5")
-    val AbortedWorkflowId = WorkflowId.fromString("0574111c-c7d3-4145-8190-7a7ed8e8324a")
-    val UnrecognizedWorkflowId = WorkflowId.fromString("2bdd06cc-e794-46c8-a897-4c86cedb6a06")
-    val RecognizedWorkflowIds = Set(ExistingWorkflowId, AbortedWorkflowId)
-
-    val QueryResponse = JsObject(Map(
-      "results" -> JsArray(JsObject(Map(
-        WorkflowMetadataKeys.Id -> JsString(ExistingWorkflowId.toString),
-        WorkflowMetadataKeys.Status -> JsString(WorkflowSucceeded.toString)
-      )))
-    ))
-
-    def fullMetadataResponse(workflowId: WorkflowId) = JsObject(Map(
-      "testKey1" -> JsString("myValue1"),
-      "testKey2" -> JsString("myValue2")
-    ))
-
-    def filteredMetadataResponse(workflowId: WorkflowId) = JsObject(Map(
-      "testKey1a" -> JsString("myValue1a"),
-      "testKey1b" -> JsString("myValue1b"),
-      "testKey2a" -> JsString("myValue2a")
-    ))
-
-    def submittedStatusResponse(workflowId: WorkflowId) = JsObject(Map(
-      WorkflowMetadataKeys.Status -> JsString(WorkflowSubmitted.toString),
-      WorkflowMetadataKeys.Id -> JsString(workflowId.toString)
-    ))
-
-    def outputResponse(workflowId: WorkflowId) = JsObject(Map(
-      WorkflowMetadataKeys.Id -> JsString(workflowId.toString),
-      WorkflowMetadataKeys.Outputs -> JsString("Some random stuff")
-    ))
-
-    def logsResponse(workflowId: WorkflowId) = JsObject(Map(
-      "calls" -> JsObject(Map(
-        "mycall" -> JsArray(JsObject(Map(
-          "stdout" -> JsString("stdout.txt"),
-          "stderr" -> JsString("stderr.txt"),
-          "backendLogs" -> JsObject(Map(
-            "log" -> JsString("backend.log")
-          ))
-        )))
-      ))
-    ))
+  class MockServiceRegistryActor extends Actor {
+    import MockServiceRegistryActor._
+    override def receive = {
+      case WorkflowQuery(_) =>
+        val response = WorkflowQuerySuccess(WorkflowQueryResponse(List(WorkflowQueryResult(ExistingWorkflowId.toString,
+          None, Some(WorkflowSucceeded.toString), None, None))), None)
+        sender ! response
+      case ValidateWorkflowId(id) =>
+        if (RecognizedWorkflowIds.contains(id)) sender ! MetadataService.RecognizedWorkflowId
+        else sender ! MetadataService.UnrecognizedWorkflowId
+      case GetStatus(id) => sender ! StatusLookupResponse(id, WorkflowSubmitted)
+      case WorkflowOutputs(id) =>
+        val event = Vector(MetadataEvent(MetadataKey(id, None, "outputs:test.hello.salutation"), MetadataValue("Hello foo!", MetadataString)))
+        sender ! WorkflowOutputsResponse(id, event)
+      case GetLogs(id) => sender ! LogsResponse(id, logsEvents(id))
+      case GetSingleWorkflowMetadataAction(id, None, None, _) => sender ! MetadataLookupResponse(metadataQuery(id), fullMetadataResponse(id))
+      case GetSingleWorkflowMetadataAction(id, Some(_), None, _) => sender ! MetadataLookupResponse(metadataQuery(id), filteredMetadataResponse(id))
+      case GetSingleWorkflowMetadataAction(id, None, Some(_), _) => sender ! MetadataLookupResponse(metadataQuery(id), filteredMetadataResponse(id))
+      case PutMetadataActionAndRespond(events, _) =>
+        events.head.key.workflowId match {
+          case CromwellApiServiceSpec.ExistingWorkflowId => sender ! MetadataWriteSuccess(events)
+          case CromwellApiServiceSpec.AbortedWorkflowId => sender ! MetadataWriteFailure(new Exception("mock exception of db failure"), events)
+          case WorkflowId(_) => throw new Exception("Something untoward happened, this situation is not believed to be possible at this time")
+        }
+    }
   }
 
   class MockWorkflowStoreActor extends Actor {
     override def receive = {
-      case SubmitWorkflow(source) => sender ! WorkflowSubmittedToStore(MockApiService.ExistingWorkflowId)
+      case SubmitWorkflow(_) => sender ! WorkflowSubmittedToStore(ExistingWorkflowId)
       case BatchSubmitWorkflows(sources) =>
-        val response = WorkflowsBatchSubmittedToStore(sources map { _ => MockApiService.ExistingWorkflowId })
+        val response = WorkflowsBatchSubmittedToStore(sources map { _ => ExistingWorkflowId })
         sender ! response
-      case AbortWorkflow(id, manager) =>
+      case AbortWorkflow(id, manager @ _) =>
         val message = id match {
-          case MockApiService.ExistingWorkflowId =>
-            WorkflowStoreEngineActor.WorkflowAborted(id)
-          case MockApiService.AbortedWorkflowId   =>
+          case ExistingWorkflowId => WorkflowStoreEngineActor.WorkflowAborted(id)
+          case AbortedWorkflowId =>
             WorkflowAbortFailed(id, new IllegalStateException(s"Workflow ID '$id' is in terminal state 'Aborted' and cannot be aborted."))
+          case WorkflowId(_) => throw new Exception("Something untoward happened")
         }
-
         sender ! message
     }
   }
+
+  class MockWorkflowManagerActor extends Actor with ActorLogging {
+    override def receive: Receive = {
+      case WorkflowManagerActor.EngineStatsCommand =>
+        val response = EngineStatsActor.EngineStats(1, 23)
+        sender ! response
+      case unexpected =>
+        val sndr = sender()
+        log.error(s"Unexpected message {} from {}", unexpected, sndr)
+        sender ! s"Unexpected message received: $unexpected"
+    }
+  }
 }
diff --git a/engine/src/test/scala/cromwell/webservice/MetadataBuilderActorSpec.scala b/engine/src/test/scala/cromwell/webservice/MetadataBuilderActorSpec.scala
index 9ecfeea42..9fb608dcf 100644
--- a/engine/src/test/scala/cromwell/webservice/MetadataBuilderActorSpec.scala
+++ b/engine/src/test/scala/cromwell/webservice/MetadataBuilderActorSpec.scala
@@ -4,42 +4,42 @@ import java.time.OffsetDateTime
 import java.util.UUID
 
 import akka.testkit._
+import akka.pattern.ask
+import akka.util.Timeout
 import cromwell.core.{TestKitSuite, WorkflowId}
 import cromwell.services.metadata.MetadataService._
 import cromwell.services.metadata._
-import cromwell.webservice.PerRequest.RequestComplete
 import cromwell.webservice.metadata.MetadataBuilderActor
+import cromwell.webservice.metadata.MetadataBuilderActor.{BuiltMetadataResponse, MetadataBuilderActorResponse}
 import org.scalatest.prop.TableDrivenPropertyChecks
-import org.scalatest.{FlatSpecLike, Matchers}
-import spray.http.{StatusCode, StatusCodes}
+import org.scalatest.{Assertion, AsyncFlatSpecLike, Matchers, Succeeded}
+import org.specs2.mock.Mockito
 import spray.json._
 
+import scala.concurrent.Future
 import scala.concurrent.duration._
 import scala.language.postfixOps
 
-class MetadataBuilderActorSpec extends TestKitSuite("Metadata") with FlatSpecLike with Matchers
+class MetadataBuilderActorSpec extends TestKitSuite("Metadata") with AsyncFlatSpecLike with Matchers with Mockito
   with TableDrivenPropertyChecks with ImplicitSender {
 
   behavior of "MetadataParser"
 
   val defaultTimeout = 200 millis
+  implicit val timeout: Timeout = defaultTimeout
+
   val mockServiceRegistry = TestProbe()
 
   def assertMetadataResponse(action: MetadataServiceAction,
                              queryReply: MetadataQuery,
                              events: Seq[MetadataEvent],
-                             expectedRes: String) = {
-    val parentProbe = TestProbe()
-    val metadataBuilder = TestActorRef(MetadataBuilderActor.props(mockServiceRegistry.ref), parentProbe.ref, s"MetadataActor-${UUID.randomUUID()}")
-    metadataBuilder ! action // Ask for everything
-    mockServiceRegistry.expectMsg(defaultTimeout, action) // TestActor runs on CallingThreadDispatcher
+                             expectedRes: String): Future[Assertion] = {
+    val mba = system.actorOf(MetadataBuilderActor.props(mockServiceRegistry.ref))
+    val response = mba.ask(action).mapTo[MetadataBuilderActorResponse]
+    mockServiceRegistry.expectMsg(defaultTimeout, action)
     mockServiceRegistry.reply(MetadataLookupResponse(queryReply, events))
-
-    parentProbe.expectMsgPF(defaultTimeout) {
-      case response: RequestComplete[(StatusCode, JsObject)] @unchecked =>
-        response.response._1 shouldBe StatusCodes.OK
-        response.response._2 shouldBe expectedRes.parseJson
-    }
+    response map { r => r shouldBe a [BuiltMetadataResponse] }
+    response.mapTo[BuiltMetadataResponse] map { b => b.response shouldBe expectedRes.parseJson}
   }
 
   it should "build workflow scope tree from metadata events" in {
@@ -102,14 +102,21 @@ class MetadataBuilderActorSpec extends TestKitSuite("Metadata") with FlatSpecLik
 
   type EventBuilder = (String, String, OffsetDateTime)
 
-  def makeEvent(workflow: WorkflowId)(key: String, value: MetadataValue, offsetDateTime: OffsetDateTime) = {
+  def makeEvent(workflow: WorkflowId)(key: String, value: MetadataValue, offsetDateTime: OffsetDateTime): MetadataEvent = {
     MetadataEvent(MetadataKey(workflow, None, key), Option(value), offsetDateTime)
   }
 
-  def assertMetadataKeyStructure(eventList: List[EventBuilder], expectedJson: String) = {
-    val workflow = WorkflowId.randomId()
+  def makeCallEvent(workflow: WorkflowId)(key: String, value: MetadataValue, offsetDateTime: OffsetDateTime) = {
+    val jobKey = MetadataJobKey("fqn", None, 1)
+    MetadataEvent(MetadataKey(workflow, Option(jobKey), key), Option(value), offsetDateTime)
+  }
+
+  def assertMetadataKeyStructure(eventList: List[EventBuilder],
+                                 expectedJson: String,
+                                 workflow: WorkflowId = WorkflowId.randomId(),
+                                 eventMaker: WorkflowId => (String, MetadataValue, OffsetDateTime) => MetadataEvent = makeEvent) = {
 
-    val events = eventList map { e => (e._1, MetadataValue(e._2), e._3) } map Function.tupled(makeEvent(workflow))
+    val events = eventList map { e => (e._1, MetadataValue(e._2), e._3) } map Function.tupled(eventMaker(workflow))
     val expectedRes = s"""{ "calls": {}, $expectedJson, "id":"$workflow" }"""
 
     val mdQuery = MetadataQuery(workflow, None, None, None, None, expandSubWorkflows = false)
@@ -117,18 +124,18 @@ class MetadataBuilderActorSpec extends TestKitSuite("Metadata") with FlatSpecLik
     assertMetadataResponse(queryAction, mdQuery, events, expectedRes)
   }
 
-  it should "keep event with later timestamp for the same key in metadata" in {
+  it should "assume the event list is ordered and keep last event if 2 events have same key" in {
     val eventBuilderList = List(
       ("a", "aLater", OffsetDateTime.parse("2000-01-02T12:00:00Z")),
       ("a", "a", OffsetDateTime.parse("2000-01-01T12:00:00Z"))
     )
     val expectedRes =
-      """"a": "aLater"""".stripMargin
+      """"a": "a"""".stripMargin
 
     assertMetadataKeyStructure(eventBuilderList, expectedRes)
   }
 
-  it should "use CRDT ordering instead of timestamp for status" in {
+  it should "use CRDT ordering instead of timestamp for workflow state" in {
     val eventBuilderList = List(
       ("status", "Succeeded", OffsetDateTime.now),
       ("status", "Running", OffsetDateTime.now.plusSeconds(1))
@@ -139,6 +146,25 @@ class MetadataBuilderActorSpec extends TestKitSuite("Metadata") with FlatSpecLik
     assertMetadataKeyStructure(eventBuilderList, expectedRes)
   }
 
+  it should "use CRDT ordering instead of timestamp for call execution status" in {
+    val eventBuilderList = List(
+      ("executionStatus", "Done", OffsetDateTime.now),
+      ("executionStatus", "Running", OffsetDateTime.now.plusSeconds(1))
+    )
+    val workflowId = WorkflowId.randomId()
+    val expectedRes =
+      s""""calls": {
+        |    "fqn": [{
+        |      "attempt": 1,
+        |      "executionStatus": "Done",
+        |      "shardIndex": -1
+        |    }]
+        |  },
+        |  "id": "$workflowId"""".stripMargin
+
+    assertMetadataKeyStructure(eventBuilderList, expectedRes, workflowId, makeCallEvent)
+  }
+
   it should "build JSON object structure from dotted key syntax" in {
     val eventBuilderList = List(
       ("a:b:c", "abc", OffsetDateTime.now),
@@ -265,15 +291,16 @@ class MetadataBuilderActorSpec extends TestKitSuite("Metadata") with FlatSpecLik
     val kisv3 = ("key[0]:subkey", "value3", OffsetDateTime.now.plusSeconds(2))
     val kiv4 = ("key[0]", "value4", OffsetDateTime.now.plusSeconds(3))
 
-    val t = Table(
-      ("list", "res"),
+    val t = List(
       (List(kv),  """"key": "value""""),
       (List(kv, ksv2),  """"key": { "subkey": "value2" }"""),
       (List(kv, ksv2, kisv3),  """"key": [ { "subkey": "value3" } ]"""),
       (List(kv, ksv2, kisv3, kiv4),  """"key": [ "value4" ]""")
     )
 
-    forAll(t) { (l, r) => assertMetadataKeyStructure(l, r) }
+    Future.sequence(t map { case (l, r) => assertMetadataKeyStructure(l, r) }) map { assertions =>
+      assertions should contain only Succeeded
+    }
   }
 
   it should "coerce values to supported types" in {
@@ -418,7 +445,7 @@ class MetadataBuilderActorSpec extends TestKitSuite("Metadata") with FlatSpecLik
     
     val parentProbe = TestProbe()
     val metadataBuilder = TestActorRef(MetadataBuilderActor.props(mockServiceRegistry.ref), parentProbe.ref, s"MetadataActor-${UUID.randomUUID()}")
-    metadataBuilder ! mainQueryAction
+    val response = metadataBuilder.ask(mainQueryAction).mapTo[MetadataBuilderActorResponse]
     mockServiceRegistry.expectMsg(defaultTimeout, mainQueryAction)
     mockServiceRegistry.reply(MetadataLookupResponse(mainQuery, mainEvents))
     mockServiceRegistry.expectMsg(defaultTimeout, subQueryAction)
@@ -444,11 +471,9 @@ class MetadataBuilderActorSpec extends TestKitSuite("Metadata") with FlatSpecLik
          |}
        """.stripMargin
 
-    parentProbe.expectMsgPF(defaultTimeout) {
-      case response: RequestComplete[(StatusCode, JsObject)] @unchecked =>
-        response.response._1 shouldBe StatusCodes.OK
-        response.response._2 shouldBe expandedRes.parseJson
-    }
+    response map { r => r shouldBe a [BuiltMetadataResponse] }
+    val bmr = response.mapTo[BuiltMetadataResponse]
+    bmr map { b => b.response shouldBe expandedRes.parseJson}
   }
   
   it should "NOT expand sub workflow metadata when NOT asked for" in {
@@ -464,7 +489,7 @@ class MetadataBuilderActorSpec extends TestKitSuite("Metadata") with FlatSpecLik
     
     val parentProbe = TestProbe()
     val metadataBuilder = TestActorRef(MetadataBuilderActor.props(mockServiceRegistry.ref), parentProbe.ref, s"MetadataActor-${UUID.randomUUID()}")
-    metadataBuilder ! queryNoExpandAction
+    val response = metadataBuilder.ask(queryNoExpandAction).mapTo[MetadataBuilderActorResponse]
     mockServiceRegistry.expectMsg(defaultTimeout, queryNoExpandAction)
     mockServiceRegistry.reply(MetadataLookupResponse(queryNoExpand, mainEvents))
 
@@ -484,11 +509,10 @@ class MetadataBuilderActorSpec extends TestKitSuite("Metadata") with FlatSpecLik
          |  "id": "$mainWorkflowId"
          |}
        """.stripMargin
-    
-    parentProbe.expectMsgPF(defaultTimeout) {
-      case response: RequestComplete[(StatusCode, JsObject)] @unchecked =>
-        response.response._1 shouldBe StatusCodes.OK
-        response.response._2 shouldBe nonExpandedRes.parseJson
-    }
+
+    response map { r => r shouldBe a [BuiltMetadataResponse] }
+    val bmr = response.mapTo[BuiltMetadataResponse]
+    bmr map { b => b.response shouldBe nonExpandedRes.parseJson}
+
   }
 }
diff --git a/engine/src/test/scala/cromwell/webservice/PartialWorkflowSourcesSpec.scala b/engine/src/test/scala/cromwell/webservice/PartialWorkflowSourcesSpec.scala
new file mode 100644
index 000000000..cd7422ffc
--- /dev/null
+++ b/engine/src/test/scala/cromwell/webservice/PartialWorkflowSourcesSpec.scala
@@ -0,0 +1,17 @@
+package cromwell.webservice
+
+import org.scalatest.{FlatSpec, Matchers}
+import spray.json._
+import spray.json.DefaultJsonProtocol._
+
+class PartialWorkflowSourcesSpec extends FlatSpec with Matchers {
+  it should "succesfully merge and override multiple input files" in {
+    val input1 = Map("wf.a1" -> "hello", "wf.a2" -> "world").toJson.toString
+    val input2 = Map.empty[String, String].toJson.toString
+    val overrideInput1 = Map("wf.a2" -> "universe").toJson.toString
+    val allInputs = PartialWorkflowSources.mergeMaps(Seq(Option(input1), Option(input2), Option(overrideInput1)))
+
+    allInputs.fields.keys should contain allOf("wf.a1", "wf.a2")
+    allInputs.fields("wf.a2") should be(JsString("universe"))
+  }
+}
diff --git a/engine/src/test/scala/cromwell/webservice/SwaggerServiceSpec.scala b/engine/src/test/scala/cromwell/webservice/SwaggerServiceSpec.scala
index 18a8f81af..20c7c3e6d 100644
--- a/engine/src/test/scala/cromwell/webservice/SwaggerServiceSpec.scala
+++ b/engine/src/test/scala/cromwell/webservice/SwaggerServiceSpec.scala
@@ -1,5 +1,7 @@
 package cromwell.webservice
 
+import akka.http.scaladsl.model.StatusCodes
+import akka.http.scaladsl.testkit.ScalatestRouteTest
 import io.swagger.models.properties.RefProperty
 import io.swagger.parser.SwaggerParser
 import org.scalatest.prop.TableDrivenPropertyChecks
@@ -8,8 +10,6 @@ import org.yaml.snakeyaml.constructor.Constructor
 import org.yaml.snakeyaml.error.YAMLException
 import org.yaml.snakeyaml.nodes.MappingNode
 import org.yaml.snakeyaml.{Yaml => SnakeYaml}
-import spray.http._
-import spray.testkit.ScalatestRouteTest
 
 import scala.collection.JavaConverters._
 
diff --git a/engine/src/test/scala/cromwell/webservice/SwaggerUiHttpServiceSpec.scala b/engine/src/test/scala/cromwell/webservice/SwaggerUiHttpServiceSpec.scala
new file mode 100644
index 000000000..292d5ad73
--- /dev/null
+++ b/engine/src/test/scala/cromwell/webservice/SwaggerUiHttpServiceSpec.scala
@@ -0,0 +1,320 @@
+package cromwell.webservice
+
+import akka.http.scaladsl.model.{StatusCodes, Uri}
+import akka.http.scaladsl.model.headers.Location
+import akka.http.scaladsl.server.Route
+import akka.http.scaladsl.testkit.ScalatestRouteTest
+import com.typesafe.config.ConfigFactory
+import cromwell.webservice.SwaggerUiHttpServiceSpec._
+import org.scalatest.prop.TableDrivenPropertyChecks
+import org.scalatest.{FlatSpec, Matchers}
+
+
+trait SwaggerUiHttpServiceSpec extends FlatSpec with Matchers with ScalatestRouteTest with SwaggerUiHttpService {
+  override def swaggerUiVersion = TestSwaggerUiVersion
+}
+
+trait SwaggerResourceHttpServiceSpec extends FlatSpec with Matchers with ScalatestRouteTest with
+TableDrivenPropertyChecks with SwaggerResourceHttpService {
+  val testPathsForOptions = Table("endpoint", "/", "/swagger", "/swagger/index.html", "/api", "/api/example",
+    "/api/example?with=param", "/api/example/path")
+}
+
+trait SwaggerUiResourceHttpServiceSpec extends SwaggerUiHttpServiceSpec with SwaggerResourceHttpServiceSpec with
+SwaggerUiResourceHttpService
+
+object SwaggerUiHttpServiceSpec {
+  val TestSwaggerUiVersion = "2.1.1"
+  val SwaggerIndexPreamble =
+    """
+      |<!DOCTYPE html>
+      |<html>
+      |<head>
+      |  <meta charset="UTF-8">
+      |  <title>Swagger UI</title>""".stripMargin.trim // workaround IDEA's weird formatting of interpolated strings
+}
+
+class BasicSwaggerUiHttpServiceSpec extends SwaggerUiHttpServiceSpec {
+  behavior of "SwaggerUiHttpService"
+
+  it should "redirect / to /swagger" in {
+    Get() ~> swaggerUiRoute ~> check {
+      status should be(StatusCodes.TemporaryRedirect)
+      header("Location") should be(Option(Location(Uri("/swagger"))))
+    }
+  }
+
+  it should "not return options for /" in {
+    Options() ~> Route.seal(swaggerUiRoute) ~> check {
+      status should be(StatusCodes.MethodNotAllowed)
+    }
+  }
+
+  it should "redirect /swagger to the index.html" in {
+    Get("/swagger") ~> swaggerUiRoute ~> check {
+      status should be(StatusCodes.TemporaryRedirect)
+      header("Location") should be(Option(Location(Uri("/swagger/index.html?url=/api-docs"))))
+    }
+  }
+
+  it should "return index.html from the swagger-ui jar" in {
+    Get("/swagger/index.html") ~> swaggerUiRoute ~> check {
+      status should be(StatusCodes.OK)
+    }
+  }
+}
+
+class NoRedirectRootSwaggerUiHttpServiceSpec extends SwaggerUiHttpServiceSpec {
+  override def swaggerUiFromRoot = false
+
+  behavior of "SwaggerUiHttpService"
+
+  it should "not redirect / to /swagger" in {
+    Get() ~> Route.seal(swaggerUiRoute) ~> check {
+      status should be(StatusCodes.NotFound)
+    }
+  }
+
+  it should "not return options for /" in {
+    Options() ~> Route.seal(swaggerUiRoute) ~> check {
+      status should be(StatusCodes.MethodNotAllowed)
+    }
+  }
+
+  it should "redirect /swagger to the index.html" in {
+    Get("/swagger") ~> swaggerUiRoute ~> check {
+      status should be(StatusCodes.TemporaryRedirect)
+      header("Location") should be(Option(Location(Uri("/swagger/index.html?url=/api-docs"))))
+    }
+  }
+
+  it should "return index.html from the swagger-ui jar" in {
+    Get("/swagger/index.html") ~> swaggerUiRoute ~> check {
+      status should be(StatusCodes.OK)
+    }
+  }
+}
+
+class DefaultSwaggerUiConfigHttpServiceSpec extends SwaggerUiHttpServiceSpec with SwaggerUiConfigHttpService {
+  override def swaggerUiConfig = ConfigFactory.parseString(s"uiVersion = $TestSwaggerUiVersion")
+
+  behavior of "SwaggerUiConfigHttpService"
+
+  it should "redirect /swagger to the index.html" in {
+    Get("/swagger") ~> swaggerUiRoute ~> check {
+      status should be(StatusCodes.TemporaryRedirect)
+      header("Location") should be(Option(Location(Uri("/swagger/index.html?url=/api-docs"))))
+    }
+  }
+
+  it should "return index.html from the swagger-ui jar" in {
+    Get("/swagger/index.html") ~> swaggerUiRoute ~> check {
+      status should be(StatusCodes.OK)
+      responseAs[String].take(SwaggerIndexPreamble.length) should be(SwaggerIndexPreamble)
+    }
+  }
+}
+
+class OverriddenSwaggerUiConfigHttpServiceSpec extends SwaggerUiHttpServiceSpec with SwaggerUiConfigHttpService {
+  override def swaggerUiConfig = ConfigFactory.parseString(
+    s"""
+       |baseUrl = /base
+       |docsPath = swagger/lenthall.yaml
+       |uiPath = ui/path
+       |uiVersion = $TestSwaggerUiVersion
+     """.stripMargin)
+
+  behavior of "SwaggerUiConfigHttpService"
+
+  it should "redirect /ui/path to the index.html under /base" in {
+    Get("/ui/path") ~> swaggerUiRoute ~> check {
+      status should be(StatusCodes.TemporaryRedirect)
+      header("Location") should be(Option(Location(Uri("/base/ui/path/index.html?url=/base/swagger/lenthall.yaml"))))
+    }
+  }
+
+  it should "return index.html from the swagger-ui jar" in {
+    Get("/ui/path/index.html") ~> swaggerUiRoute ~> check {
+      status should be(StatusCodes.OK)
+      responseAs[String].take(SwaggerIndexPreamble.length) should be(SwaggerIndexPreamble)
+    }
+  }
+}
+
+class YamlSwaggerResourceHttpServiceSpec extends SwaggerResourceHttpServiceSpec {
+  override def swaggerServiceName = "testservice"
+
+  behavior of "SwaggerResourceHttpService"
+
+  it should "service swagger yaml" in {
+    Get("/swagger/testservice.yaml") ~> swaggerResourceRoute ~> check {
+      status should be(StatusCodes.OK)
+      responseAs[String] should startWith("swagger: '2.0'\n")
+    }
+  }
+
+  it should "not service swagger json" in {
+    Get("/swagger/testservice.json") ~> Route.seal(swaggerResourceRoute) ~> check {
+      status should be(StatusCodes.NotFound)
+    }
+  }
+
+  it should "not service /swagger" in {
+    Get("/swagger") ~> Route.seal(swaggerResourceRoute) ~> check {
+      status should be(StatusCodes.NotFound)
+    }
+  }
+
+  it should "return options for all routes" in {
+    forAll(testPathsForOptions) { path =>
+      Options(path) ~> swaggerResourceRoute ~> check {
+        status should be(StatusCodes.OK)
+        responseAs[String] should be("OK")
+      }
+    }
+  }
+}
+
+class JsonSwaggerResourceHttpServiceSpec extends SwaggerResourceHttpServiceSpec {
+  override def swaggerServiceName = "testservice"
+
+  override def swaggerResourceType = "json"
+
+  behavior of "SwaggerResourceHttpService"
+
+  it should "service swagger json" in {
+    Get("/swagger/testservice.json") ~> swaggerResourceRoute ~> check {
+      status should be(StatusCodes.OK)
+      responseAs[String] should startWith("{\n  \"swagger\": \"2.0\",\n")
+    }
+  }
+
+  it should "not service swagger yaml" in {
+    Get("/swagger/testservice.yaml") ~> Route.seal(swaggerResourceRoute) ~> check {
+      status should be(StatusCodes.NotFound)
+    }
+  }
+
+  it should "not service /swagger" in {
+    Get("/swagger") ~> Route.seal(swaggerResourceRoute) ~> check {
+      status should be(StatusCodes.NotFound)
+    }
+  }
+
+  it should "return options for all routes" in {
+    forAll(testPathsForOptions) { path =>
+      Options(path) ~> swaggerResourceRoute ~> check {
+        status should be(StatusCodes.OK)
+        responseAs[String] should be("OK")
+      }
+    }
+  }
+}
+
+class NoOptionsSwaggerResourceHttpServiceSpec extends SwaggerResourceHttpServiceSpec {
+  override def swaggerServiceName = "testservice"
+
+  override def swaggerAllOptionsOk = false
+
+  behavior of "SwaggerResourceHttpService"
+
+  it should "service swagger yaml" in {
+    Get("/swagger/testservice.yaml") ~> swaggerResourceRoute ~> check {
+      status should be(StatusCodes.OK)
+      responseAs[String] should startWith("swagger: '2.0'\n")
+    }
+  }
+
+  it should "not service swagger json" in {
+    Get("/swagger/testservice.json") ~> Route.seal(swaggerResourceRoute) ~> check {
+      status should be(StatusCodes.NotFound)
+    }
+  }
+
+  it should "not service /swagger" in {
+    Get("/swagger") ~> Route.seal(swaggerResourceRoute) ~> check {
+      status should be(StatusCodes.NotFound)
+    }
+  }
+
+  it should "not return options for all routes" in {
+    forAll(testPathsForOptions) { path =>
+      Options(path) ~> Route.seal(swaggerResourceRoute) ~> check {
+        status should be(StatusCodes.MethodNotAllowed)
+      }
+    }
+  }
+}
+
+class YamlSwaggerUiResourceHttpServiceSpec extends SwaggerUiResourceHttpServiceSpec {
+  override def swaggerServiceName = "testservice"
+
+  behavior of "SwaggerUiResourceHttpService"
+
+  it should "redirect /swagger to /swagger/index.html with yaml" in {
+    Get("/swagger") ~> swaggerUiResourceRoute ~> check {
+      status should be(StatusCodes.TemporaryRedirect)
+      header("Location") should be(Option(Location(Uri("/swagger/index.html?url=/swagger/testservice.yaml"))))
+    }
+  }
+
+  it should "service swagger yaml" in {
+    Get("/swagger/testservice.yaml") ~> swaggerUiResourceRoute ~> check {
+      status should be(StatusCodes.OK)
+      responseAs[String] should startWith("swagger: '2.0'\n")
+    }
+  }
+
+  it should "not service swagger json" in {
+    Get("/swagger/testservice.json") ~> Route.seal(swaggerUiResourceRoute) ~> check {
+      status should be(StatusCodes.NotFound)
+    }
+  }
+
+  it should "return options for all routes" in {
+    forAll(testPathsForOptions) { path =>
+      Options(path) ~> swaggerUiResourceRoute ~> check {
+        status should be(StatusCodes.OK)
+        responseAs[String] should be("OK")
+      }
+    }
+  }
+}
+
+
+class JsonSwaggerUiResourceHttpServiceSpec extends SwaggerUiResourceHttpServiceSpec {
+  override def swaggerServiceName = "testservice"
+
+  override def swaggerResourceType = "json"
+
+  behavior of "SwaggerUiResourceHttpService"
+
+  it should "redirect /swagger to /swagger/index.html with yaml with json" in {
+    Get("/swagger") ~> swaggerUiResourceRoute ~> check {
+      status should be(StatusCodes.TemporaryRedirect)
+      header("Location") should be(Option(Location(Uri("/swagger/index.html?url=/swagger/testservice.json"))))
+    }
+  }
+
+  it should "service swagger json" in {
+    Get("/swagger/testservice.json") ~> swaggerUiResourceRoute ~> check {
+      status should be(StatusCodes.OK)
+      responseAs[String] should startWith("{\n  \"swagger\": \"2.0\",\n")
+    }
+  }
+
+  it should "not service swagger yaml" in {
+    Get("/swagger/testservice.yaml") ~> Route.seal(swaggerUiResourceRoute) ~> check {
+      status should be(StatusCodes.NotFound)
+    }
+  }
+
+  it should "return options for all routes" in {
+    forAll(testPathsForOptions) { path =>
+      Options(path) ~> swaggerUiResourceRoute ~> check {
+        status should be(StatusCodes.OK)
+        responseAs[String] should be("OK")
+      }
+    }
+  }
+}
diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsPathBuilder.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsPathBuilder.scala
index ae6c1f77b..08b702029 100644
--- a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsPathBuilder.scala
+++ b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsPathBuilder.scala
@@ -1,23 +1,26 @@
 package cromwell.filesystems.gcs
 
 import java.net.URI
-import java.nio.file.spi.FileSystemProvider
 
 import akka.actor.ActorSystem
 import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport
 import com.google.api.client.json.jackson2.JacksonFactory
-import com.google.cloud.RetryParams
-import com.google.cloud.storage.StorageOptions
-import com.google.cloud.storage.contrib.nio.{CloudStorageConfiguration, CloudStorageFileSystem, CloudStoragePath}
+import com.google.api.gax.retrying.RetrySettings
+import com.google.auth.Credentials
+import com.google.cloud.http.HttpTransportOptions
+import com.google.cloud.storage.contrib.nio.{CloudStorageConfiguration, CloudStorageFileSystem, CloudStorageFileSystemProvider, CloudStoragePath}
+import com.google.cloud.storage.{BlobId, StorageOptions}
 import com.google.common.base.Preconditions._
 import com.google.common.net.UrlEscapers
 import cromwell.core.WorkflowOptions
-import cromwell.core.path.proxy.{PathProxy, RetryableFileSystemProviderProxy}
-import cromwell.core.path.{CustomRetryParams, NioPath, Path, PathBuilder}
+import cromwell.core.path.{NioPath, Path, PathBuilder}
 import cromwell.filesystems.gcs.GcsPathBuilder._
 import cromwell.filesystems.gcs.auth.GoogleAuthMode
 
-import scala.util.{Failure, Try}
+import scala.concurrent.{ExecutionContext, Future}
+import scala.concurrent.duration._
+import scala.language.postfixOps
+import scala.util.Try
 
 object GcsPathBuilder {
 
@@ -35,8 +38,39 @@ object GcsPathBuilder {
     checkNotNull(uri.getHost, s"%s does not have a host", uri)
   }
 
-  def isValidGcsUrl(str: String): Boolean = {
-    Try(checkValid(getUri(str))).isSuccess
+  sealed trait GcsPathValidation
+  case object ValidFullGcsPath extends GcsPathValidation
+  case object PossiblyValidRelativeGcsPath extends GcsPathValidation
+  sealed trait InvalidGcsPath extends GcsPathValidation {
+    def pathString: String
+    def errorMessage: String
+  }
+  final case class InvalidFullGcsPath(pathString: String) extends InvalidGcsPath {
+    override def errorMessage = {
+      val prefix = s"""
+      |The bucket name in GCS path '$pathString' is not compatible with URI host name standards.
+      |URI host name compatibility is a requirement for Cromwell's GCS filesystem support.
+      |Google also generally advises against the use of underscores in GCS bucket names, as well as against
+      |the use of periods or dashes in certain patterns as described here:
+      |https://cloud.google.com/storage/docs/naming.
+      """.stripMargin.replaceAll("\n", " ").trim
+      val underscoreWarning = if (pathString.contains("_")) s"In particular, the bucket name in '$pathString' may contain an underscore which is not a valid character in a URI host." else ""
+      List(prefix, underscoreWarning).mkString(" ")
+    }
+  }
+  final case class UnparseableGcsPath(pathString: String, throwable: Throwable) extends InvalidGcsPath {
+    override def errorMessage: String =
+      List(s"The specified GCS path '$pathString' does not parse as a URI.", throwable.getMessage).mkString("\n")
+  }
+
+  def validateGcsPath(string: String): GcsPathValidation = {
+    Try {
+      val uri = getUri(string)
+      if (uri.getScheme == null) PossiblyValidRelativeGcsPath
+      else if (uri.getScheme == "gs") {
+        if (uri.getHost == null) InvalidFullGcsPath(string) else ValidFullGcsPath
+      } else InvalidFullGcsPath(string)
+    } recover { case t => UnparseableGcsPath(string, t) } get
   }
 
   def isGcsPath(nioPath: NioPath): Boolean = {
@@ -44,77 +78,89 @@ object GcsPathBuilder {
   }
 
   def getUri(string: String) = URI.create(UrlEscapers.urlFragmentEscaper().escape(string))
-}
-
-class GcsPathBuilder(authMode: GoogleAuthMode,
-                     retryParams: RetryParams,
-                     cloudStorageConfiguration: CloudStorageConfiguration,
-                     options: WorkflowOptions) extends PathBuilder {
-  authMode.validate(options)
-
-  protected val storageOptionsBuilder = StorageOptions.builder()
-                                  .authCredentials(authMode.authCredentials(options))
-                                  .retryParams(retryParams)
-
-  // Grab the google project from Workflow Options if specified and set
-  // that to be the project used by the StorageOptions Builder
-  options.get("google_project") map storageOptionsBuilder.projectId
-
-
-  protected val storageOptions = storageOptionsBuilder.build()
-
-  // The CloudStorageFileSystemProvider constructor is not public. Currently the only way to obtain one is through a CloudStorageFileSystem
-  // Moreover at this point we can use the same provider for all operations as we have usable credentials
-  // In order to avoid recreating a provider with every getPath call, create a dummy FileSystem just to get its provider
-  protected val _provider = CloudStorageFileSystem.forBucket("dummy", cloudStorageConfiguration, storageOptions).provider()
-
-  protected def provider: FileSystemProvider = _provider
-  /*
-   * The StorageService already contains a StorageRpc object that contains a com.google.api.services.storage.Storage object
-   * However it is not accessible from StorageService.
-   * com.google.cloud.storage.Storage has some batching capabilities but not for copying.
-   * In order to support batch copy, we need a com.google.api.services.storage.Storage.
-   */
-  def getHash(builtPath: Path): Try[String] = {
-    builtPath match {
-      case GcsPath(path) => path match {
-        case gcsPath: CloudStoragePath => Try(storageOptions.service().get(gcsPath.bucket(), gcsPath.toRealPath().toString).crc32c())
-        case proxy: PathProxy =>
-          val gcsPath = proxy.unbox(classOf[CloudStoragePath]).get
-          Try(storageOptions.service().get(gcsPath.bucket(), gcsPath.toRealPath().toString).crc32c())
-        case other => Failure(new IllegalArgumentException(s"$other is not a CloudStoragePath"))
-      }
-      case other => Failure(new IllegalArgumentException(s"$other is not a GcsPath"))
+  
+  def fromAuthMode(authMode: GoogleAuthMode,
+                   applicationName: String,
+                   retrySettings: Option[RetrySettings],
+                   cloudStorageConfiguration: CloudStorageConfiguration,
+                   options: WorkflowOptions)(implicit as: ActorSystem, ec: ExecutionContext): Future[GcsPathBuilder] = {
+    authMode.credential(options) map { credentials =>
+      fromCredentials(credentials,
+        applicationName,
+        retrySettings,
+        cloudStorageConfiguration,
+        options
+      )
     }
   }
+  
+  def fromCredentials(credentials: Credentials,
+                      applicationName: String,
+                      retrySettings: Option[RetrySettings],
+                      cloudStorageConfiguration: CloudStorageConfiguration,
+                      options: WorkflowOptions): GcsPathBuilder = {
+    val transportOptions = HttpTransportOptions.newBuilder()
+      .setReadTimeout(3.minutes.toMillis.toInt)
+      .build()
+
+    val storageOptionsBuilder = StorageOptions.newBuilder()
+      .setTransportOptions(transportOptions)
+      .setCredentials(credentials)
+
+    retrySettings foreach storageOptionsBuilder.setRetrySettings
+
+    // Grab the google project from Workflow Options if specified and set
+    // that to be the project used by the StorageOptions Builder
+    options.get("google_project") map storageOptionsBuilder.setProjectId
+
+
+    val storageOptions = storageOptionsBuilder.build()
+
+    // Create a com.google.api.services.storage.Storage
+    // This is the underlying api used by com.google.cloud.storage
+    // By bypassing com.google.cloud.storage, we can create low level requests that can be batched
+    val apiStorage: com.google.api.services.storage.Storage = {
+      new com.google.api.services.storage.Storage
+      .Builder(HttpTransport, JsonFactory, GoogleConfiguration.withCustomTimeouts(transportOptions.getHttpRequestInitializer(storageOptions)))
+        .setApplicationName(applicationName)
+        .build()
+    }
 
-  def getProjectId = storageOptions.projectId()
+    // Create a com.google.cloud.storage.Storage
+    // This is the "relatively" high level API, and recommended one. The nio implementation sits on top of this.
+    val cloudStorage: com.google.cloud.storage.Storage = storageOptions.getService
+
+    // The CloudStorageFileSystemProvider constructor is not public. Currently the only way to obtain one is through a CloudStorageFileSystem
+    // Moreover at this point we can use the same provider for all operations as we have usable credentials
+    // In order to avoid recreating a provider with every getPath call, create a dummy FileSystem just to get its provider
+    val provider: CloudStorageFileSystemProvider = CloudStorageFileSystem.forBucket("dummy", cloudStorageConfiguration, storageOptions).provider()
+    
+    new GcsPathBuilder(apiStorage, cloudStorage, provider, storageOptions.getProjectId)
+  }
+}
 
+class GcsPathBuilder(val apiStorage: com.google.api.services.storage.Storage,
+                     val cloudStorage: com.google.cloud.storage.Storage,
+                     provider: CloudStorageFileSystemProvider,
+                     val projectId: String) extends PathBuilder {
   def build(string: String): Try[GcsPath] = {
     Try {
       val uri = getUri(string)
       GcsPathBuilder.checkValid(uri)
-      GcsPath(provider.getPath(uri))
+      GcsPath(provider.getPath(uri), apiStorage, cloudStorage)
     }
   }
 
   override def name: String = "Gcs"
 }
 
-class RetryableGcsPathBuilder(authMode: GoogleAuthMode,
-                              googleRetryParams: RetryParams,
-                              customRetryParams: CustomRetryParams,
-                              cloudStorageConfiguration: CloudStorageConfiguration,
-                              options: WorkflowOptions)(implicit actorSystem: ActorSystem)
-  extends GcsPathBuilder(authMode, googleRetryParams, cloudStorageConfiguration, options) {
-
-  override protected def provider = new RetryableFileSystemProviderProxy(_provider, customRetryParams)
-
-  override def getHash(path: Path) = provider.withRetry(() => super.getHash(path))
-}
+case class GcsPath private[gcs](nioPath: NioPath,
+                                apiStorage: com.google.api.services.storage.Storage,
+                                cloudStorage: com.google.cloud.storage.Storage
+                               ) extends Path {
+  lazy val blob = BlobId.of(cloudStoragePath.bucket, cloudStoragePath.toRealPath().toString)
 
-case class GcsPath private[gcs](nioPath: NioPath) extends Path {
-  override protected def newPath(nioPath: NioPath): GcsPath = GcsPath(nioPath)
+  override protected def newPath(nioPath: NioPath): GcsPath = GcsPath(nioPath, apiStorage, cloudStorage)
 
   override def pathAsString: String = java.net.URLDecoder.decode(nioPath.toUri.toString, "UTF-8")
 
@@ -123,9 +169,8 @@ case class GcsPath private[gcs](nioPath: NioPath) extends Path {
     gcsPath.bucket + gcsPath.toAbsolutePath.toString
   }
 
-  private def cloudStoragePath: CloudStoragePath = nioPath match {
+  def cloudStoragePath: CloudStoragePath = nioPath match {
     case gcsPath: CloudStoragePath => gcsPath
-    case pathProxy: PathProxy => pathProxy.unbox(classOf[CloudStoragePath]).get
     case _ => throw new RuntimeException(s"Internal path was not a cloud storage path: $nioPath")
   }
 }
diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsPathBuilderFactory.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsPathBuilderFactory.scala
index 83aad3ce8..01659e42f 100644
--- a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsPathBuilderFactory.scala
+++ b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsPathBuilderFactory.scala
@@ -2,21 +2,23 @@ package cromwell.filesystems.gcs
 
 import akka.actor.ActorSystem
 import com.google.api.client.googleapis.media.MediaHttpUploader
-import com.google.cloud.RetryParams
+import com.google.api.gax.retrying.RetrySettings
+import com.google.auth.Credentials
 import com.google.cloud.storage.contrib.nio.CloudStorageConfiguration
 import com.typesafe.config.ConfigFactory
 import cromwell.core.WorkflowOptions
-import cromwell.core.path.{CustomRetryParams, PathBuilderFactory}
+import cromwell.core.path.PathBuilderFactory
 import cromwell.filesystems.gcs.auth.GoogleAuthMode
 import net.ceedubs.ficus.Ficus._
 
+import scala.concurrent.ExecutionContext
+
 object GcsPathBuilderFactory {
 
   private[this] lazy val UploadBufferBytes = {
     ConfigFactory.load().as[Option[Int]]("google.upload-buffer-bytes").getOrElse(MediaHttpUploader.MINIMUM_CHUNK_SIZE)
   }
 
-  val DefaultRetryParams = RetryParams.defaultInstance()
   val DefaultCloudStorageConfiguration = {
     CloudStorageConfiguration.builder()
       .blockSize(UploadBufferBytes)
@@ -28,21 +30,21 @@ object GcsPathBuilderFactory {
 }
 
 case class GcsPathBuilderFactory(authMode: GoogleAuthMode,
-                                 retryParams: RetryParams = GcsPathBuilderFactory.DefaultRetryParams,
+                                 applicationName: String,
+                                 retrySettings: Option[RetrySettings] = None,
                                  cloudStorageConfiguration: CloudStorageConfiguration = GcsPathBuilderFactory.DefaultCloudStorageConfiguration)
 
   extends PathBuilderFactory {
 
-  def withOptions(options: WorkflowOptions)(implicit actorSystem: ActorSystem) = new GcsPathBuilder(authMode, retryParams, cloudStorageConfiguration, options)
-}
-
-case class RetryableGcsPathBuilderFactory(authMode: GoogleAuthMode,
-                                          googleRetryParams: RetryParams = GcsPathBuilderFactory.DefaultRetryParams,
-                                          customRetryParams: CustomRetryParams = CustomRetryParams.Default,
-                                          cloudStorageConfiguration: CloudStorageConfiguration = GcsPathBuilderFactory.DefaultCloudStorageConfiguration)
-
-
-  extends PathBuilderFactory {
+  def withOptions(options: WorkflowOptions)(implicit as: ActorSystem, ec: ExecutionContext) = {
+    GcsPathBuilder.fromAuthMode(authMode, applicationName, retrySettings, cloudStorageConfiguration, options)
+  }
 
-  def withOptions(options: WorkflowOptions)(implicit actorSystem: ActorSystem) = new RetryableGcsPathBuilder(authMode, googleRetryParams, customRetryParams, cloudStorageConfiguration, options)
+  /**
+    * Ignores the authMode and creates a GcsPathBuilder using the passed credentials directly.
+    * Can be used when the Credentials are already available.
+    */
+  def fromCredentials(options: WorkflowOptions, credentials: Credentials) = {
+    GcsPathBuilder.fromCredentials(credentials, applicationName, retrySettings, cloudStorageConfiguration, options)
+  }
 }
diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GoogleConfiguration.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GoogleConfiguration.scala
index 142485c95..34c1bff53 100644
--- a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GoogleConfiguration.scala
+++ b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GoogleConfiguration.scala
@@ -1,18 +1,23 @@
 package cromwell.filesystems.gcs
 
+import java.io.IOException
+
 import cats.data.Validated._
 import cats.instances.list._
 import cats.syntax.cartesian._
 import cats.syntax.traverse._
 import cats.syntax.validated._
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential
+import com.google.api.client.http.{HttpRequest, HttpRequestInitializer}
 import com.google.api.services.storage.StorageScopes
-import com.typesafe.config.Config
+import com.typesafe.config.{Config, ConfigException}
+import cromwell.filesystems.gcs.auth.ServiceAccountMode.{JsonFileFormat, PemFileFormat}
 import cromwell.filesystems.gcs.auth._
 import lenthall.exception.MessageAggregation
 import lenthall.validation.ErrorOr._
 import lenthall.validation.Validation._
-import org.slf4j.LoggerFactory
 import net.ceedubs.ficus.Ficus._
+import org.slf4j.LoggerFactory
 
 final case class GoogleConfiguration private (applicationName: String, authsByName: Map[String, GoogleAuthMode]) {
 
@@ -28,9 +33,31 @@ final case class GoogleConfiguration private (applicationName: String, authsByNa
 
 object GoogleConfiguration {
   import scala.collection.JavaConverters._
+  import scala.concurrent.duration._
+  import scala.language.postfixOps
+
+  lazy val DefaultConnectionTimeout = 3 minutes
+  lazy val DefaultReadTimeout = 3 minutes
+
+  lazy val DefaultRequestInitializer = GoogleConfiguration.withCustomTimeouts(new GoogleCredential.Builder().build())
+
+  def withCustomTimeouts(httpRequestInitializer: HttpRequestInitializer,
+                         connectionTimeout: FiniteDuration = DefaultConnectionTimeout,
+                         readTimeout: FiniteDuration = DefaultReadTimeout) = {
+    new HttpRequestInitializer() {
+      @throws[IOException]
+      override def initialize(httpRequest: HttpRequest) = {
+        httpRequestInitializer.initialize(httpRequest)
+        httpRequest.setConnectTimeout(connectionTimeout.toMillis.toInt)
+        httpRequest.setReadTimeout(readTimeout.toMillis.toInt)
+        ()
+      }
+    }
+  }
+
   private val log = LoggerFactory.getLogger("GoogleConfiguration")
 
-  case class GoogleConfigurationException(errorMessages: List[String]) extends MessageAggregation {
+  final case class GoogleConfigurationException(errorMessages: List[String]) extends MessageAggregation {
     override val exceptionContext = "Google configuration"
   }
 
@@ -40,7 +67,7 @@ object GoogleConfiguration {
     "https://www.googleapis.com/auth/genomics",
     "https://www.googleapis.com/auth/compute"
   ).asJava
-  
+
   def apply(config: Config): GoogleConfiguration = {
 
     val googleConfig = config.getConfig("google")
@@ -50,7 +77,12 @@ object GoogleConfiguration {
     def buildAuth(authConfig: Config): ErrorOr[GoogleAuthMode] = {
 
       def serviceAccountAuth(authConfig: Config, name: String): ErrorOr[GoogleAuthMode] = validate {
-        ServiceAccountMode(name, authConfig.as[String]("service-account-id"), authConfig.as[String]("pem-file"), GoogleScopes)
+        (authConfig.getAs[String]("pem-file"), authConfig.getAs[String]("json-file")) match {
+          case (Some(pem), None) => ServiceAccountMode(name, PemFileFormat(authConfig.as[String]("service-account-id"), pem), GoogleScopes)
+          case (None, Some(json)) => ServiceAccountMode(name, JsonFileFormat(json), GoogleScopes)
+          case (None, None) => throw new ConfigException.Generic(s"""No credential configuration was found for service account "$name". See reference.conf under the google.auth, service-account section for supported credential formats.""")
+          case (Some(_), Some(_)) => throw new ConfigException.Generic(s"""Both a pem file and a json file were supplied for service account "$name" in the configuration file. Only one credential file can be supplied for the same service account. Please choose between the two.""")
+        }
       }
 
       def userAccountAuth(authConfig: Config, name: String): ErrorOr[GoogleAuthMode] =  validate {
diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/auth/GoogleAuthMode.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/auth/GoogleAuthMode.scala
index 67f4c2be7..e9ba4eb8a 100644
--- a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/auth/GoogleAuthMode.scala
+++ b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/auth/GoogleAuthMode.scala
@@ -1,31 +1,37 @@
 package cromwell.filesystems.gcs.auth
 
-import java.io.{FileNotFoundException, InputStreamReader}
-import java.nio.file.Paths
+import java.io.FileNotFoundException
 
+import akka.actor.ActorSystem
+import akka.http.scaladsl.model.StatusCodes
 import better.files._
-import com.google.api.client.auth.oauth2.Credential
-import com.google.api.client.extensions.java6.auth.oauth2.AuthorizationCodeInstalledApp
-import com.google.api.client.googleapis.auth.oauth2.{GoogleAuthorizationCodeFlow, GoogleClientSecrets, GoogleCredential}
-import com.google.api.client.googleapis.extensions.java6.auth.oauth2.GooglePromptReceiver
 import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport
-import com.google.api.client.googleapis.testing.auth.oauth2.MockGoogleCredential
+import com.google.api.client.http.HttpResponseException
 import com.google.api.client.json.jackson2.JacksonFactory
-import com.google.api.client.util.store.FileDataStoreFactory
 import com.google.api.services.storage.StorageScopes
-import com.google.auth.oauth2.{ClientId, ServiceAccountCredentials}
-import com.google.cloud.AuthCredentials
+import com.google.auth.Credentials
+import com.google.auth.http.HttpTransportFactory
+import com.google.auth.oauth2.{GoogleCredentials, ServiceAccountCredentials, UserCredentials}
+import com.google.cloud.NoCredentials
 import cromwell.core.WorkflowOptions
+import cromwell.core.retry.Retry
 import cromwell.filesystems.gcs.auth.GoogleAuthMode._
+import cromwell.filesystems.gcs.auth.ServiceAccountMode.{CredentialFileFormat, JsonFileFormat, PemFileFormat}
 import org.slf4j.LoggerFactory
 
 import scala.collection.JavaConverters._
+import scala.concurrent.{ExecutionContext, Future}
 import scala.util.{Failure, Success, Try}
 
 object GoogleAuthMode {
 
   lazy val jsonFactory = JacksonFactory.getDefaultInstance
   lazy val httpTransport = GoogleNetHttpTransport.newTrustedTransport
+  lazy val HttpTransportFactory = new HttpTransportFactory {
+    override def create() = {
+      httpTransport
+    }
+  }
 
   val RefreshTokenOptionKey = "refresh_token"
   val GcsScopes = List(
@@ -37,11 +43,22 @@ object GoogleAuthMode {
     if (!file.isReadable) throw new FileNotFoundException(s"File $file does not exist or is not readable")
   }
 
-  case object NoAuthMode extends GoogleAuthMode {
+  case object MockAuthMode extends GoogleAuthMode {
     override def name = "no_auth"
-
-    override def authCredentials(options: WorkflowOptions): AuthCredentials = AuthCredentials.noAuth()
-    override def credential(options: WorkflowOptions): Credential = new MockGoogleCredential.Builder().build()
+    override def credential(options: WorkflowOptions)(implicit as: ActorSystem, ec: ExecutionContext): Future[Credentials] = {
+      Future.successful(NoCredentials.getInstance())
+    }
+  }
+  
+  def isFatal(ex: Throwable) = {
+    // We wrap the actual exception in a RuntimeException so get the cause
+    ex.getCause match {
+      case http: HttpResponseException =>
+        http.getStatusCode == StatusCodes.Unauthorized.intValue ||
+        http.getStatusCode == StatusCodes.Forbidden.intValue ||
+        http.getStatusCode == StatusCodes.BadRequest.intValue
+      case _ => false
+    }
   }
 }
 
@@ -55,96 +72,79 @@ sealed trait GoogleAuthMode {
   def validate(options: WorkflowOptions): Unit = {()}
 
   def name: String
-  // Create an AuthCredentials object from the google-cloud library (https://github.com/GoogleCloudPlatform/google-cloud-java using https://github.com/google/google-auth-library-java under the hood)
-  def authCredentials(options: WorkflowOptions): AuthCredentials
   // Create a Credential object from the google.api.client.auth library (https://github.com/google/google-api-java-client)
-  def credential(options: WorkflowOptions): Credential
-  
-  def credentialBundle(options: WorkflowOptions) = {
-    GoogleCredentialBundle(credential(options), authCredentials(options))
-  }
+  def credential(options: WorkflowOptions)(implicit as: ActorSystem, ec: ExecutionContext): Future[Credentials]
 
   def requiresAuthFile: Boolean = false
 
-  protected def validateAuthCredentials(authCredentials: AuthCredentials, scopes: java.util.Collection[String]): AuthCredentials = validate(authCredentials, authCredentials.credentials().createScoped(scopes).refresh)
-
-  protected def validateCredential(credential: Credential) = validate(credential, credential.refreshToken)
-
-  private def validate[T](credential: T, validation: () => Any): T = {
-    Try(validation()) match {
-      case Failure(ex) => throw new RuntimeException(s"Google credentials are invalid: ${ex.getMessage}")
+  protected def validateCredential(credential: Credentials) = {
+    Try(credential.refresh()) match {
+      case Failure(ex) => throw new RuntimeException(s"Google credentials are invalid: ${ex.getMessage}", ex)
       case Success(_) => credential
     }
   }
 }
 
+object ServiceAccountMode {
+  sealed trait CredentialFileFormat {
+    def file: String
+  }
+  case class PemFileFormat(accountId: String, file: String) extends CredentialFileFormat
+  case class JsonFileFormat(file: String) extends CredentialFileFormat
+}
+
 final case class ServiceAccountMode(override val name: String,
-                                    accountId: String,
-                                    pemPath: String,
+                                    fileFormat: CredentialFileFormat,
                                     scopes: java.util.List[String]) extends GoogleAuthMode {
-  private val pemFile = File(pemPath)
-  checkReadable(pemFile)
-
-  private lazy val _authCredentials: AuthCredentials = {
-    val saCredentials = ServiceAccountCredentials.fromPkcs8(accountId, accountId, pemFile.contentAsString, null, scopes)
-    validateAuthCredentials(AuthCredentials.createFor(saCredentials.getClientId, saCredentials.getPrivateKey), scopes)
+  private val credentialsFile = File(fileFormat.file)
+  checkReadable(credentialsFile)
+
+  private lazy val _credential: Credentials = {
+    val serviceAccount = fileFormat match {
+      case PemFileFormat(accountId, _) => 
+        log.warn("The PEM file format will be deprecated in the upcoming Cromwell version. Please use JSON instead.")
+        ServiceAccountCredentials.fromPkcs8(accountId, accountId, credentialsFile.contentAsString, null, scopes)
+      case _: JsonFileFormat => ServiceAccountCredentials.fromStream(credentialsFile.newInputStream).createScoped(scopes)
+    }
+    
+    // Validate credentials synchronously here, without retry.
+    // It's very unlikely to fail as it should not happen more than a few times 
+    // (one for the engine and for each backend using it) per Cromwell instance.
+    validateCredential(serviceAccount)
   }
 
-  private lazy val _credential: Credential = {
-    validateCredential(
-      new GoogleCredential.Builder().setTransport(httpTransport)
-        .setJsonFactory(jsonFactory)
-        .setServiceAccountId(accountId)
-        .setServiceAccountScopes(scopes)
-        .setServiceAccountPrivateKeyFromPemFile(pemFile.toJava)
-        .build()
-    )
+  override def credential(options: WorkflowOptions)(implicit as: ActorSystem, ec: ExecutionContext): Future[Credentials] = {
+    Future.successful(_credential)
   }
-
-  override def authCredentials(options: WorkflowOptions) = _authCredentials
-
-  override def credential(options: WorkflowOptions): Credential = _credential
 }
 
 final case class UserMode(override val name: String,
                           user: String,
-                          val secretsPath: String,
+                          secretsPath: String,
                           datastoreDir: String,
                           scopes: java.util.List[String]) extends GoogleAuthMode {
 
-  private lazy val secrets = {
+  private lazy val secretsStream = {
     val secretsFile = File(secretsPath)
     checkReadable(secretsFile)
-
-    val secretStream = new InputStreamReader(secretsFile.newInputStream)
-
-    GoogleClientSecrets.load(jsonFactory, secretStream)
-  }
-
-  private lazy val _credential: Credential = {
-    val dataStore = Paths.get(datastoreDir).toAbsolutePath
-    val dataStoreFactory = new FileDataStoreFactory(dataStore.toFile)
-    val flow = new GoogleAuthorizationCodeFlow.Builder(httpTransport, jsonFactory, secrets, scopes).setDataStoreFactory(dataStoreFactory).build
-    validateCredential(new AuthorizationCodeInstalledApp(flow, new GooglePromptReceiver).authorize(user))
+    secretsFile.newInputStream
   }
 
-  private lazy val _authCredentials: AuthCredentials = {
-    new RefreshableOAuth2Credentials(_credential.getRefreshToken, new ClientId(secrets.getDetails.getClientId, secrets.getDetails.getClientSecret))
+  private lazy val _credential: Credentials = {
+    validateCredential(UserCredentials.fromStream(secretsStream))
   }
 
-  override def credential(options: WorkflowOptions) = _credential
-
-  override def authCredentials(options: WorkflowOptions) = _authCredentials
+  override def credential(options: WorkflowOptions)(implicit as: ActorSystem, ec: ExecutionContext): Future[Credentials] = Future.successful(_credential)
 }
 
 private object ApplicationDefault {
-  private [auth] lazy val _AuthCredentials = AuthCredentials.createApplicationDefaults()
-  private [auth] lazy val _Credential: Credential = GoogleCredential.getApplicationDefault()
+  private [auth] lazy val _Credential: Credentials = GoogleCredentials.getApplicationDefault
 }
 
 final case class ApplicationDefaultMode(name: String) extends GoogleAuthMode {
-  override def authCredentials(options: WorkflowOptions) = ApplicationDefault._AuthCredentials
-  override def credential(options: WorkflowOptions) = ApplicationDefault._Credential
+  override def credential(options: WorkflowOptions)(implicit as: ActorSystem, ec: ExecutionContext): Future[Credentials] = {
+    Future.successful(ApplicationDefault._Credential)
+  }
 }
 
 final case class RefreshTokenMode(name: String,
@@ -162,23 +162,17 @@ final case class RefreshTokenMode(name: String,
 
   override def validate(options: WorkflowOptions) = {
     extractRefreshToken(options)
-
     ()
   }
 
-  override def authCredentials(options: WorkflowOptions): AuthCredentials = {
-    val refreshToken = extractRefreshToken(options)
-    validateAuthCredentials(new RefreshableOAuth2Credentials(refreshToken, new ClientId(clientId, clientSecret)), scopes)
-  }
-
-  override def credential(options: WorkflowOptions): Credential = {
+  override def credential(options: WorkflowOptions)(implicit as: ActorSystem, ec: ExecutionContext): Future[Credentials] = {
     val refreshToken = extractRefreshToken(options)
-    validateCredential(
-      new GoogleCredential.Builder().setTransport(httpTransport)
-        .setJsonFactory(jsonFactory)
-        .setClientSecrets(clientId, clientSecret)
-        .build()
-        .setRefreshToken(refreshToken)
+    Retry.withRetry(
+      () => Future(validateCredential(
+        new UserCredentials(clientId, clientSecret, refreshToken, null, GoogleAuthMode.HttpTransportFactory, null)
+      )),
+      isFatal = isFatal,
+      maxRetries = Option(3)
     )
   }
 }
diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/auth/GoogleCredentialBundle.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/auth/GoogleCredentialBundle.scala
deleted file mode 100644
index 83487e63d..000000000
--- a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/auth/GoogleCredentialBundle.scala
+++ /dev/null
@@ -1,6 +0,0 @@
-package cromwell.filesystems.gcs.auth
-
-import com.google.api.client.auth.oauth2.Credential
-import com.google.cloud.AuthCredentials
-
-case class GoogleCredentialBundle(credential: Credential, authCredential: AuthCredentials)
diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/auth/RefreshableOAuth2Credentials.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/auth/RefreshableOAuth2Credentials.scala
deleted file mode 100644
index ae1e32ef5..000000000
--- a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/auth/RefreshableOAuth2Credentials.scala
+++ /dev/null
@@ -1,31 +0,0 @@
-package cromwell.filesystems.gcs.auth
-
-import java.io.Serializable
-import java.util.Objects
-
-import com.google.auth.oauth2.{ClientId, GoogleCredentials, UserCredentials}
-import com.google.cloud.{AuthCredentials, RestorableState}
-
-class RefreshableOAuth2Credentials(refreshToken: String, clientId: ClientId) extends AuthCredentials {
-  private val _credentials: GoogleCredentials = new UserCredentials(clientId.getClientId, clientId.getClientSecret, refreshToken)
-
-  private class RefreshableOAuth2CredentialsState(val refreshToken: String, val clientId: ClientId) extends RestorableState[AuthCredentials] with Serializable {
-
-    override def restore: AuthCredentials = new RefreshableOAuth2Credentials(refreshToken, clientId)
-
-    override def hashCode: Int = Objects.hash(refreshToken, clientId.getClientId, clientId.getClientSecret)
-
-    override def equals(obj: Any): Boolean = {
-      obj.isInstanceOf[RefreshableOAuth2CredentialsState] && {
-        val other = obj.asInstanceOf[RefreshableOAuth2CredentialsState]
-        Objects.equals(refreshToken, other.refreshToken) &&
-        Objects.equals(clientId.getClientId, other.clientId.getClientId) &&
-        Objects.equals(clientId.getClientSecret, other.clientId.getClientSecret)
-      }
-    }
-  }
-
-  override def credentials: GoogleCredentials = _credentials
-
-  def capture: RestorableState[AuthCredentials] = new RefreshableOAuth2CredentialsState(refreshToken, clientId)
-}
diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/batch/GcsBatchCommandBuilder.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/batch/GcsBatchCommandBuilder.scala
new file mode 100644
index 000000000..ce09725fa
--- /dev/null
+++ b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/batch/GcsBatchCommandBuilder.scala
@@ -0,0 +1,32 @@
+package cromwell.filesystems.gcs.batch
+
+import cromwell.core.io._
+import cromwell.core.path.Path
+import cromwell.filesystems.gcs.GcsPath
+
+trait GcsBatchCommandBuilder extends DefaultIoCommandBuilder {
+  override def sizeCommand(path: Path) = path match {
+    case gcsPath: GcsPath => GcsBatchSizeCommand(gcsPath)
+    case _ => super.sizeCommand(path)
+  }
+  
+  override def deleteCommand(path: Path, swallowIoExceptions: Boolean = false) = path match {
+    case gcsPath: GcsPath => GcsBatchDeleteCommand(gcsPath, swallowIoExceptions)
+    case _ => super.deleteCommand(path, swallowIoExceptions)
+  }
+  
+  override def copyCommand(src: Path, dest: Path, overwrite: Boolean = true) =  (src, dest) match {
+    case (gcsSrc: GcsPath, gcsDest: GcsPath) => GcsBatchCopyCommand(gcsSrc, gcsDest, overwrite)
+    case _ => super.copyCommand(src, dest, overwrite)
+  }
+  
+  override def hashCommand(path: Path) =  path match {
+    case gcsPath: GcsPath => GcsBatchCrc32Command(gcsPath)
+    case _ => super.hashCommand(path)
+  }
+
+  override def touchCommand(path: Path) =  path match {
+    case gcsPath: GcsPath => GcsBatchTouchCommand(gcsPath)
+    case _ => super.touchCommand(path)
+  }
+}
diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/batch/GcsBatchIoCommand.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/batch/GcsBatchIoCommand.scala
new file mode 100644
index 000000000..02c91c169
--- /dev/null
+++ b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/batch/GcsBatchIoCommand.scala
@@ -0,0 +1,96 @@
+package cromwell.filesystems.gcs.batch
+
+import com.google.api.client.http.HttpHeaders
+import com.google.api.services.storage.StorageRequest
+import com.google.api.services.storage.model.{RewriteResponse, StorageObject}
+import cromwell.core.io._
+import cromwell.filesystems.gcs._
+
+/**
+  * Io commands with GCS paths and some logic enabling batching of request.
+  * @tparam T Return type of the IoCommand
+  * @tparam U Return type of the Google response
+  */
+sealed trait GcsBatchIoCommand[T, U] extends IoCommand[T] {
+  /**
+    * StorageRequest operation to be executed by this command
+    */
+  def operation: StorageRequest[U]
+
+  /**
+    * Maps the google response of type U to the Cromwell Io response of type T
+    */
+  protected def mapGoogleResponse(response: U): T
+
+  /**
+    * Method called in the success callback of a batched request to decide what to do next.
+    * Returns an Either[T, GcsBatchIoCommand[T, U]]
+    *   Left(value) means the command is complete, and the result can be sent back to the sender.
+    *   Right(newCommand) means the command is not complete and needs another request to be executed.
+    * Most commands will reply with Left(value).
+    */
+  def onSuccess(response: U, httpHeaders: HttpHeaders): Either[T, GcsBatchIoCommand[T, U]] = {
+    Left(mapGoogleResponse(response))
+  }
+}
+
+case class GcsBatchCopyCommand(
+                           override val source: GcsPath,
+                           override val destination: GcsPath,
+                           override val overwrite: Boolean,
+                           rewriteToken: Option[String] = None
+                         ) extends IoCopyCommand(source, destination, overwrite) with GcsBatchIoCommand[Unit, RewriteResponse] {
+  val sourceBlob = source.blob
+  val destinationBlob = destination.blob
+  
+  override def operation: StorageRequest[RewriteResponse] = {
+    val rewriteOperation = source.apiStorage.objects().rewrite(sourceBlob.getBucket, sourceBlob.getName, destinationBlob.getBucket, destinationBlob.getName, null)
+    // Set the rewrite token if present
+    rewriteToken foreach rewriteOperation.setRewriteToken 
+    rewriteOperation
+  }
+
+  /**
+    * Clone this command with the give rewrite token
+    */
+  def withRewriteToken(rewriteToken: String) = copy(rewriteToken = Option(rewriteToken))
+  
+  override def onSuccess(response: RewriteResponse, httpHeaders: HttpHeaders) = {
+    if (response.getDone) super.onSuccess(response, httpHeaders)
+    else {
+      Right(withRewriteToken(response.getRewriteToken))
+    }
+  }
+  
+  override def mapGoogleResponse(response: RewriteResponse): Unit = ()
+}
+
+case class GcsBatchDeleteCommand(
+                                  override val file: GcsPath,
+                                  override val swallowIOExceptions: Boolean
+                                ) extends IoDeleteCommand(file, swallowIOExceptions) with GcsBatchIoCommand[Unit, Void] {
+  private val blob = file.blob
+  def operation = file.apiStorage.objects().delete(blob.getBucket, blob.getName)
+  override protected def mapGoogleResponse(response: Void): Unit = ()
+}
+
+/**
+  * Base trait for commands that use the objects.get() operation. (e.g: size, crc32, ...)
+  */
+sealed trait GcsBatchGetCommand[T] extends GcsBatchIoCommand[T, StorageObject] {
+  def file: GcsPath
+  private val blob = file.blob
+  override def operation: StorageRequest[StorageObject] = file.apiStorage.objects().get(blob.getBucket, blob.getName)
+}
+
+case class GcsBatchSizeCommand(override val file: GcsPath) extends IoSizeCommand(file) with GcsBatchGetCommand[Long] {
+  override def mapGoogleResponse(response: StorageObject): Long = response.getSize.longValue()
+}
+
+case class GcsBatchCrc32Command(override val file: GcsPath) extends IoHashCommand(file) with GcsBatchGetCommand[String] {
+  override def mapGoogleResponse(response: StorageObject): String = response.getCrc32c
+}
+
+case class GcsBatchTouchCommand(override val file: GcsPath) extends IoTouchCommand(file) with GcsBatchGetCommand[Unit] {
+  override def mapGoogleResponse(response: StorageObject): Unit = ()
+}
diff --git a/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GcsPathBuilderSpec.scala b/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GcsPathBuilderSpec.scala
index 697313b41..fbfbf10ef 100644
--- a/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GcsPathBuilderSpec.scala
+++ b/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GcsPathBuilderSpec.scala
@@ -1,11 +1,10 @@
 package cromwell.filesystems.gcs
 
-import com.google.cloud.RetryParams
+import com.google.cloud.NoCredentials
 import com.google.cloud.storage.contrib.nio.CloudStorageConfiguration
 import cromwell.core.path._
-import cromwell.core.path.proxy.RetryableFileSystemProviderProxy
 import cromwell.core.{TestKitSuite, WorkflowOptions}
-import cromwell.filesystems.gcs.auth.{GoogleAuthMode, GoogleAuthModeSpec}
+import cromwell.filesystems.gcs.auth.GoogleAuthModeSpec
 import org.scalatest.prop.Tables.Table
 import org.scalatest.{FlatSpecLike, Matchers}
 
@@ -13,35 +12,30 @@ class GcsPathBuilderSpec extends TestKitSuite with FlatSpecLike with Matchers wi
 
   behavior of "GcsPathBuilder"
 
-  it should "create a path with a retryable provider" in {
-    val path = retryablePathBuilder.build("gs://bucket/object")
-    path.isSuccess shouldBe true
-    path.get.nioPath.getFileSystem.provider() shouldBe a[RetryableFileSystemProviderProxy[_]]
-  }
-
   it should "use google project credentials when provided in the workflow options" in {
     GoogleAuthModeSpec.assumeHasApplicationDefaultCredentials()
 
     val wfOptionsWithProject = WorkflowOptions.fromMap(Map("google_project" -> "my_project")).get
 
-    val gcsPathBuilderWithProjectInfo = new GcsPathBuilder(
-      GoogleAuthMode.NoAuthMode,
-      RetryParams.defaultInstance(),
+    val gcsPathBuilderWithProjectInfo = GcsPathBuilder.fromCredentials(
+      NoCredentials.getInstance(),
+      "cromwell-test",
+      None,
       CloudStorageConfiguration.DEFAULT,
       wfOptionsWithProject
     )
 
-    gcsPathBuilderWithProjectInfo.getProjectId shouldBe "my_project"
+    gcsPathBuilderWithProjectInfo.projectId shouldBe "my_project"
   }
 
-  it should behave like truncateCommonRoots(retryablePathBuilder, pathsToTruncate)
+  it should behave like truncateCommonRoots(pathBuilder, pathsToTruncate)
 
   goodPaths foreach { goodPath =>
-    it should behave like buildGoodPath(retryablePathBuilder, goodPath)
+    it should behave like buildGoodPath(pathBuilder, goodPath)
   }
 
   badPaths foreach { badPath =>
-    it should behave like buildBadPath(retryablePathBuilder, badPath)
+    it should behave like buildBadPath(pathBuilder, badPath)
   }
 
   private def pathsToTruncate = Table(
@@ -361,13 +355,13 @@ class GcsPathBuilderSpec extends TestKitSuite with FlatSpecLike with Matchers wi
     BadPath("an absolute file path", "/hello/world", "/hello/world does not have a gcs scheme")
   )
 
-  private lazy val retryablePathBuilder = {
+  private lazy val pathBuilder = {
     GoogleAuthModeSpec.assumeHasApplicationDefaultCredentials()
 
-    new RetryableGcsPathBuilder(
-      GoogleAuthMode.NoAuthMode,
-      RetryParams.defaultInstance(),
-      CustomRetryParams.Default,
+    GcsPathBuilder.fromCredentials(
+      NoCredentials.getInstance(),
+      "cromwell-test",
+      None,
       CloudStorageConfiguration.DEFAULT,
       WorkflowOptions.empty
     )
diff --git a/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GoogleConfigurationSpec.scala b/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GoogleConfigurationSpec.scala
index 80fd1c137..ab868fbcc 100644
--- a/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GoogleConfigurationSpec.scala
+++ b/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/GoogleConfigurationSpec.scala
@@ -3,6 +3,7 @@ package cromwell.filesystems.gcs
 import better.files.File
 import com.typesafe.config.{ConfigException, ConfigFactory}
 import cromwell.filesystems.gcs.GoogleConfiguration.GoogleConfigurationException
+import cromwell.filesystems.gcs.auth.ServiceAccountMode.{JsonFileFormat, PemFileFormat}
 import cromwell.filesystems.gcs.auth.{ApplicationDefaultMode, RefreshTokenMode, ServiceAccountMode, UserMode}
 import org.scalatest.{FlatSpec, Matchers}
 
@@ -12,7 +13,8 @@ class GoogleConfigurationSpec extends FlatSpec with Matchers {
   behavior of "GoogleConfiguration"
 
   it should "parse all manner of well-formed auths" in {
-    val mockFile = File.newTemporaryFile()
+    val pemMockFile = File.newTemporaryFile()
+    val jsonMockFile = File.newTemporaryFile()
 
     val righteousGoogleConfig =
       s"""
@@ -34,14 +36,19 @@ class GoogleConfigurationSpec extends FlatSpec with Matchers {
         |      name = "name-user"
         |      scheme = "user_account"
         |      user = "me"
-        |      secrets-file = "${mockFile.pathAsString}"
+        |      secrets-file = "${pemMockFile.pathAsString}"
         |      data-store-dir = "/where/the/data/at"
         |    },
         |    {
-        |      name = "name-service"
+        |      name = "name-pem-service"
         |      scheme = "service_account"
         |      service-account-id = "my-google-account"
-        |      pem-file = "${mockFile.pathAsString}"
+        |      pem-file = "${pemMockFile.pathAsString}"
+        |    },
+        |    {
+        |      name = "name-json-service"
+        |      scheme = "service_account"
+        |      json-file = "${jsonMockFile.pathAsString}"
         |    }
         |  ]
         |}
@@ -51,7 +58,7 @@ class GoogleConfigurationSpec extends FlatSpec with Matchers {
     val gconf = GoogleConfiguration(ConfigFactory.parseString(righteousGoogleConfig))
 
     gconf.applicationName shouldBe "cromwell"
-    gconf.authsByName should have size 4
+    gconf.authsByName should have size 5
 
     val auths = gconf.authsByName.values
 
@@ -65,15 +72,21 @@ class GoogleConfigurationSpec extends FlatSpec with Matchers {
 
     val user = (auths collectFirst { case a: UserMode => a }).get
     user.name shouldBe "name-user"
-    user.secretsPath shouldBe mockFile.pathAsString
+    user.secretsPath shouldBe pemMockFile.pathAsString
     user.datastoreDir shouldBe "/where/the/data/at"
 
-    val service = (auths collectFirst { case a: ServiceAccountMode => a }).get
-    service.name shouldBe "name-service"
-    service.accountId shouldBe "my-google-account"
-    service.pemPath shouldBe mockFile.pathAsString
+    val servicePem = (auths collectFirst { case a: ServiceAccountMode if a.name == "name-pem-service" => a }).get
+    servicePem.name shouldBe "name-pem-service"
+    servicePem.fileFormat.asInstanceOf[PemFileFormat].accountId shouldBe "my-google-account"
+    servicePem.fileFormat.file shouldBe pemMockFile.pathAsString
+
+    val serviceJson = (auths collectFirst { case a: ServiceAccountMode if a.name == "name-json-service" => a }).get
+    serviceJson.name shouldBe "name-json-service"
+    serviceJson.fileFormat.isInstanceOf[JsonFileFormat] shouldBe true
+    serviceJson.fileFormat.file shouldBe jsonMockFile.pathAsString
 
-    mockFile.delete(true)
+    pemMockFile.delete(true)
+    jsonMockFile.delete(true)
   }
 
 
@@ -96,6 +109,46 @@ class GoogleConfigurationSpec extends FlatSpec with Matchers {
   }
 
   it should "not parse a configuration stanza with wrong cromwell auth" in {
+    val doubleServiceAccountCredentials =
+      """
+        |google {
+        |  application-name = "cromwell"
+        |
+        |  auths = [
+        |    {
+        |      name = "service-account"
+        |      scheme = "service-account"
+        |      service-account-id = "my-google-account"
+        |      pem-file = "path/to/file.pem"
+        |      json-file = "path/to/json.pem"
+        |    }
+        |  ]
+        |}
+      """.stripMargin
+
+    a[GoogleConfigurationException] shouldBe thrownBy {
+      GoogleConfiguration(ConfigFactory.parseString(doubleServiceAccountCredentials))
+    }
+
+    val noServiceAccountCredentials =
+      """
+        |google {
+        |  application-name = "cromwell"
+        |
+        |  auths = [
+        |    {
+        |      name = "service-account"
+        |      scheme = "service-account"
+        |      service-account-id = "my-google-account"
+        |    }
+        |  ]
+        |}
+      """.stripMargin
+
+    a[GoogleConfigurationException] shouldBe thrownBy {
+      GoogleConfiguration(ConfigFactory.parseString(noServiceAccountCredentials))
+    }
+    
     val unsupported =
       """
         |google {
diff --git a/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/auth/GoogleAuthModeSpec.scala b/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/auth/GoogleAuthModeSpec.scala
index 4f3dcd56f..1ce3d5986 100644
--- a/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/auth/GoogleAuthModeSpec.scala
+++ b/filesystems/gcs/src/test/scala/cromwell/filesystems/gcs/auth/GoogleAuthModeSpec.scala
@@ -1,6 +1,6 @@
 package cromwell.filesystems.gcs.auth
 
-import cromwell.core.WorkflowOptions
+import com.google.auth.oauth2.GoogleCredentials
 import org.scalatest.Assertions._
 
 import scala.util.{Failure, Try}
@@ -15,10 +15,7 @@ object GoogleAuthModeSpec {
   }
 
   private lazy val tryApplicationDefaultCredentials: Try[Unit] = Try {
-    val authMode = ApplicationDefaultMode("application-default")
-    val workflowOptions = WorkflowOptions.empty
-    authMode.authCredentials(workflowOptions)
-    authMode.credential(workflowOptions)
+    GoogleCredentials.getApplicationDefault
     ()
   }
 }
diff --git a/project/Dependencies.scala b/project/Dependencies.scala
index f1e3b83e2..628a0ea5d 100644
--- a/project/Dependencies.scala
+++ b/project/Dependencies.scala
@@ -1,55 +1,55 @@
 import sbt._
 
 object Dependencies {
-  lazy val lenthallV = "0.21"
-  lazy val wdl4sV = "0.10"
-  lazy val sprayV = "1.3.3"
-  /*
-  spray-json is an independent project from the "spray suite"
-  - https://github.com/spray/spray
-  - https://github.com/spray/spray-json
-  - http://spray.io/documentation/1.2.2/spray-httpx/spray-json-support/
-  - http://doc.akka.io/docs/akka/2.4/scala/http/common/json-support.html#akka-http-spray-json
-   */
-  lazy val sprayJsonV = "1.3.2"
-  lazy val akkaV = "2.4.16"
-  lazy val akkaHttpV = "2.4.8"
-  lazy val slickV = "3.1.1"
+  lazy val lenthallV = "0.25"
+  lazy val wdl4sV = "0.14-7c693a3-SNAP"
+
+  lazy val akkaV = "2.5.3"
+  lazy val akkaHttpV = "10.0.9"
+
+  lazy val slickV = "3.2.0"
+
   lazy val googleClientApiV = "1.22.0"
   lazy val googleGenomicsServicesApiV = "1.22.0"
-  lazy val betterFilesV = "2.16.0"
-  lazy val catsV = "0.7.2"
+  lazy val betterFilesV = "2.17.1"
+  lazy val catsV = "0.9.0"
+  lazy val fs2V = "0.9.7"
+
+  lazy val pegdownV = "1.6.0"
+  lazy val scalatestV = "3.0.2"
 
   // Internal collections of dependencies
 
+  private val fs2Test = "co.fs2" %% "fs2-io" % fs2V % "test"
+
   private val catsDependencies = List(
-    "org.typelevel" %% "cats" % "0.7.2",
-    "com.github.benhutchison" %% "mouse" % "0.5"
+    "org.typelevel" %% "cats" % catsV,
+    "com.github.benhutchison" %% "mouse" % "0.9"
   ) map (_
     /*
     Exclude test framework cats-laws and its transitive dependency scalacheck.
     If sbt detects scalacheck, it tries to run it.
     Explicitly excluding the two problematic artifacts instead of including the three (or four?).
     https://github.com/typelevel/cats/tree/v0.7.2#getting-started
-    Re "_2.11", see also: https://github.com/sbt/sbt/issues/1518
+    Re "_2.12", see also: https://github.com/sbt/sbt/issues/1518
      */
-    exclude("org.typelevel", "cats-laws_2.11")
-    exclude("org.typelevel", "cats-kernel-laws_2.11")
+    exclude("org.typelevel", "cats-laws_2.12")
+    exclude("org.typelevel", "cats-kernel-laws_2.12")
     )
 
   private val baseDependencies = List(
     "org.broadinstitute" %% "lenthall" % lenthallV,
-    "com.iheart" %% "ficus" % "1.3.0",
-    "org.scalatest" %% "scalatest" % "3.0.0" % Test,
-    "org.pegdown" % "pegdown" % "1.6.0" % Test,
-    "org.specs2" %% "specs2-mock" % "3.8.5" % Test
-  ) ++ catsDependencies
+    "com.iheart" %% "ficus" % "1.4.1",
+    "org.scalatest" %% "scalatest" % scalatestV % Test,
+    "org.pegdown" % "pegdown" % pegdownV % Test,
+    "org.specs2" %% "specs2-mock" % "3.8.9" % Test // 3.9.X doesn't enjoy the spark backend or refined
+  ) ++ catsDependencies :+ fs2Test
 
   private val slf4jBindingDependencies = List(
     // http://logback.qos.ch/dependencies.html
-    "ch.qos.logback" % "logback-classic" % "1.2.1",
-    "ch.qos.logback" % "logback-access" % "1.2.1",
-    "org.codehaus.janino" % "janino" % "3.0.1"
+    "ch.qos.logback" % "logback-classic" % "1.2.3",
+    "ch.qos.logback" % "logback-access" % "1.2.3",
+    "org.codehaus.janino" % "janino" % "3.0.7"
   )
 
   private val slickDependencies = List(
@@ -64,17 +64,17 @@ object Dependencies {
     "com.mattbertolini" % "liquibase-slf4j" % "2.0.0"
   )
 
-  private val sprayServerDependencies = List(
-    "io.spray" %% "spray-can" % sprayV,
-    "io.spray" %% "spray-routing-shapeless2" % sprayV,
-    "io.spray" %% "spray-http" % sprayV,
-    "io.spray" %% "spray-testkit" % sprayV % Test
+  val akkaHttpDependencies = List(
+    "com.typesafe.akka" %% "akka-http" % akkaHttpV,
+    "com.typesafe.akka" %% "akka-http-testkit" % akkaHttpV % Test
   )
 
+  val akkaHttpServerDependencies = akkaHttpDependencies :+  "org.webjars" % "swagger-ui" % "2.1.1"
+
   private val googleApiClientDependencies = List(
     // Used by swagger, but only in tests.  This overrides an older 2.1.3 version of jackson-core brought in by
     // these Google dependencies, but which isn't properly evicted by IntelliJ's sbt integration.
-    "com.fasterxml.jackson.core" % "jackson-core" % "2.8.2",
+    "com.fasterxml.jackson.core" % "jackson-core" % "2.8.9",
     // The exclusions prevent guava 13 from colliding at assembly time with guava 18 brought in elsewhere.
     "com.google.api-client" % "google-api-client-java6" % googleClientApiV exclude("com.google.guava", "guava-jdk5"),
     "com.google.api-client" % "google-api-client-jackson2" % googleClientApiV exclude("com.google.guava", "guava-jdk5")
@@ -82,11 +82,11 @@ object Dependencies {
 
   private val googleCloudDependencies = List(
     "com.google.apis" % "google-api-services-genomics" % ("v1alpha2-rev64-" + googleGenomicsServicesApiV),
-    "com.google.cloud" % "google-cloud-nio" % "0.3.0"
+    "com.google.cloud" % "google-cloud-nio" % "0.20.1-alpha"
       exclude("com.google.api.grpc", "grpc-google-common-protos")
       exclude("com.google.cloud.datastore", "datastore-v1-protos")
       exclude("org.apache.httpcomponents", "httpclient"),
-    "org.apache.httpcomponents" % "httpclient" % "4.5.2"
+    "org.apache.httpcomponents" % "httpclient" % "4.5.3"
   )
 
   private val dbmsDependencies = List(
@@ -99,7 +99,12 @@ object Dependencies {
     - serverTimezone=UTC via http://stackoverflow.com/a/36793896/3320205
     - nullNamePatternMatchesAll=true via https://liquibase.jira.com/browse/CORE-2723
      */
-    "mysql" % "mysql-connector-java" % "5.1.39"
+    "mysql" % "mysql-connector-java" % "5.1.42"
+  )
+
+  private val refinedTypeDependenciesList = List(
+    "org.scala-lang" % "scala-compiler" % Settings.ScalaVersion,
+    "eu.timepit" %% "refined" % "0.8.2"
   )
 
   // Sub-project dependencies, added in addition to any dependencies inherited from .dependsOn().
@@ -108,21 +113,22 @@ object Dependencies {
     "com.github.pathikrit" %% "better-files" % betterFilesV
   )
 
-  val databaseSqlDependencies = baseDependencies ++ slickDependencies ++ dbmsDependencies
+  val databaseSqlDependencies = baseDependencies ++ slickDependencies ++ dbmsDependencies ++ refinedTypeDependenciesList
 
   val coreDependencies = List(
-    "com.typesafe.scala-logging" %% "scala-logging" % "3.4.0",
-    "org.broadinstitute" %% "wdl4s" % wdl4sV,
-    "org.apache.commons" % "commons-lang3" % "3.4",
-    "io.spray" %% "spray-json" % sprayJsonV,
-    "com.typesafe" % "config" % "1.3.0",
+    "com.typesafe.scala-logging" %% "scala-logging" % "3.6.0",
+    "org.broadinstitute" %% "wdl4s-wdl" % wdl4sV,
+    "org.apache.commons" % "commons-lang3" % "3.6",
+    "com.typesafe.akka" %% "akka-http-spray-json" % akkaHttpV,
+    "com.typesafe" % "config" % "1.3.1",
     "com.typesafe.akka" %% "akka-actor" % akkaV,
     "com.typesafe.akka" %% "akka-slf4j" % akkaV,
     "com.typesafe.akka" %% "akka-testkit" % akkaV % Test,
-    "com.google.guava" % "guava" % "20.0",
-    "com.typesafe.akka" %% "akka-http-core" % akkaHttpV,
-    "com.typesafe.akka" %% "akka-stream-testkit" % akkaHttpV,
-    "com.typesafe.akka" %% "akka-http-spray-json-experimental" % akkaHttpV
+    "com.google.guava" % "guava" % "22.0",
+    "com.google.auth" % "google-auth-library-oauth2-http" % "0.7.0",
+    "com.typesafe.akka" %% "akka-stream-testkit" % akkaV,
+    "com.chuusai" %% "shapeless" % "2.3.2",
+    "com.github.scopt" %% "scopt" % "3.6.0"
   ) ++ baseDependencies ++ googleApiClientDependencies ++
     // TODO: We're not using the "F" in slf4j. Core only supports logback, specifically the WorkflowLogger.
     slf4jBindingDependencies
@@ -131,31 +137,30 @@ object Dependencies {
     "com.github.pathikrit" %% "better-files" % betterFilesV % Test
   ) ++ liquibaseDependencies ++ dbmsDependencies
 
-  val htCondorBackendDependencies = List(
-    "com.twitter" %% "chill" % "0.8.0",
-    "org.mongodb" %% "casbah" % "3.0.0"
-  )
-
-  val jesBackendDependencies = List(
-    "org.scala-lang" % "scala-compiler" % Settings.ScalaVersion,
-    "eu.timepit" %% "refined" % "0.7.0"
-  )
-
-  val tesBackendDependencies = List(
-    "io.spray" %% "spray-client" % sprayV
-  ) ++ sprayServerDependencies
-
-  val sparkBackendDependencies = List(
-    "io.spray" %% "spray-client" % sprayV
-  ) ++ sprayServerDependencies
+  val cromwellApiClientDependencies = List(
+    "com.typesafe.akka" %% "akka-actor" % akkaV,
+    "com.typesafe.akka" %% "akka-http-spray-json" % akkaHttpV,
+    "com.github.pathikrit" %% "better-files" % betterFilesV,
+    "org.scalatest" %% "scalatest" % scalatestV % Test,
+    "org.pegdown" % "pegdown" % pegdownV % Test
+  ) ++ akkaHttpDependencies
 
   val engineDependencies = List(
-    "org.webjars" % "swagger-ui" % "2.1.1",
     "commons-codec" % "commons-codec" % "1.10",
     "commons-io" % "commons-io" % "2.5",
+    "com.storm-enroute" %% "scalameter" % "0.8.2"
+      exclude("com.fasterxml.jackson.core", "jackson-databind")
+      exclude("com.fasterxml.jackson.module", "jackson-module-scala")
+      exclude("org.scala-tools.testing", "test-interface"),
+    "com.fasterxml.jackson.core" % "jackson-databind" % "2.8.9",
+    "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.8.9",
     "io.swagger" % "swagger-parser" % "1.0.22" % Test,
     "org.yaml" % "snakeyaml" % "1.17" % Test
-  ) ++ sprayServerDependencies
+  ) ++ akkaHttpServerDependencies
 
   val rootDependencies = slf4jBindingDependencies
+
+  val jesBackendDependencies = refinedTypeDependenciesList
+  val tesBackendDependencies = akkaHttpDependencies
+  val sparkBackendDependencies = akkaHttpDependencies
 }
diff --git a/project/Publishing.scala b/project/Publishing.scala
index 244ece80b..56301c899 100644
--- a/project/Publishing.scala
+++ b/project/Publishing.scala
@@ -7,7 +7,7 @@ object Publishing {
   private def artifactoryResolver(isSnapshot: Boolean): Resolver = {
     val repoType = if (isSnapshot) "snapshot" else "release"
     val repoUrl =
-      s"https://artifactory.broadinstitute.org/artifactory/libs-$repoType-local;build.timestamp=$buildTimestamp"
+      s"https://broadinstitute.jfrog.io/broadinstitute/libs-$repoType-local;build.timestamp=$buildTimestamp"
     val repoName = "artifactory-publish"
     repoName at repoUrl
   }
@@ -15,7 +15,7 @@ object Publishing {
   private val artifactoryCredentials: Credentials = {
     val username = sys.env.getOrElse("ARTIFACTORY_USERNAME", "")
     val password = sys.env.getOrElse("ARTIFACTORY_PASSWORD", "")
-    Credentials("Artifactory Realm", "artifactory.broadinstitute.org", username, password)
+    Credentials("Artifactory Realm", "broadinstitute.jfrog.io", username, password)
   }
 
   def publishingSettings: Seq[Setting[_]] =
diff --git a/project/Settings.scala b/project/Settings.scala
index aa217201a..cbfc0a807 100644
--- a/project/Settings.scala
+++ b/project/Settings.scala
@@ -13,8 +13,8 @@ object Settings {
 
   val commonResolvers = List(
     Resolver.jcenterRepo,
-    "Broad Artifactory Releases" at "https://artifactory.broadinstitute.org/artifactory/libs-release/",
-    "Broad Artifactory Snapshots" at "https://artifactory.broadinstitute.org/artifactory/libs-snapshot/"
+    "Broad Artifactory Releases" at "https://broadinstitute.jfrog.io/broadinstitute/libs-release/",
+    "Broad Artifactory Snapshots" at "https://broadinstitute.jfrog.io/broadinstitute/libs-snapshot/"
   )
 
   /*
@@ -25,17 +25,16 @@ object Settings {
     https://github.com/sbt/sbt-assembly/issues/69
     https://github.com/scala/pickling/issues/10
 
-    Other fancy flags from
+    Other fancy flags from https://tpolecat.github.io/2017/04/25/scalac-flags.html.
 
-    http://blog.threatstack.com/useful-scalac-options-for-better-scala-development-part-1
-
-    and
-
-    https://tpolecat.github.io/2014/04/11/scalac-flags.html
+    The following isn't used (yet), and in general is an exercise in pain for 2.12 with Cromwell.
+    It'd certainly be nice to have, but params causes a world of hurt. Interested parties are encouraged
+    to take a stab at it.
 
+    "-Ywarn-unused:params"              // Warn if a value parameter is unused.
   */
   val compilerSettings = List(
-    "-Xlint",
+    "-explaintypes",
     "-feature",
     "-Xmax-classfile-name", "200",
     "-target:jvm-1.8",
@@ -43,13 +42,37 @@ object Settings {
     "-unchecked",
     "-deprecation",
     "-Xfuture",
+    "-Xlint:adapted-args",
+    "-Xlint:by-name-right-associative",
+    "-Xlint:constant",
+    "-Xlint:delayedinit-select",
+    "-Xlint:doc-detached",
+    "-Xlint:inaccessible",
+    "-Xlint:infer-any",
+    "-Xlint:missing-interpolator",
+    "-Xlint:nullary-override",
+    "-Xlint:nullary-unit",
+    "-Xlint:option-implicit",
+    "-Xlint:package-object-classes",
+    "-Xlint:poly-implicit-overload",
+    "-Xlint:private-shadow",
+    "-Xlint:stars-align",
+    "-Xlint:type-parameter-shadow",
+    "-Xlint:unsound-match",
     "-Yno-adapted-args",
     "-Ywarn-dead-code",
     "-Ywarn-numeric-widen",
     "-Ywarn-value-discard",
-    "-Ywarn-unused",
-    "-Ywarn-unused-import",
-    "-Xfatal-warnings"
+    "-Ywarn-inaccessible",
+    "-Ywarn-unused:implicits",
+    "-Ywarn-unused:privates",
+    "-Ywarn-unused:locals",
+    "-Ywarn-unused:patvars"
+  )
+
+  val consoleHostileSettings = List(
+    "-Ywarn-unused:imports", // warns about every unused import on every command.
+    "-Xfatal-warnings"       // makes those warnings fatal.
   )
 
   val docSettings = List(
@@ -100,14 +123,15 @@ object Settings {
     )
   )
 
-  val ScalaVersion = "2.11.8"
+  val ScalaVersion = "2.12.3"
   val commonSettings = ReleasePlugin.projectSettings ++ testSettings ++ assemblySettings ++
     dockerSettings ++ cromwellVersionWithGit ++ publishingSettings ++ List(
     organization := "org.broadinstitute",
     scalaVersion := ScalaVersion,
     resolvers ++= commonResolvers,
-    scalacOptions ++= compilerSettings,
+    scalacOptions ++= (compilerSettings ++ consoleHostileSettings),
     scalacOptions in (Compile, doc) ++= docSettings,
+    scalacOptions in (Compile, console) := compilerSettings,
     parallelExecution := false
   )
 
@@ -135,6 +159,22 @@ object Settings {
     libraryDependencies ++= databaseMigrationDependencies
   ) ++ commonSettings
 
+  val cromwellApiClientSettings = List(
+    name := "cromwell-api-client",
+    libraryDependencies ++= cromwellApiClientDependencies,
+    organization := "org.broadinstitute",
+    scalaVersion := ScalaVersion,
+    scalacOptions ++= (compilerSettings ++ consoleHostileSettings),
+    scalacOptions in (Compile, doc) ++= docSettings,
+    scalacOptions in (Compile, console) := compilerSettings,
+    resolvers ++= commonResolvers
+  ) ++ ReleasePlugin.projectSettings ++ testSettings ++ assemblySettings ++
+    cromwellVersionWithGit ++ publishingSettings
+
+  val dockerHashingSettings = List(
+    name := "cromwell-docker-hashing"
+  ) ++ commonSettings
+
   val backendSettings = List(
     name := "cromwell-backend"
   ) ++ commonSettings
@@ -148,11 +188,6 @@ object Settings {
     libraryDependencies ++= tesBackendDependencies
   ) ++ commonSettings
 
-  val htCondorBackendSettings = List(
-    name := "cromwell-htcondor-backend",
-    libraryDependencies ++= htCondorBackendDependencies
-  ) ++ commonSettings
-
   val sparkBackendSettings = List(
     name := "cromwell-spark-backend",
     libraryDependencies ++= sparkBackendDependencies
diff --git a/project/Testing.scala b/project/Testing.scala
index 46ca8f155..157e6bc98 100644
--- a/project/Testing.scala
+++ b/project/Testing.scala
@@ -6,6 +6,7 @@ object Testing {
   lazy val DockerTest = config("docker") extend Test
   lazy val NoDockerTest = config("nodocker") extend Test
   lazy val CromwellIntegrationTest = config("integration") extend Test
+  lazy val CromwellBenchmarkTest = config("benchmark") extend Test
   lazy val CromwellNoIntegrationTest = config("nointegration") extend Test
   lazy val DbmsTest = config("dbms") extend Test
 
@@ -14,10 +15,8 @@ object Testing {
   lazy val DontUseDockerTaggedTests = Tests.Argument(TestFrameworks.ScalaTest, "-l", DockerTestTag)
 
   lazy val CromwellIntegrationTestTag = "CromwellIntegrationTest"
-  lazy val UseCromwellIntegrationTaggedTests =
-    Tests.Argument(TestFrameworks.ScalaTest, "-n", CromwellIntegrationTestTag)
-  lazy val DontUseCromwellIntegrationTaggedTests =
-    Tests.Argument(TestFrameworks.ScalaTest, "-l", CromwellIntegrationTestTag)
+  lazy val UseCromwellIntegrationTaggedTests = Tests.Argument(TestFrameworks.ScalaTest, "-n", CromwellIntegrationTestTag)
+  lazy val DontUseCromwellIntegrationTaggedTests = Tests.Argument(TestFrameworks.ScalaTest, "-l", CromwellIntegrationTestTag)
 
   lazy val GcsIntegrationTestTag = "GcsIntegrationTest"
   lazy val UseGcsIntegrationTaggedTests = Tests.Argument(TestFrameworks.ScalaTest, "-n", GcsIntegrationTestTag)
@@ -54,7 +53,11 @@ object Testing {
     // `nointegration:test` - Run all tests, except integration
     testOptions in CromwellNoIntegrationTest := (testOptions in AllTests).value ++ Seq(DontUseCromwellIntegrationTaggedTests, DontUseGcsIntegrationTaggedTests, DontUsePostMVPTaggedTests),
     // `dbms:test` - Run database management tests.
-    testOptions in DbmsTest := (testOptions in AllTests).value ++ Seq(UseDbmsTaggedTests)
+    testOptions in DbmsTest := (testOptions in AllTests).value ++ Seq(UseDbmsTaggedTests),
+    // Add scalameter as a test framework in the CromwellBenchmarkTest scope
+    testFrameworks in CromwellBenchmarkTest += new TestFramework("org.scalameter.ScalaMeterFramework"),
+    // Don't execute benchmarks in parallel
+    parallelExecution in CromwellBenchmarkTest := false
   )
   /*
     TODO: This syntax of test in (NoTests, assembly) isn't correct
@@ -84,6 +87,7 @@ object Testing {
       .configs(DockerTest).settings(inConfig(DockerTest)(Defaults.testTasks): _*)
       .configs(NoDockerTest).settings(inConfig(NoDockerTest)(Defaults.testTasks): _*)
       .configs(CromwellIntegrationTest).settings(inConfig(CromwellIntegrationTest)(Defaults.testTasks): _*)
+      .configs(CromwellBenchmarkTest).settings(inConfig(CromwellBenchmarkTest)(Defaults.testTasks): _*)
       .configs(CromwellNoIntegrationTest).settings(inConfig(CromwellNoIntegrationTest)(Defaults.testTasks): _*)
       .configs(DbmsTest).settings(inConfig(DbmsTest)(Defaults.testTasks): _*)
   }
diff --git a/project/Version.scala b/project/Version.scala
index a4dc20d69..3e4ee2bf0 100644
--- a/project/Version.scala
+++ b/project/Version.scala
@@ -4,7 +4,7 @@ import sbt._
 
 object Version {
   // Upcoming release, or current if we're on a master / hotfix branch
-  val cromwellVersion = "25"
+  val cromwellVersion = "29"
 
   // Adapted from SbtGit.versionWithGit
   def cromwellVersionWithGit: Seq[Setting[_]] =
diff --git a/project/build.properties b/project/build.properties
index 35c88bab7..64317fdae 100644
--- a/project/build.properties
+++ b/project/build.properties
@@ -1 +1 @@
-sbt.version=0.13.12
+sbt.version=0.13.15
diff --git a/project/plugins.sbt b/project/plugins.sbt
index 4de0d6862..c4bfd6abc 100644
--- a/project/plugins.sbt
+++ b/project/plugins.sbt
@@ -1,4 +1,4 @@
-addSbtPlugin("se.marcuslonnberg" % "sbt-docker" % "1.4.0")
+addSbtPlugin("se.marcuslonnberg" % "sbt-docker" % "1.4.1")
 addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.3")
 addSbtPlugin("io.spray" % "sbt-revolver" % "0.8.0")
 /*
@@ -7,5 +7,5 @@ See https://github.com/broadinstitute/cromwell/issues/645
  */
 addSbtPlugin("com.typesafe.sbt" % "sbt-git" % "0.7.1")
 addSbtPlugin("com.github.gseitz" % "sbt-release" % "1.0.3")
-addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.3.5")
+addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.5.0")
 addSbtPlugin("org.scoverage" % "sbt-coveralls" % "1.1.0")
diff --git a/release/release_inputs.json b/release/release_inputs.json
new file mode 100644
index 000000000..5a018c652
--- /dev/null
+++ b/release/release_inputs.json
@@ -0,0 +1,4 @@
+{
+  "release_cromwell.githubToken": "<<GithubAPIToken>>",
+  "release_cromwell.organization": "broadinstitute"
+}
diff --git a/release/release_workflow.wdl b/release/release_workflow.wdl
new file mode 100644
index 000000000..a1297556d
--- /dev/null
+++ b/release/release_workflow.wdl
@@ -0,0 +1,377 @@
+task do_release {
+   # Repo to release
+   String repo
+   # Current version being released
+   String releaseV
+   # Next version
+   String nextV
+   # Command that will update the appropriate file for the current release
+   String updateVersionCommand
+   
+   # Commands that will update previously released/published dependencies in this repo
+   Array[String] dependencyCommands = []
+   
+   # Can be swapped out to try this on a fork
+   String organization
+    
+   command {
+     set -e
+     set -x 
+     
+     # Clone repo and checkout develop
+     git clone git@github.com:${organization}/${repo}.git
+     cd ${repo}
+     git checkout develop
+     git pull --rebase
+
+     # Expect the version number on develop to be the version TO BE RELEASED
+
+     echo "Updating dependencies"
+     ${sep='\n' dependencyCommands}
+     
+     git add .
+     # If there is nothing to commit, git commit will return 1 which will fail the script.
+     # This ensures we only commit if build.sbt was effectively updated
+     git diff-index --quiet HEAD || git commit -m "Update ${repo} version to ${releaseV}"
+       
+     # wdl4s needs a scala docs update
+     if [ ${repo} == "wdl4s" ]; then
+     
+       # Generate new scaladoc
+       sbt 'set scalacOptions in (Compile, doc) := List("-skip-packages", "better")' doc
+       git checkout gh-pages
+       for subproj in cwl wdl wom; do
+         mkdir -p ${releaseV}/$subproj
+         mv $subproj/target/scala-2.12/api ${releaseV}/$subproj
+       done
+       git add ${releaseV}
+       
+       # Update latest pointer
+       git rm --ignore-unmatch latest
+       ln -s ${releaseV} latest
+       git add latest
+       
+       git diff-index --quiet HEAD || git commit -m "Update Scaladoc"
+       git push origin gh-pages
+       
+       # Update badges on README
+       git checkout develop
+       curl -o scaladoc.png https://img.shields.io/badge/scaladoc-${releaseV}-blue.png
+       curl -o version.png https://img.shields.io/badge/version-${releaseV}-blue.png
+       
+       git add scaladoc.png
+       git add version.png
+       
+       git diff-index --quiet HEAD || git commit -m "Update README badges"
+       git push origin develop
+     fi
+     
+     # Merge develop into master
+     git checkout master
+     git pull --rebase
+     git merge develop --no-edit
+     
+     # Make sure tests pass
+     sbt update
+     JAVA_OPTS=-XX:MaxMetaspaceSize=1024m sbt test
+     
+     # Tag the release
+     git tag ${releaseV}
+     
+     # Push master and push the tags
+     git push origin master
+     git push --tags
+     
+     # Create and push the hotfix branch
+     git checkout -b ${releaseV}_hotfix
+     
+     # Pin centaur for cromwell
+     if [ ${repo} == "cromwell" ]; then
+        centaurDevelopHEAD=$(git ls-remote git://github.com/${organization}/centaur.git | grep refs/heads/develop | cut -f 1)
+        sed -i '' s/CENTAUR_BRANCH=.*/CENTAUR_BRANCH="$centaurDevelopHEAD"/g .travis.yml
+        git add .travis.yml
+        git commit -m "Pin release to centaur branch"
+     fi 
+     
+     git push origin ${releaseV}_hotfix
+     
+     # Assemble jar for cromwell
+     if [ ${repo} == "cromwell" ]; then
+        sbt -Dproject.version=${releaseV} -Dproject.isSnapshot=false assembly
+     fi  
+     
+     # Update develop to point to next release version
+     git checkout develop
+     ${updateVersionCommand}
+     git add .
+     git diff-index --quiet HEAD || git commit -m "Update ${repo} version from ${releaseV} to ${nextV}"
+     git push origin develop
+     
+     pwd > executionDir.txt
+   }
+
+   output {
+     String version = releaseV
+     String executionDir = read_string(repo + "/executionDir.txt")
+   }
+}
+
+task wait_for_published_artifact {
+    String repo
+    String version
+    
+    command <<<
+        checkIfPresent() {
+            isPresent=$(curl -s --head https://broadinstitute.jfrog.io/broadinstitute/libs-release-local/org/broadinstitute/${repo}/${version}/ | head -n 1 | grep -q "HTTP/1.[01] [23]..")
+        }
+        
+        elapsedTime=0
+        checkIfPresent
+        
+        # Allow 1 hour for the file to appear as a published artifact
+        while [ $? -ne 0 ] && [ $elapsedTime -lt 3600 ]; do
+            sleep 10;
+            let "elapsedTime+=10"
+            checkIfPresent
+        done
+        
+        exit $?
+    >>>
+    
+    output {
+        String publishedVersion = version
+    }
+}
+
+task create_update_dependency_command {
+    String dependencyName
+    String newVersion
+    String dependencyFilePath = "build.sbt"
+    
+    command {
+        echo "sed -i '' \"s/${dependencyName}[[:space:]]=.*/${dependencyName} = \\\"${newVersion}\\\"/g\" ${dependencyFilePath}"
+    }
+    
+    output {
+      String updateCommand = read_string(stdout())
+    }
+}
+
+task versionPrep {
+    String organization
+    String repo
+    String file
+    String regexPrefix
+    String updateCommandTemplate
+    
+    String bash_rematch = "{BASH_REMATCH[1]}"
+    command <<<
+        curl -o versionFile https://raw.githubusercontent.com/${organization}/${repo}/develop/${file}
+        regex="${regexPrefix}\"(([0-9]+\.)?([0-9]+))\""
+        
+        if [[ $(cat versionFile) =~ $regex ]]
+        then
+            version="$${bash_rematch}"
+            echo $version > version
+            echo $version | perl -ne 'if (/^([0-9]+\.)?([0-9]+)$/) { $incr = $2 + 1; print "$1$incr\n" }' > nextVersion
+        else
+            exit 1
+        fi
+    >>>
+    
+    output {
+        String version = read_string("version")
+        String nextVersion = read_string("nextVersion")
+        String updateCommand = sub(updateCommandTemplate, "<<VERSION>>", nextVersion)
+    }
+}
+
+task makeGithubRelease {
+    String githubToken
+    String organization
+    File cromwellJar
+    Int oldVersion
+    Int newVersion
+    
+    command <<<
+        set -e
+        set -x
+        
+        # download changelog from master
+        curl https://raw.githubusercontent.com/${organization}/cromwell/master/CHANGELOG.md -o CHANGELOG.md
+        
+        # Extract the latest piece of the changelog corresponding to this release
+        # head remove the last line, next sed escapes all ", and last sed/tr replaces all new lines with \n so it can be used as a JSON string
+        BODY=$(sed -n '/## ${newVersion}/,/## ${oldVersion}/p' CHANGELOG.md | head -n -1 | sed -e 's/"/\\"/g' | sed 's/$/\\n/' | tr -d '\n')
+        
+        # Build the json body for the POST release
+        API_JSON="{\"tag_name\": \"${newVersion}\",\"name\": \"${newVersion}\",\"body\": \"$BODY\",\"draft\": true,\"prerelease\": false}"
+        
+        # POST the release as a draft
+        curl --data "$API_JSON" https://api.github.com/repos/${organization}/cromwell/releases?access_token=${githubToken} -o release_response
+        
+        # parse the response to get the release id and the asset upload url
+        RELEASE_ID=$(python -c "import sys, json; print json.load(sys.stdin)['id']" < release_response)
+        UPLOAD_URL=$(python -c "import sys, json; print json.load(sys.stdin)['upload_url']" < release_response)
+        # Maybe update this when we have basename !
+        UPLOAD_URL=$(sed 's/{.*}/?name=cromwell-${newVersion}.jar/' <<< "$UPLOAD_URL")
+        
+        # Upload the cromwell jar as an asset
+        curl -X POST --data-binary @${cromwellJar} -H "Authorization: token ${githubToken}" -H "Content-Type: application/octet-stream" "$UPLOAD_URL"
+        
+        # Publish the draft
+        curl -X PATCH -d '{"draft": false}' https://api.github.com/repos/${organization}/cromwell/releases/"$RELEASE_ID"?access_token=${githubToken}
+    >>>
+    runtime {
+        docker: "python:2.7"
+    }
+}
+
+workflow release_cromwell {
+  String githubToken
+  String organization
+    
+  Pair[String, String] lenthallAsDependency = ("lenthallV", waitForLenthall.publishedVersion) 
+  Pair[String, String] wdl4sAsDependency = ("wdl4sV", waitForWdl4s.publishedVersion)
+  
+  Array[Pair[String, String]] wdl4sDependencies = [lenthallAsDependency]
+  Array[Pair[String, String]] wdltoolDependencies = [wdl4sAsDependency]
+  Array[Pair[String, String]] cromwellDependencies = [lenthallAsDependency, wdl4sAsDependency]
+  
+  # Regex to find the line setting the current version
+  String dependencyRegexPrefix = "git\\.baseVersion[[:space:]]:=[[:space:]]"
+  # Template command to update the version
+  String dependencyTemplate = "sed -i '' \"s/git\\.baseVersion[[:space:]]:=.*/git.baseVersion := \\\"<<VERSION>>\\\",/g\" build.sbt"
+  
+  String cromwellTemplate = "sed -i '' \"s/cromwellVersion[[:space:]]=.*/cromwellVersion = \\\"<<VERSION>>\\\"/g\" project/Version.scala"
+  String cromwellRegexPrefix = "cromwellVersion[[:space:]]=[[:space:]]"
+
+  String wdl4sTemplate = "sed -i '' \"s/wdl4sVersion[[:space:]]=.*/wdl4sVersion = \\\"<<VERSION>>\\\"/g\" project/Version.scala"
+  String wdl4sRegexPrefix = "wdl4sVersion[[:space:]]=[[:space:]]"
+
+  # Prepare releases by finding out the current version, next version, and update version command
+  call versionPrep as lenthallPrep { input: 
+    organization = organization,
+    repo = "lenthall",
+    file = "build.sbt",
+    regexPrefix = dependencyRegexPrefix,
+    updateCommandTemplate = dependencyTemplate
+  }
+  
+  call versionPrep as wdl4sPrep { input: 
+    organization = organization,
+    repo = "wdl4s",
+    file = "project/Version.scala",
+    regexPrefix = wdl4sRegexPrefix,
+    updateCommandTemplate = wdl4sTemplate
+  }
+  
+  call versionPrep as wdltoolPrep { input: 
+    organization = organization,
+    repo = "wdltool",
+    file = "build.sbt",
+    regexPrefix = dependencyRegexPrefix,
+    updateCommandTemplate = dependencyTemplate
+  }
+  
+  call versionPrep as cromwellPrep { input: 
+    organization = organization,
+    repo = "cromwell",
+    file = "project/Version.scala",
+    regexPrefix = cromwellRegexPrefix,
+    updateCommandTemplate = cromwellTemplate
+  }
+  
+  # Release calls  
+  call do_release as release_lenthall { input: 
+        organization = organization, 
+        repo = "lenthall", 
+        releaseV = lenthallPrep.version,
+        nextV = lenthallPrep.nextVersion,
+        updateVersionCommand = lenthallPrep.updateCommand,
+       }
+       
+  call do_release as release_wdl4s { input: 
+        organization = organization, 
+        repo = "wdl4s", 
+        releaseV = wdl4sPrep.version,
+        nextV = wdl4sPrep.nextVersion,
+        updateVersionCommand = wdl4sPrep.updateCommand,
+        dependencyCommands = wdl4sDependencyCommands.updateCommand
+       }
+       
+  call do_release as release_wdltool { input:
+        organization = organization,
+        repo = "wdltool", 
+        releaseV = wdltoolPrep.version,
+        nextV = wdltoolPrep.nextVersion,
+        updateVersionCommand = wdltoolPrep.updateCommand,
+        dependencyCommands = wdltoolDependencyCommands.updateCommand
+       }  
+       
+  call do_release as release_cromwell { input: 
+        organization = organization, 
+        repo = "cromwell", 
+        releaseV = cromwellPrep.version,
+        nextV = cromwellPrep.nextVersion,
+        updateVersionCommand = cromwellPrep.updateCommand,
+        dependencyCommands = cromwellDependencyCommands.updateCommand
+       }
+  
+  call wait_for_published_artifact as waitForLenthall { input: repo = "lenthall_2.12", version = release_lenthall.version }
+  call wait_for_published_artifact as waitForWdl4s { input: repo = "wdl4s-wdl_2.12", version = release_wdl4s.version }
+  
+  # Generates commands to update wdl4s dependencies
+  scatter(wdl4sDependency in wdl4sDependencies) {
+    String wdl4sDepName = wdl4sDependency.left
+    String wdl4sVersionName = wdl4sDependency.right
+    
+    call create_update_dependency_command as wdl4sDependencyCommands { input: 
+       dependencyName = wdl4sDepName,
+       newVersion = wdl4sVersionName,
+       dependencyFilePath = "build.sbt"
+    }
+  }
+  
+  # Generates commands to update wdltool dependencies
+  scatter(wdltoolDependency in wdltoolDependencies) {
+    String wdltoolDepName = wdltoolDependency.left
+    String wdltoolVersionName = wdltoolDependency.right
+    
+    call create_update_dependency_command as wdltoolDependencyCommands { input: 
+       dependencyName = wdltoolDepName,
+       newVersion = wdltoolVersionName,
+       dependencyFilePath = "build.sbt"
+    }
+  }
+  
+  # Generates commands to update cromwell dependencies
+  scatter(cromwellDependency in cromwellDependencies) {
+    String cromwellDepName = cromwellDependency.left
+    String cromwellVersionName = cromwellDependency.right
+    
+    call create_update_dependency_command as cromwellDependencyCommands { input: 
+       dependencyName = cromwellDepName,
+       newVersion = cromwellVersionName,
+       dependencyFilePath = "project/Dependencies.scala"
+    }
+  }
+  
+  File cromwellJar = release_cromwell.executionDir + "/target/scala-2.12/cromwell-" + cromwellPrep.version + ".jar"
+  # Version that was just released
+  Int cromwellVersionAsInt = cromwellPrep.version
+  # Previous version
+  Int cromwellPreviousVersion = cromwellVersionAsInt - 1
+  
+  call makeGithubRelease { input:
+           githubToken = githubToken,
+           organization = organization,
+           cromwellJar = cromwellJar,
+           newVersion = cromwellVersionAsInt,
+           oldVersion = cromwellPreviousVersion
+  }
+  
+  output {
+    File cromwellReleasedJar = cromwellJar
+  }
+}
diff --git a/scripts/docker-compose-mysql/compose/cromwell/app-config/application.conf b/scripts/docker-compose-mysql/compose/cromwell/app-config/application.conf
index 2869bc5fd..72feb2457 100644
--- a/scripts/docker-compose-mysql/compose/cromwell/app-config/application.conf
+++ b/scripts/docker-compose-mysql/compose/cromwell/app-config/application.conf
@@ -51,5 +51,5 @@ database {
   db.user = "cromwell"
   db.password = "cromwell"
   db.driver = "com.mysql.jdbc.Driver"
-  driver = "slick.driver.MySQLDriver$"
+  profile = "slick.jdbc.MySQLProfile$"
 }
diff --git a/scripts/docker-compose-mysql/jes-cromwell/jes-config/application.conf b/scripts/docker-compose-mysql/jes-cromwell/jes-config/application.conf
index aab9925ff..64b47218f 100644
--- a/scripts/docker-compose-mysql/jes-cromwell/jes-config/application.conf
+++ b/scripts/docker-compose-mysql/jes-cromwell/jes-config/application.conf
@@ -64,5 +64,5 @@ database {
   db.user = "cromwell"
   db.password = "cromwell"
   db.driver = "com.mysql.jdbc.Driver"
-  driver = "slick.driver.MySQLDriver$"
+  profile = "slick.jdbc.MySQLProfile$"
 }
diff --git a/scripts/test_upgrade/custom_labels.json b/scripts/test_upgrade/custom_labels.json
new file mode 100644
index 000000000..d3e90d2bf
--- /dev/null
+++ b/scripts/test_upgrade/custom_labels.json
@@ -0,0 +1,3 @@
+{
+	"test-upgrade-label": "goodvalue"
+}
diff --git a/scripts/test_upgrade/defaultDocker.json b/scripts/test_upgrade/defaultDocker.json
new file mode 100644
index 000000000..f4713ec3e
--- /dev/null
+++ b/scripts/test_upgrade/defaultDocker.json
@@ -0,0 +1,5 @@
+{
+  "default_runtime_attributes": {
+    "docker": "library/ubuntu:latest"
+  }
+}
diff --git a/scripts/test_upgrade/scatter_files.wdl b/scripts/test_upgrade/scatter_files.wdl
new file mode 100644
index 000000000..636faae37
--- /dev/null
+++ b/scripts/test_upgrade/scatter_files.wdl
@@ -0,0 +1,60 @@
+task mkFile {
+
+ Int index
+
+ command {
+   echo "content-${index}"
+ }
+
+ output {
+  File f = stdout()
+ }
+ runtime { docker: "ubuntu:latest" }
+}
+
+task catFile {
+ File f_in
+
+ command {
+   sleep 50
+   cat ${f_in}
+ }
+
+ output {
+  File f = stdout()
+ }
+}
+
+task gather {
+ Array[File] inputs
+
+ command {
+   cat ${sep=" " inputs}
+ }
+
+ output {
+  String result = stdout()
+ }
+ runtime { docker: "ubuntu:latest" }
+}
+
+workflow scatter_files {
+
+ Int scatter_width_part_1
+ Int scatter_width_part_2
+ Int scatter_width_part_3
+ Int scatter_width_part_4
+ Int scatter_width_part_5
+ Int scatter_width_part_6
+
+ Int sum = scatter_width_part_1 + scatter_width_part_2 + scatter_width_part_3 + scatter_width_part_4 + scatter_width_part_5 + scatter_width_part_6
+
+ Array[Int] xs = range(sum / 4)
+
+ scatter (x in xs) {
+  call mkFile { input: index = x }
+  call catFile { input: f_in = mkFile.f}
+ }
+
+ call gather { input: inputs = catFile.f }
+}
diff --git a/scripts/test_upgrade/scatter_files_input_part1_ab.json b/scripts/test_upgrade/scatter_files_input_part1_ab.json
new file mode 100644
index 000000000..a706d3f72
--- /dev/null
+++ b/scripts/test_upgrade/scatter_files_input_part1_ab.json
@@ -0,0 +1,5 @@
+[{
+  "scatter_files.scatter_width_part_1": 2
+},{
+  "scatter_files.scatter_width_part_1": 2
+}]
diff --git a/scripts/test_upgrade/scatter_files_input_part1_c.json b/scripts/test_upgrade/scatter_files_input_part1_c.json
new file mode 100644
index 000000000..1b04da787
--- /dev/null
+++ b/scripts/test_upgrade/scatter_files_input_part1_c.json
@@ -0,0 +1,3 @@
+{
+  "scatter_files.scatter_width_part_1": 2
+}
diff --git a/scripts/test_upgrade/scatter_files_input_part2.json b/scripts/test_upgrade/scatter_files_input_part2.json
new file mode 100644
index 000000000..5208451da
--- /dev/null
+++ b/scripts/test_upgrade/scatter_files_input_part2.json
@@ -0,0 +1,3 @@
+{
+  "scatter_files.scatter_width_part_2": 2
+}
diff --git a/scripts/test_upgrade/scatter_files_input_part3.json b/scripts/test_upgrade/scatter_files_input_part3.json
new file mode 100644
index 000000000..a9611598c
--- /dev/null
+++ b/scripts/test_upgrade/scatter_files_input_part3.json
@@ -0,0 +1,3 @@
+{
+  "scatter_files.scatter_width_part_3": 2
+}
diff --git a/scripts/test_upgrade/scatter_files_input_part4.json b/scripts/test_upgrade/scatter_files_input_part4.json
new file mode 100644
index 000000000..e61982340
--- /dev/null
+++ b/scripts/test_upgrade/scatter_files_input_part4.json
@@ -0,0 +1,3 @@
+{
+  "scatter_files.scatter_width_part_4": 2
+}
diff --git a/scripts/test_upgrade/scatter_files_input_part5.json b/scripts/test_upgrade/scatter_files_input_part5.json
new file mode 100644
index 000000000..4cdabf350
--- /dev/null
+++ b/scripts/test_upgrade/scatter_files_input_part5.json
@@ -0,0 +1,3 @@
+{
+  "scatter_files.scatter_width_part_5": 2
+}
diff --git a/scripts/test_upgrade/scatter_files_input_part6.json b/scripts/test_upgrade/scatter_files_input_part6.json
new file mode 100644
index 000000000..d0d5e4ecd
--- /dev/null
+++ b/scripts/test_upgrade/scatter_files_input_part6.json
@@ -0,0 +1,3 @@
+{
+  "scatter_files.scatter_width_part_6": 2
+}
diff --git a/scripts/test_upgrade/test_upgrade.sh b/scripts/test_upgrade/test_upgrade.sh
new file mode 100755
index 000000000..7eb6a22e6
--- /dev/null
+++ b/scripts/test_upgrade/test_upgrade.sh
@@ -0,0 +1,182 @@
+#!/bin/bash
+#
+# test_upgrade.sh
+#
+# What this script does:
+# - Starts Cromwell using the "previous" cromwell JAR/configuration
+# - Sends 3 jobs to Cromwell (2 batched, one single)
+# - Waits 6 minutes
+# - Sends the same 3 jobs again, in the same way
+# - Shutdown Cromwell with jobs still running
+# - Restarts Cromwell using the "new" cromwell JAR/configuration
+# - Sends the same three jobs again
+# - Waits for everything to complete
+#
+# What it doesn't do (yet... but maybe for the C27 release!):
+# - Guarantee that any jobs are in any specific states at the point that it shuts down.
+# - Check the status of jobs after completing.
+# - Check that operations IDs weren't duplicated after restarting.
+# - DRY it up with functions replacing the outrageous copy/pasting.
+#
+
+PREVIOUS_CROMWELL_JAR=cromwell-25-31ae549-SNAP.jar
+PREVIOUS_CROMWELL_CONF=jes.conf
+
+NEW_CROMWELL_JAR=cromwell-26-88630db-SNAP.jar
+NEW_CROMWELL_CONF=jes.conf
+
+OUT_DIR=out
+
+PREVIOUS_CROMWELL_LOG="$OUT_DIR/previous_version.log"
+NEW_CROMWELL_LOG="$OUT_DIR/new_version.log"
+
+START_DT=$(date "+%Y-%m-%dT%H:%M:%S.000-04:00")
+echo "Starting $0 at: $START_DT"
+# Now we've printed it, make it URL-safe
+START_DT="${START_DT//:/%3A}"
+
+pgrep -q cromwell
+ALREADY_RUNNING=$?
+if [ $ALREADY_RUNNING -ne 1 ];
+then
+	echo "Oops! Cromwell (🐖 ) is already running!"
+	pgrep cromwell
+	exit 1
+fi
+
+rm -r $OUT_DIR
+mkdir $OUT_DIR
+
+echo -n "Starting Cromwell (🐖 )..."
+java -Dconfig.file="$PREVIOUS_CROMWELL_CONF" -jar "$PREVIOUS_CROMWELL_JAR" server &> "$PREVIOUS_CROMWELL_LOG" &
+CROMWELL_PID=$!
+echo "started (PID: $CROMWELL_PID)."
+
+echo -n "Waiting for (🐖 ) API..."
+READY=-1
+while [ $READY -ne 0 ];
+do
+	sleep 1
+	curl -X GET --header "Accept: application/json" "http://localhost:8000/api/workflows/v1/backends" &>/dev/null
+	READY=$?
+done
+echo "ready."
+
+for i in 1 2
+do
+	echo -n "submitting job ${i}a and ${i}b..."
+	RESULT_FILE="$OUT_DIR/submitResult_${i}_ab.json"
+	curl -X POST --header "Accept: application/json" "http://localhost:8000/api/workflows/v1/batch" \
+	  -F workflowSource=@scatter_files.wdl \
+	  -F workflowInputs=@scatter_files_input_part1_ab.json \
+	  -F workflowInputs_2=@scatter_files_input_part2.json \
+	  -F workflowInputs_3=@scatter_files_input_part3.json \
+	  -F workflowInputs_4=@scatter_files_input_part4.json \
+	  -F workflowInputs_5=@scatter_files_input_part5.json \
+	  -F workflowInputs_6=@scatter_files_input_part6.json \
+	  -F workflowOptions=@defaultDocker.json \
+	  -F customLabels=@custom_labels.json &> $"RESULT_FILE"
+    echo "done (Response in: $RESULT_FILE)."
+	echo -n "submitting job ${i}c..."
+	RESULT_FILE="$OUT_DIR/submitResult_${i}_c.json"
+	curl -X POST --header "Accept: application/json" "http://localhost:8000/api/workflows/v1/batch" \
+	  -F workflowSource=@scatter_files.wdl \
+	  -F workflowInputs=@scatter_files_input_part1_c.json \
+	  -F workflowInputs_2=@scatter_files_input_part2.json \
+	  -F workflowInputs_3=@scatter_files_input_part3.json \
+	  -F workflowInputs_4=@scatter_files_input_part4.json \
+	  -F workflowInputs_5=@scatter_files_input_part5.json \
+	  -F workflowInputs_6=@scatter_files_input_part6.json \
+	  -F workflowOptions=@defaultDocker.json \
+	  -F customLabels=@custom_labels.json &> $"RESULT_FILE"
+    echo "done (Response in: $RESULT_FILE)."
+	[ "$i" -eq 1 ] && sleep 360
+done
+
+# Step two: upgrade cromwell
+kill $CROMWELL_PID
+
+pgrep -q cromwell
+STILL_RUNNING=$?
+while [ $STILL_RUNNING -eq 0 ];
+do
+	echo "Waiting for Cromwell(🐖 ) to exit..."
+	sleep 1
+	pgrep -q cromwell
+	STILL_RUNNING=$?
+done
+
+echo -n "Starting Cromwell(🐖 )... "
+java -Dconfig.file=$NEW_CROMWELL_CONF -jar $NEW_CROMWELL_JAR server &> $NEW_CROMWELL_LOG &
+CROMWELL_PID=$!
+echo "started (PID=$CROMWELL_PID)."
+
+echo -n "Waiting for 🐖  API..."
+READY=-1
+while [ "$READY" -ne "0" ];
+do
+	sleep 1
+	curl -X GET --header "Accept: application/json" "http://localhost:8000/api/workflows/v1/backends" &>/dev/null
+	READY=$?
+done
+echo "ready."
+
+i=3
+echo -n "submitting job ${i}a and ${i}b..."
+RESULT_FILE="$OUT_DIR/submitResult_${i}_ab.json"
+curl -X POST --header "Accept: application/json" "http://localhost:8000/api/workflows/v1/batch" \
+  -F workflowSource=@scatter_files.wdl \
+  -F workflowInputs=@scatter_files_input_part1_ab.json \
+  -F workflowInputs_2=@scatter_files_input_part2.json \
+  -F workflowInputs_3=@scatter_files_input_part3.json \
+  -F workflowInputs_4=@scatter_files_input_part4.json \
+  -F workflowInputs_5=@scatter_files_input_part5.json \
+  -F workflowInputs_6=@scatter_files_input_part6.json \
+  -F workflowOptions=@defaultDocker.json \
+  -F customLabels=@custom_labels.json &> $"RESULT_FILE"
+echo "done (Response in: $RESULT_FILE)."
+echo -n "submitting job ${i}c..."
+RESULT_FILE="$OUT_DIR/submitResult_${i}_c.json"
+curl -X POST --header "Accept: application/json" "http://localhost:8000/api/workflows/v1/batch" \
+  -F workflowSource=@scatter_files.wdl \
+  -F workflowInputs=@scatter_files_input_part1_c.json \
+  -F workflowInputs_2=@scatter_files_input_part2.json \
+  -F workflowInputs_3=@scatter_files_input_part3.json \
+  -F workflowInputs_4=@scatter_files_input_part4.json \
+  -F workflowInputs_5=@scatter_files_input_part5.json \
+  -F workflowInputs_6=@scatter_files_input_part6.json \
+  -F workflowOptions=@defaultDocker.json \
+  -F customLabels=@custom_labels.json &> $"RESULT_FILE"
+echo "done (Response in: $RESULT_FILE)."
+
+# Step 3: Wait until everything's done:
+echo -n "Waiting for the run to complete at a rate of one 🐷  per minute..."
+DONE=1
+while [ "$DONE" -ne "0" ];
+do
+	CURLED=$(curl -X GET --header "Accept: application/json" "http://localhost:8000/api/workflows/v1/query?start=${START_DT}&status=Running" 2>/dev/null)
+	grep -q ": \[\]" <<< "$CURLED"
+	FINISHED=$?
+	grep -q '"status": "fail"' <<< "$CURLED"
+	ERROR=$?
+	[ $ERROR -eq 0 ] && ( echo "Error: $CURLED" )
+	if [ $ERROR -eq 0 ] || [ $FINISHED -eq 0 ];
+	then
+		DONE=0
+	else
+		echo -n ".🐷 ."
+		sleep 60
+	fi
+done
+echo "...done"
+
+kill $CROMWELL_PID
+
+# Step 4: analyse logs to make sure things worked out:
+echo "Previous version's operations IDs:"
+grep operations "$PREVIOUS_CROMWELL_LOG" | sed "s/.*\ - \(.*\)/\1/g"
+
+echo
+echo "New version's operations IDs:"
+grep operations "$NEW_CROMWELL_LOG" | sed "s/.*\ - \(.*\)/\1/g"
+# TODO...
diff --git a/services/src/main/scala/cromwell/services/ServiceRegistryActor.scala b/services/src/main/scala/cromwell/services/ServiceRegistryActor.scala
index c4ff1e476..66ec1dbe4 100644
--- a/services/src/main/scala/cromwell/services/ServiceRegistryActor.scala
+++ b/services/src/main/scala/cromwell/services/ServiceRegistryActor.scala
@@ -2,9 +2,13 @@ package cromwell.services
 
 import akka.actor.SupervisorStrategy.Escalate
 import akka.actor.{Actor, ActorInitializationException, ActorLogging, ActorRef, OneForOneStrategy, Props}
-import cromwell.core.Dispatcher.ServiceDispatcher
+import cats.data.NonEmptyList
 import com.typesafe.config.{Config, ConfigFactory, ConfigObject}
+import cromwell.core.Dispatcher.ServiceDispatcher
+import cromwell.util.GracefulShutdownHelper
+import cromwell.util.GracefulShutdownHelper.ShutdownCommand
 import net.ceedubs.ficus.Ficus._
+
 import scala.collection.JavaConverters._
 
 object ServiceRegistryActor {
@@ -32,11 +36,14 @@ object ServiceRegistryActor {
 
   private def serviceProps(serviceName: String, globalConfig: Config, serviceStanza: Config): Props = {
     val serviceConfigStanza = serviceStanza.as[Option[Config]]("config").getOrElse(ConfigFactory.parseString(""))
+
+    val dispatcher = serviceStanza.as[Option[String]]("dispatcher").getOrElse(ServiceDispatcher)
     val className = serviceStanza.as[Option[String]]("class").getOrElse(
       throw new IllegalArgumentException(s"Invalid configuration for service $serviceName: missing 'class' definition")
     )
+
     try {
-      Props.create(Class.forName(className), serviceConfigStanza, globalConfig)
+      Props.create(Class.forName(className), serviceConfigStanza, globalConfig).withDispatcher(dispatcher)
     } catch {
       case e: ClassNotFoundException => throw new RuntimeException(
         s"Class $className for service $serviceName cannot be found in the class path.", e
@@ -45,7 +52,7 @@ object ServiceRegistryActor {
   }
 }
 
-class ServiceRegistryActor(serviceProps: Map[String, Props]) extends Actor with ActorLogging {
+class ServiceRegistryActor(serviceProps: Map[String, Props]) extends Actor with ActorLogging with GracefulShutdownHelper {
   import ServiceRegistryActor._
 
   val services: Map[String, ActorRef] = serviceProps map {
@@ -60,6 +67,11 @@ class ServiceRegistryActor(serviceProps: Map[String, Props]) extends Actor with
           log.error("Received ServiceRegistryMessage requesting service '{}' for which no service is configured.  Message: {}", msg.serviceName, msg)
           sender ! ServiceRegistryFailure(msg.serviceName)
       }
+    case ShutdownCommand =>
+      services.values.toList match {
+        case Nil => context stop self
+        case head :: tail => waitForActorsAndShutdown(NonEmptyList.of(head, tail: _*))
+      }
     case fool =>
       log.error("Received message which is not a ServiceRegistryMessage: {}", fool)
       sender ! ServiceRegistryFailure("Message is not a ServiceRegistryMessage: " + fool)
@@ -70,7 +82,7 @@ class ServiceRegistryActor(serviceProps: Map[String, Props]) extends Actor with
     * the error up the chain
     */
   override val supervisorStrategy = OneForOneStrategy() {
-    case aie: ActorInitializationException => Escalate
+    case _: ActorInitializationException => Escalate
     case t => super.supervisorStrategy.decider.applyOrElse(t, (_: Any) => Escalate)
   }
 }
diff --git a/services/src/main/scala/cromwell/services/ServicesStore.scala b/services/src/main/scala/cromwell/services/ServicesStore.scala
index c00aad09c..e8a8b940d 100644
--- a/services/src/main/scala/cromwell/services/ServicesStore.scala
+++ b/services/src/main/scala/cromwell/services/ServicesStore.scala
@@ -5,7 +5,6 @@ import cromwell.database.migration.liquibase.LiquibaseUtils
 import cromwell.database.slick.SlickDatabase
 import cromwell.database.sql.SqlDatabase
 import net.ceedubs.ficus.Ficus._
-import org.slf4j.LoggerFactory
 
 trait ServicesStore {
   def databaseInterface: SqlDatabase
@@ -30,20 +29,45 @@ trait SingletonServicesStore extends ServicesStore {
 
 object SingletonServicesStore {
 
-  private lazy val log = LoggerFactory.getLogger("SingletonServicesStore")
-  private val databaseConfig = {
-    val config = ConfigFactory.load.getConfig("database")
-    if (config.hasPath("config")) {
-      log.warn(
-        """
-          |Use of configuration path 'database.config' is deprecated.
-          |
-          |Move the configuration directly under the 'database' element, and remove the key 'database.config'.
-          |""".stripMargin)
-      config.getConfig(config.getString("config"))
-    } else {
-      config
-    }
+  private val databaseConfig = ConfigFactory.load.getConfig("database")
+
+  if (databaseConfig.hasPath("config")) {
+      val msg = """
+        |*******************************
+        |***** DEPRECATION MESSAGE *****
+        |*******************************
+        |
+        |Use of configuration path 'database.config' has been deprecated.
+        |
+        |Move the configuration directly under the 'database' element, and remove the key 'database.config'.
+        |
+        |""".stripMargin
+    throw new Exception(msg)
+  } else if (databaseConfig.hasPath("driver")) {
+    val msg =
+      """
+      |*******************************
+      |***** DEPRECATION MESSAGE *****
+      |*******************************
+      |
+      |Use of configuration path 'database.driver' has been deprecated. Replace with a "profile" element instead, e.g:
+      |
+      |database {
+      |  #driver = "slick.driver.MySQLDriver$" #old
+      |  profile = "slick.jdbc.MySQLProfile$"  #new
+      |  db {
+      |    driver = "com.mysql.jdbc.Driver"
+      |    url = "jdbc:mysql://host/cromwell?rewriteBatchedStatements=true"
+      |    user = "user"
+      |    password = "pass"
+      |    connectionTimeout = 5000
+      |  }
+      |}
+      |
+      |Cromwell thanks you.
+      |""".stripMargin
+    throw
+      new Exception(msg)
   }
 
   import ServicesStore.EnhancedSqlDatabase
diff --git a/services/src/main/scala/cromwell/services/keyvalue/KeyValueServiceActor.scala b/services/src/main/scala/cromwell/services/keyvalue/KeyValueServiceActor.scala
index 1d25ee88f..49e220936 100644
--- a/services/src/main/scala/cromwell/services/keyvalue/KeyValueServiceActor.scala
+++ b/services/src/main/scala/cromwell/services/keyvalue/KeyValueServiceActor.scala
@@ -1,45 +1,57 @@
 package cromwell.services.keyvalue
 
 import akka.actor.{Actor, ActorRef}
-import com.typesafe.config.Config
-import cromwell.core.WorkflowId
+import cromwell.core.{JobKey, MonitoringCompanionHelper, WorkflowId}
 import cromwell.services.ServiceRegistryActor.ServiceRegistryMessage
 import cromwell.services.keyvalue.KeyValueServiceActor._
 
 import scala.concurrent.{ExecutionContextExecutor, Future}
 import scala.util.{Failure, Success}
 
-
 object KeyValueServiceActor {
-  sealed  trait KvMessage
+  final case class KvJobKey(callFqn: String, callIndex: Option[Int], callAttempt: Int)
+  object KvJobKey {
+    def apply(jobKey: JobKey): KvJobKey = KvJobKey(jobKey.scope.fullyQualifiedName, jobKey.index, jobKey.attempt)
+  }
+
+  final case class ScopedKey(workflowId: WorkflowId, jobKey: KvJobKey, key: String)
 
-  sealed trait KvAction extends KvMessage with ServiceRegistryMessage {
-    def serviceName = "KeyValue"
+  sealed trait KvMessage {
+    def key: ScopedKey
   }
-  case class KvJobKey(callFqn: String, callIndex: Option[Int], callAttempt: Int)
-  case class ScopedKey(workflowId: WorkflowId, jobKey: KvJobKey, key: String)
-  case class KvPut(pair: KvPair) extends KvAction
-  case class KvGet(key: ScopedKey) extends KvAction
+  sealed trait KvMessageWithAction extends KvMessage {
+    val action: KvAction
+    def key = action.key
+  }
+
+  sealed trait KvAction extends KvMessage with ServiceRegistryMessage { override val serviceName = "KeyValue" }
+
+  final case class KvPut(pair: KvPair) extends KvAction { override def key = pair.key }
+  final case class KvGet(key: ScopedKey) extends KvAction
 
   sealed trait KvResponse extends KvMessage
-  case class KvPair(key: ScopedKey, value: Option[String]) extends KvResponse
-  case class KvFailure(action: KvAction, failure: Throwable) extends KvResponse
-  case class KvKeyLookupFailed(action: KvGet) extends KvResponse
-  case class KvPutSuccess(action: KvPut) extends KvResponse
+
+  final case class KvPair(key: ScopedKey, value: Option[String]) extends KvResponse
+  final case class KvFailure(action: KvAction, failure: Throwable) extends KvResponse with KvMessageWithAction
+  final case class KvKeyLookupFailed(action: KvGet) extends KvResponse with KvMessageWithAction
+  final case class KvPutSuccess(action: KvPut) extends KvResponse with KvMessageWithAction
 }
 
-trait KeyValueServiceActor extends Actor {
+trait KeyValueServiceActor extends Actor with MonitoringCompanionHelper {
   implicit val ec: ExecutionContextExecutor
-  val serviceConfig: Config
-  val globalConfig: Config
 
-  def receive = {
+  val kvReceive: Receive = {
     case action: KvGet => respond(sender(), action, doGet(action))
-    case action: KvPut => respond(sender(), action, doPut(action))
+    case action: KvPut =>
+      addWork()
+      val putAction = doPut(action)
+      putAction andThen { case _ => removeWork() }
+      respond(sender(), action, putAction)
   }
 
-  def doPut(put: KvPut): Future[KvResponse]
+  override def receive = kvReceive.orElse(monitoringReceive)
 
+  def doPut(put: KvPut): Future[KvResponse]
   def doGet(get: KvGet): Future[KvResponse]
 
   private def respond(replyTo: ActorRef, action: KvAction, response: Future[KvResponse]): Unit = {
diff --git a/services/src/main/scala/cromwell/services/keyvalue/KvClient.scala b/services/src/main/scala/cromwell/services/keyvalue/KvClient.scala
new file mode 100644
index 000000000..c8ddbcb88
--- /dev/null
+++ b/services/src/main/scala/cromwell/services/keyvalue/KvClient.scala
@@ -0,0 +1,41 @@
+package cromwell.services.keyvalue
+
+import akka.actor.{Actor, ActorLogging, ActorRef}
+import cromwell.services.keyvalue.KeyValueServiceActor._
+
+import scala.concurrent.{ExecutionContext, Future, Promise}
+
+trait KvClient { this: Actor with ActorLogging =>
+
+  def serviceRegistryActor: ActorRef
+  private[keyvalue] var currentKvClientRequests: Map[ScopedKey, Promise[KvResponse]] = Map.empty
+
+  final def makeKvRequest(actions: Seq[KvAction])(implicit ec: ExecutionContext): Future[Seq[KvResponse]] = {
+    if (actions.exists(action => currentKvClientRequests.contains(action.key))) {
+      val msg = "Programmer Error! KvClient does not support multiple KvActions active for the same ScopedKey concurrently. Mi Scusi!"
+      log.error(msg)
+      Future.failed(new RuntimeException(msg))
+    } else {
+      createResponseSet(actions)
+    }
+  }
+
+  final def kvClientReceive: Actor.Receive = {
+    case response: KvResponse => fulfillOrLog(response)
+  }
+
+  private def createResponseSet(newActions: Seq[KvAction])(implicit ec: ExecutionContext) = {
+    val actionsAndPromises = newActions.map(a => a.key -> Promise[KvResponse]())
+    currentKvClientRequests ++= actionsAndPromises.toMap
+    newActions foreach { serviceRegistryActor ! _ }
+    Future.sequence(actionsAndPromises.map(_._2.future))
+  }
+
+  private def fulfillOrLog(response: KvResponse) = currentKvClientRequests.get(response.key) match {
+    case Some(fulfilledPromise) =>
+      fulfilledPromise.success(response)
+      currentKvClientRequests -= response.key
+    case None => log.error(s"Programmer Error: Got a KV response for a request that was never sent: $response. Did you use the KV store without KvClient? Current key set: ${currentKvClientRequests.keys.mkString("")}")
+  }
+}
+
diff --git a/services/src/main/scala/cromwell/services/keyvalue/impl/SqlKeyValueServiceActor.scala b/services/src/main/scala/cromwell/services/keyvalue/impl/SqlKeyValueServiceActor.scala
index 57452bf21..3ea62743d 100644
--- a/services/src/main/scala/cromwell/services/keyvalue/impl/SqlKeyValueServiceActor.scala
+++ b/services/src/main/scala/cromwell/services/keyvalue/impl/SqlKeyValueServiceActor.scala
@@ -13,14 +13,14 @@ object SqlKeyValueServiceActor {
   def props(serviceConfig: Config, globalConfig: Config) = Props(SqlKeyValueServiceActor(serviceConfig, globalConfig)).withDispatcher(ServiceDispatcher)
 }
 
-case class SqlKeyValueServiceActor(override val serviceConfig: Config, override val globalConfig: Config)
+final case class SqlKeyValueServiceActor(serviceConfig: Config, globalConfig: Config)
   extends KeyValueServiceActor with BackendKeyValueDatabaseAccess with SingletonServicesStore {
   override implicit val ec = context.dispatcher
   private implicit val system = context.system
 
   override def doPut(put: KvPut): Future[KvResponse] = {
     put.pair.value match {
-      case Some(backendVal) => updateBackendKeyValuePair(put.pair.key.workflowId,
+      case Some(_) => updateBackendKeyValuePair(put.pair.key.workflowId,
         put.pair.key.jobKey,
         put.pair.key.key,
         put.pair.value.get).map(_ => KvPutSuccess(put))
diff --git a/services/src/main/scala/cromwell/services/metadata/CallMetadataKeys.scala b/services/src/main/scala/cromwell/services/metadata/CallMetadataKeys.scala
index ef0dc1091..5d1db5e4f 100644
--- a/services/src/main/scala/cromwell/services/metadata/CallMetadataKeys.scala
+++ b/services/src/main/scala/cromwell/services/metadata/CallMetadataKeys.scala
@@ -21,4 +21,16 @@ object CallMetadataKeys {
   val SubWorkflowId = "subWorkflowId"
   val SubWorkflowMetadata = "subWorkflowMetadata"
   val CallCaching = "callCaching"
+  val BackendLabels = "backendLabels"
+  val Labels = "labels"
+
+  object CallCachingKeys {
+    val EffectiveModeKey = CallCaching + MetadataKey.KeySeparator + "effectiveCallCachingMode"
+    val ReadResultMetadataKey = CallCaching + MetadataKey.KeySeparator + "result"
+    val HitResultMetadataKey = CallCaching + MetadataKey.KeySeparator + "hit"
+    val AllowReuseMetadataKey = CallCaching + MetadataKey.KeySeparator + "allowResultReuse"
+    val HitFailuresKey = CallCaching + MetadataKey.KeySeparator + "hitFailures"
+    val HashFailuresKey = CallCaching + MetadataKey.KeySeparator + "hashFailures"
+    val HashesKey = CallCaching + MetadataKey.KeySeparator + "hashes"
+  }
 }
diff --git a/services/src/main/scala/cromwell/services/metadata/MetadataQuery.scala b/services/src/main/scala/cromwell/services/metadata/MetadataQuery.scala
index a176bc1c5..818a4def6 100644
--- a/services/src/main/scala/cromwell/services/metadata/MetadataQuery.scala
+++ b/services/src/main/scala/cromwell/services/metadata/MetadataQuery.scala
@@ -5,14 +5,22 @@ import java.time.OffsetDateTime
 import cats.data.NonEmptyList
 import cromwell.core.WorkflowId
 import org.slf4j.{Logger, LoggerFactory}
-import wdl4s.values.{WdlBoolean, WdlFloat, WdlInteger, WdlOptionalValue, WdlValue}
+import wdl4s.wdl.values._
 
 case class MetadataJobKey(callFqn: String, index: Option[Int], attempt: Int)
 
-case class MetadataKey(workflowId: WorkflowId, jobKey: Option[MetadataJobKey], key: String)
+case class MetadataKey private (workflowId: WorkflowId, jobKey: Option[MetadataJobKey], key: String)
 
 object MetadataKey {
 
+  implicit class KeyMetacharacterEscaper(val key: String) extends AnyVal {
+    // The escapes are necessary on the first arguments to `replaceAll` since they're treated like regular expressions
+    // and square braces are character class delimiters.  Backslashes must be escaped in both parameters.
+    // Ignore the red in some of the "raw" strings, IntelliJ and GitHub don't seem to understand them.
+    def escapeMeta = key.replaceAll(raw"\[", raw"\\[").replaceAll(raw"\]", raw"\\]").replaceAll(":", raw"\\:")
+    def unescapeMeta = key.replaceAll(raw"\\\[", "[").replaceAll(raw"\\\]", "]").replaceAll(raw"\\:", ":")
+  }
+
   val KeySeparator = ':'
 
   def apply(workflowId: WorkflowId, jobKey: Option[MetadataJobKey], keys: String*): MetadataKey = {
@@ -68,10 +76,10 @@ case class MetadataValue(value: String, valueType: MetadataType)
 
 case class MetadataEvent(key: MetadataKey, value: Option[MetadataValue], offsetDateTime: OffsetDateTime)
 
-case class MetadataQueryJobKey(callFqn: String, index: Option[Int], attempt: Int)
+case class MetadataQueryJobKey(callFqn: String, index: Option[Int], attempt: Option[Int])
 
 object MetadataQueryJobKey {
-  def forMetadataJobKey(jobKey: MetadataJobKey) = MetadataQueryJobKey(jobKey.callFqn, jobKey.index, jobKey.attempt)
+  def forMetadataJobKey(jobKey: MetadataJobKey) = MetadataQueryJobKey(jobKey.callFqn, jobKey.index, Option(jobKey.attempt))
 }
 
 case class MetadataQuery(workflowId: WorkflowId, jobKey: Option[MetadataQueryJobKey], key: Option[String],
diff --git a/services/src/main/scala/cromwell/services/metadata/MetadataService.scala b/services/src/main/scala/cromwell/services/metadata/MetadataService.scala
index 3eb13fbe3..daf567c7e 100644
--- a/services/src/main/scala/cromwell/services/metadata/MetadataService.scala
+++ b/services/src/main/scala/cromwell/services/metadata/MetadataService.scala
@@ -6,19 +6,21 @@ import akka.actor.ActorRef
 import cats.data.NonEmptyList
 import cromwell.core.{FullyQualifiedName, JobKey, WorkflowId, WorkflowState}
 import cromwell.services.ServiceRegistryActor.ServiceRegistryMessage
-import lenthall.exception.ThrowableAggregation
-import wdl4s.values._
+import lenthall.exception.{MessageAggregation, ThrowableAggregation}
+import wdl4s.wdl.values._
+
+import scala.util.Random
 
 
 object MetadataService {
 
   final val MetadataServiceName = "MetadataService"
 
-  case class WorkflowQueryResult(id: String, name: Option[String], status: Option[String], start: Option[OffsetDateTime], end: Option[OffsetDateTime])
+  final case class WorkflowQueryResult(id: String, name: Option[String], status: Option[String], start: Option[OffsetDateTime], end: Option[OffsetDateTime])
 
-  case class WorkflowQueryResponse(results: Seq[WorkflowQueryResult])
+  final case class WorkflowQueryResponse(results: Seq[WorkflowQueryResult])
 
-  case class QueryMetadata(page: Option[Int], pageSize: Option[Int], totalRecords: Option[Int])
+  final case class QueryMetadata(page: Option[Int], pageSize: Option[Int], totalRecords: Option[Int])
 
   trait MetadataServiceMessage
   /**
@@ -59,14 +61,16 @@ object MetadataService {
     }
   }
 
-  case class PutMetadataAction(events: Iterable[MetadataEvent]) extends MetadataServiceAction
+  final case class PutMetadataAction(events: Iterable[MetadataEvent]) extends MetadataServiceAction
+  final case class PutMetadataActionAndRespond(events: Iterable[MetadataEvent], replyTo: ActorRef) extends MetadataServiceAction
+
   case class GetSingleWorkflowMetadataAction(workflowId: WorkflowId, includeKeysOption: Option[NonEmptyList[String]],
                                              excludeKeysOption: Option[NonEmptyList[String]],
                                              expandSubWorkflows: Boolean)
     extends ReadAction
   case class GetMetadataQueryAction(key: MetadataQuery) extends ReadAction
   case class GetStatus(workflowId: WorkflowId) extends ReadAction
-  case class WorkflowQuery[A](uri: A, parameters: Seq[(String, String)]) extends ReadAction
+  case class WorkflowQuery(parameters: Seq[(String, String)]) extends ReadAction
   case class WorkflowOutputs(workflowId: WorkflowId) extends ReadAction
   case class GetLogs(workflowId: WorkflowId) extends ReadAction
   case object RefreshSummary extends MetadataServiceAction
@@ -76,8 +80,8 @@ object MetadataService {
     def onUnrecognized(possibleWorkflowId: String): Unit
     def onFailure(possibleWorkflowId: String, throwable: Throwable): Unit
   }
-  final case class ValidateWorkflowIdAndExecute(possibleWorkflowId: String,
-                                                validationCallback: ValidationCallback) extends MetadataServiceAction
+
+  final case class ValidateWorkflowId(possibleWorkflowId: WorkflowId) extends MetadataServiceAction
 
   /**
     * Responses
@@ -87,15 +91,11 @@ object MetadataService {
     def reason: Throwable
   }
 
-  case class MetadataLookupResponse(query: MetadataQuery, eventList: Seq[MetadataEvent]) extends MetadataServiceResponse
-  case class MetadataServiceKeyLookupFailed(query: MetadataQuery, reason: Throwable) extends MetadataServiceFailure
+  final case class MetadataLookupResponse(query: MetadataQuery, eventList: Seq[MetadataEvent]) extends MetadataServiceResponse
+  final case class MetadataServiceKeyLookupFailed(query: MetadataQuery, reason: Throwable) extends MetadataServiceFailure
 
-  case class StatusLookupResponse(workflowId: WorkflowId, status: WorkflowState) extends MetadataServiceResponse
-  case class StatusLookupFailed(workflowId: WorkflowId, reason: Throwable) extends MetadataServiceFailure
-
-  final case class WorkflowQuerySuccess[A](uri: A, response: WorkflowQueryResponse, meta: Option[QueryMetadata])
-    extends MetadataServiceResponse
-  final case class WorkflowQueryFailure(reason: Throwable) extends MetadataServiceFailure
+  final case class StatusLookupResponse(workflowId: WorkflowId, status: WorkflowState) extends MetadataServiceResponse
+  final case class StatusLookupFailed(workflowId: WorkflowId, reason: Throwable) extends MetadataServiceFailure
 
   final case class WorkflowOutputsResponse(id: WorkflowId, outputs: Seq[MetadataEvent]) extends MetadataServiceResponse
   final case class WorkflowOutputsFailure(id: WorkflowId, reason: Throwable) extends MetadataServiceFailure
@@ -103,6 +103,18 @@ object MetadataService {
   final case class LogsResponse(id: WorkflowId, logs: Seq[MetadataEvent]) extends MetadataServiceResponse
   final case class LogsFailure(id: WorkflowId, reason: Throwable) extends MetadataServiceFailure
 
+  final case class MetadataWriteSuccess(events: Iterable[MetadataEvent]) extends MetadataServiceResponse
+  final case class MetadataWriteFailure(reason: Throwable, events: Iterable[MetadataEvent]) extends MetadataServiceFailure
+
+  sealed abstract class WorkflowValidationResponse extends MetadataServiceResponse
+  case object RecognizedWorkflowId extends WorkflowValidationResponse
+  case object UnrecognizedWorkflowId extends WorkflowValidationResponse
+  final case class FailedToCheckWorkflowId(cause: Throwable) extends WorkflowValidationResponse
+
+  sealed abstract class MetadataQueryResponse extends MetadataServiceResponse
+  final case class WorkflowQuerySuccess(response: WorkflowQueryResponse, meta: Option[QueryMetadata]) extends MetadataQueryResponse
+  final case class WorkflowQueryFailure(reason: Throwable) extends MetadataQueryResponse
+
   def wdlValueToMetadataEvents(metadataKey: MetadataKey, wdlValue: WdlValue): Iterable[MetadataEvent] = wdlValue match {
     case WdlArray(_, valueSeq) =>
       if (valueSeq.isEmpty) {
@@ -126,13 +138,41 @@ object MetadataService {
       List(MetadataEvent(metadataKey, MetadataValue(value)))
   }
 
-  def throwableToMetadataEvents(metadataKey: MetadataKey, t: Throwable): List[MetadataEvent] = {
+  def throwableToMetadataEvents(metadataKey: MetadataKey, t: Throwable, failureIndex: Int = Random.nextInt(Int.MaxValue)): List[MetadataEvent] = {
+    val emptyCauseList = List(MetadataEvent.empty(metadataKey.copy(key = metadataKey.key + s"[$failureIndex]:causedBy[]")))
+    val metadataKeyAndFailureIndex = s"${metadataKey.key}[$failureIndex]"
+
     t match {
       case aggregation: ThrowableAggregation =>
-        aggregation.errorMessages.toList map { message => MetadataEvent(metadataKey.copy(key = s"${metadataKey.key}:message"), MetadataValue(s"${aggregation.exceptionContext}: $message")) }
-      case other =>
-        val message = List(MetadataEvent(metadataKey.copy(key = s"${metadataKey.key}:message"), MetadataValue(t.getMessage)))
-        val cause = Option(t.getCause) map { cause => throwableToMetadataEvents(metadataKey.copy(key = s"${metadataKey.key}:causedBy"), cause) } getOrElse List.empty
+        val message = List(MetadataEvent(metadataKey.copy(key = s"$metadataKeyAndFailureIndex:message"), MetadataValue(aggregation.exceptionContext)))
+        val indexedCauses = aggregation.throwables.toList.zipWithIndex
+        val indexedCauseEvents = if (indexedCauses.nonEmpty) {
+          indexedCauses flatMap { case (cause, index) =>
+            val causeKey = metadataKey.copy(key = s"$metadataKeyAndFailureIndex:causedBy")
+            throwableToMetadataEvents(causeKey, cause, index)
+          }
+        } else {
+          emptyCauseList
+        }
+        message ++ indexedCauseEvents
+      case aggregation: MessageAggregation =>
+        val message = List(MetadataEvent(metadataKey.copy(key = s"$metadataKeyAndFailureIndex:message"), MetadataValue(aggregation.exceptionContext)))
+        val indexedCauses = aggregation.errorMessages.toList.zipWithIndex
+        val indexedCauseEvents = if (indexedCauses.nonEmpty) {
+          indexedCauses flatMap { case (cause, index) =>
+            val causeMessageKey = metadataKey.copy(key = s"$metadataKeyAndFailureIndex:causedBy[$index]:message")
+            val causeCausedByKey = metadataKey.copy(key = s"$metadataKeyAndFailureIndex:causedBy[$index]:causedBy[]")
+            List(MetadataEvent(causeMessageKey, MetadataValue(cause)), MetadataEvent.empty(causeCausedByKey))
+          }
+        } else {
+          emptyCauseList
+        }
+        message ++ indexedCauseEvents
+
+      case other @ _ =>
+        val message = List(MetadataEvent(metadataKey.copy(key = s"$metadataKeyAndFailureIndex:message"), MetadataValue(t.getMessage)))
+        val causeKey = metadataKey.copy(key = s"$metadataKeyAndFailureIndex:causedBy")
+        val cause = Option(t.getCause) map { cause => throwableToMetadataEvents(causeKey, cause, 0) } getOrElse emptyCauseList
         message ++ cause
     }
   }
diff --git a/services/src/main/scala/cromwell/services/metadata/WorkflowQueryKey.scala b/services/src/main/scala/cromwell/services/metadata/WorkflowQueryKey.scala
index e5e134ca7..1e3847712 100644
--- a/services/src/main/scala/cromwell/services/metadata/WorkflowQueryKey.scala
+++ b/services/src/main/scala/cromwell/services/metadata/WorkflowQueryKey.scala
@@ -2,16 +2,19 @@ package cromwell.services.metadata
 
 import java.time.OffsetDateTime
 
-import cats.instances.list._
+import cats.data
 import cats.syntax.traverse._
 import cats.syntax.validated._
+import cromwell.core.labels.Label
 import cromwell.core.{WorkflowId, WorkflowState}
 import lenthall.validation.ErrorOr._
+import cats.data.Validated._
+import cats.instances.list._
 
 import scala.util.{Success, Try}
 
 object WorkflowQueryKey {
-  val ValidKeys = Set(StartDate, EndDate, Name, Id, Status, Page, PageSize) map { _.name }
+  val ValidKeys = Set(StartDate, EndDate, Name, Id, Status, LabelKeyValue, Page, PageSize) map { _.name }
 
   case object StartDate extends DateTimeWorkflowQueryKey {
     override val name = "Start"
@@ -33,21 +36,37 @@ object WorkflowQueryKey {
     override def displayName = "page size"
   }
 
-  case object Name extends SeqStringWorkflowQueryKey {
+  case object Name extends SeqWorkflowQueryKey[String] {
     override val name = "Name"
 
     override def validate(grouped: Map[String, Seq[(String, String)]]): ErrorOr[List[String]] = {
-
       val values = valuesFromMap(grouped).toList
-      val nels = values map {
+      val nels:List[data.ValidatedNel[String,String]] = values map {
         case Patterns.WorkflowName(n) => n.validNel[String]
         case v => v.invalidNel[String]
       }
-      sequenceListOfValidatedNels(s"Name values do not match allowed workflow naming pattern", nels)
+      sequenceListOfValidatedNels("Name values do not match allowed workflow naming pattern", nels)
+    }
+  }
+
+  case object LabelKeyValue extends SeqWorkflowQueryKey[Label] {
+    override val name = "Label"
+
+    override def validate(grouped: Map[String, Seq[(String, String)]]): ErrorOr[List[Label]] = {
+      val values = valuesFromMap(grouped).toList
+
+      def validateLabelRegex(labelKeyValue: String): ErrorOr[Label] = {
+        labelKeyValue.split("\\:", 2) match {
+          case Array(k, v) => Label.validateLabel(k, v)
+          case other @ _ => s"$labelKeyValue".invalidNel
+        }
+      }
+      val nels: List[ErrorOr[Label]] = values map validateLabelRegex
+      sequenceListOfValidatedNels("Label values do not match allowed pattern label-key:label-value", nels)
     }
   }
 
-  case object Id extends SeqStringWorkflowQueryKey {
+  case object Id extends SeqWorkflowQueryKey[String] {
     override val name = "Id"
 
     override def validate(grouped: Map[String, Seq[(String, String)]]): ErrorOr[List[String]] = {
@@ -55,17 +74,17 @@ object WorkflowQueryKey {
       val nels = values map { v =>
         if (Try(WorkflowId.fromString(v.toLowerCase.capitalize)).isSuccess) v.validNel[String] else v.invalidNel[String]
       }
-      sequenceListOfValidatedNels(s"Id values do match allowed workflow id pattern", nels)
+      sequenceListOfValidatedNels("Id values do match allowed workflow id pattern", nels)
     }
   }
 
-  case object Status extends SeqStringWorkflowQueryKey {
+  case object Status extends SeqWorkflowQueryKey[String] {
     override val name = "Status"
 
     override def validate(grouped: Map[String, Seq[(String, String)]]): ErrorOr[List[String]] = {
       val values = valuesFromMap(grouped).toList
       val nels = values map { v =>
-        if (Try(WorkflowState.fromString(v.toLowerCase.capitalize)).isSuccess) v.validNel[String] else v.invalidNel[String]
+        if (Try(WorkflowState.withName(v.toLowerCase.capitalize)).isSuccess) v.validNel[String] else v.invalidNel[String]
       }
       sequenceListOfValidatedNels("Unrecognized status values", nels)
     }
@@ -96,10 +115,10 @@ sealed trait DateTimeWorkflowQueryKey extends WorkflowQueryKey[Option[OffsetDate
   def displayName: String
 }
 
-sealed trait SeqStringWorkflowQueryKey extends WorkflowQueryKey[Seq[String]] {
-  /** `sequence` the `List[ErrorOr[String]]` to a single `ErrorOr[List[String]]` */
-  protected def sequenceListOfValidatedNels(prefix: String, errorOrList: List[ErrorOr[String]]): ErrorOr[List[String]] = {
-    val errorOr = errorOrList.sequence[ErrorOr, String]
+sealed trait SeqWorkflowQueryKey[A] extends WorkflowQueryKey[Seq[A]] {
+  /** `sequence` the `List[ErrorOr[A]]` to a single `ErrorOr[List[A]]` */
+  protected def sequenceListOfValidatedNels(prefix: String, errorOrList: List[ErrorOr[A]]): ErrorOr[List[A]] = {
+    val errorOr = errorOrList.sequence[ErrorOr, A]
     // With a leftMap, prepend an error message to the concatenated error values if there are error values.
     // This turns the ValidatedNel into a Validated, force it back to a ValidatedNel with toValidationNel.
     errorOr.leftMap(prefix + ": " + _.toList.mkString(", ")).toValidatedNel
diff --git a/services/src/main/scala/cromwell/services/metadata/WorkflowQueryParameters.scala b/services/src/main/scala/cromwell/services/metadata/WorkflowQueryParameters.scala
index 59618d1c8..ba2c7256d 100644
--- a/services/src/main/scala/cromwell/services/metadata/WorkflowQueryParameters.scala
+++ b/services/src/main/scala/cromwell/services/metadata/WorkflowQueryParameters.scala
@@ -6,13 +6,14 @@ import cats.data.Validated._
 import cats.syntax.cartesian._
 import cats.syntax.validated._
 import cromwell.core.WorkflowId
+import cromwell.core.labels.Label
 import cromwell.services.metadata.WorkflowQueryKey._
 import lenthall.validation.ErrorOr._
 
-
 case class WorkflowQueryParameters private(statuses: Set[String],
                                            names: Set[String],
                                            ids: Set[WorkflowId],
+                                           labels: Set[Label],
                                            startDate: Option[OffsetDateTime],
                                            endDate: Option[OffsetDateTime],
                                            page: Option[Int],
@@ -60,10 +61,13 @@ object WorkflowQueryParameters {
       rawParameters groupBy { case (key, _) => key.toLowerCase.capitalize }
 
     val Seq(startDate, endDate) = Seq(StartDate, EndDate) map { _.validate(valuesByCanonicalCapitalization) }
+
     val Seq(statuses, names, ids) = Seq(Status, Name, WorkflowQueryKey.Id) map {
       _.validate(valuesByCanonicalCapitalization)
     }
 
+    val Seq(labels) = Seq(WorkflowQueryKey.LabelKeyValue) map { _.validate(valuesByCanonicalCapitalization) }
+
     val Seq(page, pageSize) = Seq(Page, PageSize) map { _.validate(valuesByCanonicalCapitalization) }
 
     // Only validate start before end if both of the individual date parsing validations have already succeeded.
@@ -72,10 +76,10 @@ object WorkflowQueryParameters {
       case _ => ().validNel[String]
     }
 
-    (onlyRecognizedKeys |@| startBeforeEnd |@| statuses |@| names |@| ids |@| startDate |@| endDate |@| page |@| pageSize) map {
-      case (_, _, status, name, uuid, start, end, _page, _pageSize) =>
+    (onlyRecognizedKeys |@| startBeforeEnd |@| statuses |@| names |@| ids |@| labels |@| startDate |@| endDate |@| page |@| pageSize) map {
+      case (_, _, status, name, uuid, label, start, end, _page, _pageSize) =>
         val workflowId = uuid map WorkflowId.fromString
-        WorkflowQueryParameters(status.toSet, name.toSet, workflowId.toSet, start, end, _page, _pageSize)
+        WorkflowQueryParameters(status.toSet, name.toSet, workflowId.toSet, label.toSet, start, end, _page, _pageSize)
     }
   }
 
diff --git a/services/src/main/scala/cromwell/services/metadata/impl/MetadataDatabaseAccess.scala b/services/src/main/scala/cromwell/services/metadata/impl/MetadataDatabaseAccess.scala
index 066c5eeab..6e1b356df 100644
--- a/services/src/main/scala/cromwell/services/metadata/impl/MetadataDatabaseAccess.scala
+++ b/services/src/main/scala/cromwell/services/metadata/impl/MetadataDatabaseAccess.scala
@@ -5,6 +5,7 @@ import cats.data.NonEmptyList
 import cats.syntax.semigroup._
 import cromwell.core.{WorkflowId, WorkflowMetadataKeys, WorkflowState}
 import cromwell.database.sql.SqlConverters._
+import cromwell.database.sql.joins.{CallOrWorkflowQuery, CallQuery, WorkflowQuery}
 import cromwell.database.sql.tables.{MetadataEntry, WorkflowMetadataSummaryEntry}
 import cromwell.services.ServicesStore
 import cromwell.services.metadata.MetadataService.{QueryMetadata, WorkflowQueryResponse}
@@ -20,8 +21,8 @@ object MetadataDatabaseAccess {
       // Resolve the status if both `this` and `that` have defined statuses.  This will evaluate to `None`
       // if one or both of the statuses is not defined.
       val resolvedStatus = for {
-        thisStatus <- summary1.workflowStatus map WorkflowState.fromString
-        thatStatus <- summary2.workflowStatus map WorkflowState.fromString
+        thisStatus <- summary1.workflowStatus map WorkflowState.withName
+        thatStatus <- summary2.workflowStatus map WorkflowState.withName
       } yield (thisStatus |+| thatStatus).toString
 
       WorkflowMetadataSummaryEntry(
@@ -77,7 +78,7 @@ trait MetadataDatabaseAccess {
       val valueType = metadataEvent.value map { _.valueType.typeName }
       val jobKey = key.jobKey map { jk => (jk.callFqn, jk.index, jk.attempt) }
       MetadataEntry(workflowUuid, jobKey.map(_._1), jobKey.flatMap(_._2), jobKey.map(_._3),
-        key.key, value.toClob, valueType, timestamp)
+        key.key, value.toClobOption, valueType, timestamp)
     }
     databaseInterface.addMetadataEntries(metadata)
   }
@@ -111,15 +112,20 @@ trait MetadataDatabaseAccess {
         databaseInterface.queryMetadataEntries(uuid, key, jobKey.callFqn, jobKey.index, jobKey.attempt)
       case MetadataQuery(_, None, None, Some(includeKeys), None, _) =>
         databaseInterface.
-          queryMetadataEntriesLikeMetadataKeys(uuid, includeKeys.map(_ + "%"), requireEmptyJobKey = false)
+          queryMetadataEntriesLikeMetadataKeys(uuid, includeKeys.map(_ + "%"), CallOrWorkflowQuery)
+      case MetadataQuery(_, Some(MetadataQueryJobKey(callFqn, index, attempt)), None, Some(includeKeys), None, _) =>
+        databaseInterface.
+          queryMetadataEntriesLikeMetadataKeys(uuid, includeKeys.map(_ + "%"), CallQuery(callFqn, index, attempt))
       case MetadataQuery(_, None, None, None, Some(excludeKeys), _) =>
         databaseInterface.
-          queryMetadataEntryNotLikeMetadataKeys(uuid, excludeKeys.map(_ + "%"), requireEmptyJobKey = false)
+          queryMetadataEntryNotLikeMetadataKeys(uuid, excludeKeys.map(_ + "%"), CallOrWorkflowQuery)
+      case MetadataQuery(_, Some(MetadataQueryJobKey(callFqn, index, attempt)), None, None, Some(excludeKeys), _) =>
+        databaseInterface.
+          queryMetadataEntryNotLikeMetadataKeys(uuid, excludeKeys.map(_ + "%"), CallQuery(callFqn, index, attempt))
       case MetadataQuery(_, None, None, Some(includeKeys), Some(excludeKeys), _) => Future.failed(
         new IllegalArgumentException(
           s"Include/Exclude keys may not be mixed: include = $includeKeys, exclude = $excludeKeys"))
-      case invalidQuery => Future.failed(new IllegalArgumentException(
-        s"Include/Exclude keys are only supported when querying the workflow, not when querying calls: $invalidQuery"))
+      case _ => Future.failed(new IllegalArgumentException(s"Invalid MetadataQuery: $query"))
     }
 
     futureMetadata map metadataToMetadataEvents(query.workflowId)
@@ -129,7 +135,7 @@ trait MetadataDatabaseAccess {
                           (implicit ec: ExecutionContext): Future[Seq[MetadataEvent]] = {
     val uuid = id.id.toString
     databaseInterface.queryMetadataEntriesLikeMetadataKeys(
-      uuid, NonEmptyList.of(s"${WorkflowMetadataKeys.Outputs}:%"), requireEmptyJobKey = true).
+      uuid, NonEmptyList.of(s"${WorkflowMetadataKeys.Outputs}:%"), WorkflowQuery).
       map(metadataToMetadataEvents(id))
   }
 
@@ -138,18 +144,18 @@ trait MetadataDatabaseAccess {
     import cromwell.services.metadata.CallMetadataKeys._
 
     val keys = NonEmptyList.of(Stdout, Stderr, BackendLogsPrefix + ":%")
-    databaseInterface.queryMetadataEntriesLikeMetadataKeys(id.id.toString, keys, requireEmptyJobKey = false) map
+    databaseInterface.queryMetadataEntriesLikeMetadataKeys(id.id.toString, keys, CallOrWorkflowQuery) map
       metadataToMetadataEvents(id)
   }
 
   def refreshWorkflowMetadataSummaries()(implicit ec: ExecutionContext): Future[Long] = {
     databaseInterface.refreshMetadataSummaryEntries(WorkflowMetadataKeys.StartTime, WorkflowMetadataKeys.EndTime, WorkflowMetadataKeys.Name,
-      WorkflowMetadataKeys.Status, MetadataDatabaseAccess.buildUpdatedSummary)
+      WorkflowMetadataKeys.Status, WorkflowMetadataKeys.Labels, MetadataDatabaseAccess.buildUpdatedSummary)
   }
 
   def getWorkflowStatus(id: WorkflowId)
                        (implicit ec: ExecutionContext): Future[Option[WorkflowState]] = {
-    databaseInterface.getWorkflowStatus(id.toString) map { _ map WorkflowState.fromString }
+    databaseInterface.getWorkflowStatus(id.toString) map { _ map WorkflowState.withName }
   }
 
   def workflowExistsWithId(possibleWorkflowId: String)(implicit ec: ExecutionContext): Future[Boolean] = {
@@ -158,24 +164,22 @@ trait MetadataDatabaseAccess {
 
   def queryWorkflowSummaries(queryParameters: WorkflowQueryParameters)
                             (implicit ec: ExecutionContext): Future[(WorkflowQueryResponse, Option[QueryMetadata])] = {
+
+    val labelsToQuery = queryParameters.labels.map(label => (label.key, label.value))
+
     val workflowSummaries = databaseInterface.queryWorkflowSummaries(
-      queryParameters.statuses, queryParameters.names, queryParameters.ids.map(_.toString),
+      queryParameters.statuses, queryParameters.names, queryParameters.ids.map(_.toString), labelsToQuery,
       queryParameters.startDate.map(_.toSystemTimestamp), queryParameters.endDate.map(_.toSystemTimestamp),
       queryParameters.page, queryParameters.pageSize)
 
     val workflowSummaryCount = databaseInterface.countWorkflowSummaries(
-      queryParameters.statuses, queryParameters.names, queryParameters.ids.map(_.toString),
+      queryParameters.statuses, queryParameters.names, queryParameters.ids.map(_.toString), queryParameters.labels.map(label => (label.key, label.value)),
       queryParameters.startDate.map(_.toSystemTimestamp), queryParameters.endDate.map(_.toSystemTimestamp))
 
     workflowSummaryCount flatMap { count =>
       workflowSummaries map { workflows =>
         (WorkflowQueryResponse(workflows.toSeq map { workflow =>
-          MetadataService.WorkflowQueryResult(
-            id = workflow.workflowExecutionUuid,
-            name = workflow.workflowName,
-            status = workflow.workflowStatus,
-            start = workflow.startTimestamp map { _.toSystemOffsetDateTime },
-            end = workflow.endTimestamp map { _.toSystemOffsetDateTime })
+          MetadataService.WorkflowQueryResult(id = workflow.workflowExecutionUuid, name = workflow.workflowName, status = workflow.workflowStatus, start = workflow.startTimestamp map { _.toSystemOffsetDateTime }, end = workflow.endTimestamp map { _.toSystemOffsetDateTime })
         }),
           //only return metadata if page is defined
           queryParameters.page map { _ => QueryMetadata(queryParameters.page, queryParameters.pageSize, Option(count)) })
diff --git a/services/src/main/scala/cromwell/services/metadata/impl/MetadataServiceActor.scala b/services/src/main/scala/cromwell/services/metadata/impl/MetadataServiceActor.scala
index 4c343d173..2312d0df9 100644
--- a/services/src/main/scala/cromwell/services/metadata/impl/MetadataServiceActor.scala
+++ b/services/src/main/scala/cromwell/services/metadata/impl/MetadataServiceActor.scala
@@ -1,21 +1,24 @@
 package cromwell.services.metadata.impl
 
-import java.util.UUID
 
 import akka.actor.SupervisorStrategy.{Decider, Directive, Escalate, Resume}
-import akka.actor.{Actor, ActorContext, ActorInitializationException, ActorLogging, ActorRef, OneForOneStrategy, Props}
+import akka.actor.{Actor, ActorContext, ActorInitializationException, ActorLogging, ActorRef, Cancellable, OneForOneStrategy, Props}
+import cats.data.NonEmptyList
 import com.typesafe.config.{Config, ConfigFactory}
 import cromwell.core.Dispatcher.ServiceDispatcher
 import cromwell.core.WorkflowId
 import cromwell.services.SingletonServicesStore
-import cromwell.services.metadata.MetadataService.{PutMetadataAction, ReadAction, RefreshSummary, ValidateWorkflowIdAndExecute}
+import cromwell.services.metadata.MetadataService._
 import cromwell.services.metadata.impl.MetadataServiceActor._
 import cromwell.services.metadata.impl.MetadataSummaryRefreshActor.{MetadataSummaryFailure, MetadataSummarySuccess, SummarizeMetadata}
+import cromwell.services.metadata.impl.WriteMetadataActor.CheckPendingWrites
+import cromwell.util.GracefulShutdownHelper
+import cromwell.util.GracefulShutdownHelper.ShutdownCommand
 import net.ceedubs.ficus.Ficus._
 
 import scala.concurrent.duration._
 import scala.language.postfixOps
-import scala.util.{Failure, Success, Try}
+import scala.util.{Failure, Success}
 
 object MetadataServiceActor {
 
@@ -28,7 +31,7 @@ object MetadataServiceActor {
 }
 
 case class MetadataServiceActor(serviceConfig: Config, globalConfig: Config)
-  extends Actor with ActorLogging with MetadataDatabaseAccess with SingletonServicesStore {
+  extends Actor with ActorLogging with MetadataDatabaseAccess with SingletonServicesStore with GracefulShutdownHelper {
   
   private val decider: Decider = {
     case _: ActorInitializationException => Escalate
@@ -41,20 +44,28 @@ case class MetadataServiceActor(serviceConfig: Config, globalConfig: Config)
       log.error(s"The $childName Metadata Actor died unexpectedly, metadata events might have been lost. Restarting it...", cause)
     }
   }
-  
+
   private val summaryActor: Option[ActorRef] = buildSummaryActor
 
   val readActor = context.actorOf(ReadMetadataActor.props(), "read-metadata-actor")
 
   val dbFlushRate = serviceConfig.as[Option[FiniteDuration]]("services.MetadataService.db-flush-rate").getOrElse(5 seconds)
-  val dbBatchSize = serviceConfig.as[Option[Int]]("services.MetadataService.db-batch-size").getOrElse(1)
+  val dbBatchSize = serviceConfig.as[Option[Int]]("services.MetadataService.db-batch-size").getOrElse(200)
   val writeActor = context.actorOf(WriteMetadataActor.props(dbBatchSize, dbFlushRate), "write-metadata-actor")
   implicit val ec = context.dispatcher
-
+  private var summaryRefreshCancellable: Option[Cancellable] = None
+  
   summaryActor foreach { _ => self ! RefreshSummary }
 
   private def scheduleSummary(): Unit = {
-    MetadataSummaryRefreshInterval foreach { context.system.scheduler.scheduleOnce(_, self, RefreshSummary)(context.dispatcher, self) }
+    MetadataSummaryRefreshInterval foreach { interval =>
+      summaryRefreshCancellable = Option(context.system.scheduler.scheduleOnce(interval, self, RefreshSummary)(context.dispatcher, self)) 
+    }
+  }
+
+  override def postStop(): Unit = {
+    summaryRefreshCancellable foreach { _.cancel() }
+    super.postStop()
   }
 
   private def buildSummaryActor: Option[ActorRef] = {
@@ -69,27 +80,20 @@ case class MetadataServiceActor(serviceConfig: Config, globalConfig: Config)
     actor
   }
 
-  private def validateWorkflowId(validation: ValidateWorkflowIdAndExecute): Unit = {
-    val possibleWorkflowId = validation.possibleWorkflowId
-    val callback = validation.validationCallback
-
-    Try(UUID.fromString(possibleWorkflowId)) match {
-      case Failure(t) => callback.onMalformed(possibleWorkflowId)
-      case Success(uuid) =>
-        workflowExistsWithId(possibleWorkflowId) onComplete {
-          case Success(true) =>
-            callback.onRecognized(WorkflowId(uuid))
-          case Success(false) =>
-            callback.onUnrecognized(possibleWorkflowId)
-          case Failure(t) =>
-            callback.onFailure(possibleWorkflowId, t)
-        }
+  private def validateWorkflowId(possibleWorkflowId: WorkflowId, sender: ActorRef): Unit = {
+    workflowExistsWithId(possibleWorkflowId.toString) onComplete {
+      case Success(true) => sender ! RecognizedWorkflowId
+      case Success(false) => sender ! UnrecognizedWorkflowId
+      case Failure(e) => sender ! FailedToCheckWorkflowId(new RuntimeException(s"Failed lookup attempt for workflow ID $possibleWorkflowId", e))
     }
   }
 
   def receive = {
-    case action@PutMetadataAction(events) => writeActor forward action
-    case v: ValidateWorkflowIdAndExecute => validateWorkflowId(v)
+    case ShutdownCommand => waitForActorsAndShutdown(NonEmptyList.of(writeActor))
+    case action: PutMetadataAction => writeActor forward action
+    case action: PutMetadataActionAndRespond => writeActor forward action
+    case CheckPendingWrites => writeActor forward CheckPendingWrites
+    case v: ValidateWorkflowId => validateWorkflowId(v.possibleWorkflowId, sender())
     case action: ReadAction => readActor forward action
     case RefreshSummary => summaryActor foreach { _ ! SummarizeMetadata(sender()) }
     case MetadataSummarySuccess => scheduleSummary()
diff --git a/services/src/main/scala/cromwell/services/metadata/impl/MetadataSummaryRefreshActor.scala b/services/src/main/scala/cromwell/services/metadata/impl/MetadataSummaryRefreshActor.scala
index ef16bdb89..172d0012b 100644
--- a/services/src/main/scala/cromwell/services/metadata/impl/MetadataSummaryRefreshActor.scala
+++ b/services/src/main/scala/cromwell/services/metadata/impl/MetadataSummaryRefreshActor.scala
@@ -41,9 +41,9 @@ class MetadataSummaryRefreshActor()
   startWith(WaitingForRequest, SummaryRefreshData)
 
   when (WaitingForRequest) {
-    case (Event(SummarizeMetadata(respondTo), data)) =>
+    case (Event(SummarizeMetadata(respondTo), _)) =>
       refreshWorkflowMetadataSummaries() onComplete {
-        case Success(id) =>
+        case Success(_) =>
           respondTo ! MetadataSummarySuccess
           self ! MetadataSummaryComplete
         case Failure(t) =>
diff --git a/services/src/main/scala/cromwell/services/metadata/impl/ReadMetadataActor.scala b/services/src/main/scala/cromwell/services/metadata/impl/ReadMetadataActor.scala
index 5308d69f2..2ad5a9d5c 100644
--- a/services/src/main/scala/cromwell/services/metadata/impl/ReadMetadataActor.scala
+++ b/services/src/main/scala/cromwell/services/metadata/impl/ReadMetadataActor.scala
@@ -27,7 +27,7 @@ class ReadMetadataActor extends Actor with ActorLogging with MetadataDatabaseAcc
     case GetMetadataQueryAction(query@MetadataQuery(_, _, _, _, _, _)) => queryAndRespond(query)
     case GetStatus(workflowId) => queryStatusAndRespond(workflowId)
     case GetLogs(workflowId) => queryLogsAndRespond(workflowId)
-    case query: WorkflowQuery[_] => queryWorkflowsAndRespond(query.uri, query.parameters)
+    case query: WorkflowQuery => queryWorkflowsAndRespond(query.parameters)
     case WorkflowOutputs(id) => queryWorkflowOutputsAndRespond(id)
   }
 
@@ -50,7 +50,7 @@ class ReadMetadataActor extends Actor with ActorLogging with MetadataDatabaseAcc
     }
   }
 
-  private def queryWorkflowsAndRespond[A](uri: A, rawParameters: Seq[(String, String)]): Unit = {
+  private def queryWorkflowsAndRespond(rawParameters: Seq[(String, String)]): Unit = {
     def queryWorkflows: Future[(WorkflowQueryResponse, Option[QueryMetadata])] = {
       for {
       // Future/Try to wrap the exception that might be thrown from WorkflowQueryParameters.apply.
@@ -62,7 +62,7 @@ class ReadMetadataActor extends Actor with ActorLogging with MetadataDatabaseAcc
     val sndr = sender()
 
     queryWorkflows onComplete {
-      case Success((response, metadata)) => sndr ! WorkflowQuerySuccess(uri, response, metadata)
+      case Success((response, metadata)) => sndr ! WorkflowQuerySuccess(response, metadata)
       case Failure(t) => sndr ! WorkflowQueryFailure(t)
     }
   }
diff --git a/services/src/main/scala/cromwell/services/metadata/impl/WriteMetadataActor.scala b/services/src/main/scala/cromwell/services/metadata/impl/WriteMetadataActor.scala
index b4db0320e..cdd6e9237 100644
--- a/services/src/main/scala/cromwell/services/metadata/impl/WriteMetadataActor.scala
+++ b/services/src/main/scala/cromwell/services/metadata/impl/WriteMetadataActor.scala
@@ -1,98 +1,96 @@
 package cromwell.services.metadata.impl
 
-import akka.actor.{ActorLogging, LoggingFSM, Props}
-import cats.data.NonEmptyVector
+import akka.actor.{ActorLogging, ActorRef, LoggingFSM, Props}
 import cromwell.core.Dispatcher.ServiceDispatcher
+import cromwell.core.actor.BatchingDbWriter._
+import cromwell.core.actor.{BatchingDbWriter, BatchingDbWriterActor}
 import cromwell.services.SingletonServicesStore
 import cromwell.services.metadata.MetadataEvent
-import cromwell.services.metadata.MetadataService.PutMetadataAction
-import cromwell.services.metadata.impl.WriteMetadataActor.{WriteMetadataActorData, WriteMetadataActorState}
-import org.slf4j.LoggerFactory
+import cromwell.services.metadata.MetadataService._
 
+import scala.concurrent.ExecutionContext
 import scala.concurrent.duration._
-import scala.util.{Failure, Success, Try}
+import scala.util.{Failure, Success}
 
-final case class WriteMetadataActor(batchRate: Int, flushRate: FiniteDuration)
-  extends LoggingFSM[WriteMetadataActorState, WriteMetadataActorData] with ActorLogging with
-  MetadataDatabaseAccess with SingletonServicesStore {
-  import WriteMetadataActor._
 
-  implicit val ec = context.dispatcher
+class WriteMetadataActor(batchSize: Int, override val dbFlushRate: FiniteDuration)
+  extends LoggingFSM[BatchingDbWriterState, BatchingDbWriter.BatchingDbWriterData] with ActorLogging with
+  MetadataDatabaseAccess with SingletonServicesStore with BatchingDbWriterActor {
+  import WriteMetadataActor._
 
-  override def preStart(): Unit = {
-    context.system.scheduler.schedule(0.seconds, flushRate, self, ScheduledFlushToDb)
-    super.preStart()
-  }
+  implicit val ec: ExecutionContext = context.dispatcher
 
-  startWith(WaitingToWrite, NoEvents)
+  startWith(WaitingToWrite, NoData)
 
   when(WaitingToWrite) {
     case Event(PutMetadataAction(events), curData) =>
-      curData.addEvents(events) match {
-        case data@HasEvents(e) if e.length > batchRate => goto(WritingToDb) using data
-        case e => stay using e
+      curData.addData(events) match {
+        case newData: HasData[_] if newData.length > batchSize => goto(WritingToDb) using newData
+        case newData => stay using newData
       }
     case Event(ScheduledFlushToDb, curData) =>
       log.debug("Initiating periodic metadata flush to DB")
       goto(WritingToDb) using curData
+    case Event(CheckPendingWrites, NoData) =>
+      sender() ! NoPendingWrites
+      stay()
+    case Event(CheckPendingWrites, _: HasData[_]) =>
+      sender() ! HasPendingWrites
+      stay()
+    case Event(e: PutMetadataActionAndRespond, curData) =>
+      curData.addData(e) match {
+        case newData: HasData[_] if newData.length > batchSize => goto(WritingToDb) using newData
+        case newData => stay using newData
+      }
   }
 
   when(WritingToDb) {
+    case Event(CheckPendingWrites, _) => 
+      sender() ! HasPendingWrites
+      stay()
     case Event(ScheduledFlushToDb, curData) => stay using curData
-    case Event(PutMetadataAction(events), curData) => stay using curData.addEvents(events)
-    case Event(FlushBatchToDb, NoEvents) =>
+    case Event(PutMetadataAction(events), curData) => stay using curData.addData(events)
+    case Event(FlushBatchToDb, NoData) =>
       log.debug("Attempted metadata flush to DB but had nothing to write")
-      goto(WaitingToWrite) using NoEvents
-    case Event(FlushBatchToDb, HasEvents(e)) =>
+      goto(WaitingToWrite) using NoData
+    case Event(FlushBatchToDb, HasData(e)) =>
       log.debug("Flushing {} metadata events to the DB", e.length)
-      addMetadataEvents(e.toVector) onComplete {
-        case Success(_) => self ! DbWriteComplete
+      // blech
+      //Partitioning the current data into put events that require a response and those that don't
+      val empty = (Vector.empty[MetadataEvent], Map.empty[Iterable[MetadataEvent], ActorRef])
+      val (putWithoutResponse, putWithResponse) = e.toVector.foldLeft(empty)({
+        case ((putEvents, putAndRespondEvents), events) =>
+          events match {
+            case putEvent: MetadataEvent => (putEvents :+ putEvent, putAndRespondEvents)
+            case PutMetadataActionAndRespond(ev, replyTo) => (putEvents, putAndRespondEvents + (ev -> replyTo))
+          }
+      })
+      val allPutEvents: Iterable[MetadataEvent] = putWithoutResponse ++ putWithResponse.keys.flatten
+      addMetadataEvents(allPutEvents) onComplete {
+        case Success(_) =>
+          self ! DbWriteComplete
+          putWithResponse foreach { case(ev, replyTo) => replyTo ! MetadataWriteSuccess(ev) }
         case Failure(regerts) =>
-          log.error("Failed to properly flush metadata to database", regerts)
+          log.error(regerts, "Failed to properly flush metadata to database")
           self ! DbWriteComplete
+          putWithResponse foreach { case(ev, replyTo) => replyTo ! MetadataWriteFailure(regerts, ev) }
       }
-
-      stay using NoEvents
+      stay using NoData
     case Event(DbWriteComplete, curData) =>
       log.debug("Flush of metadata events complete")
       goto(WaitingToWrite) using curData
-  }
-
-  onTransition {
-    case WaitingToWrite -> WritingToDb => self ! FlushBatchToDb
+    // When receiving a put&respond message, add it to the current data so that when flushing metadata events, we have
+    // enough information to be able to send an acknowledgement of success/failure of metadata event writes to the original requester.
+    case Event(PutMetadataActionAndRespond(events, replyTo), curData) =>
+      stay using curData.addData(PutMetadataActionAndRespond(events, replyTo))
   }
 }
 
 object WriteMetadataActor {
-  def props(batchRate: Int, flushRate: FiniteDuration) = Props(new WriteMetadataActor(batchRate, flushRate)).withDispatcher(ServiceDispatcher)
-  private lazy val logger = LoggerFactory.getLogger("WriteMetadataActor")
+  def props(batchSize: Int, flushRate: FiniteDuration): Props = Props(new WriteMetadataActor(batchSize, flushRate)).withDispatcher(ServiceDispatcher)
 
   sealed trait WriteMetadataActorMessage
-  case object DbWriteComplete extends WriteMetadataActorMessage
-  case object FlushBatchToDb extends WriteMetadataActorMessage
-  case object ScheduledFlushToDb extends WriteMetadataActorMessage
-
-  sealed trait WriteMetadataActorState
-  case object WaitingToWrite extends WriteMetadataActorState
-  case object WritingToDb extends WriteMetadataActorState
-
-  sealed trait WriteMetadataActorData {
-    def addEvents(newEvents: Iterable[MetadataEvent]): WriteMetadataActorData = {
-      Try(NonEmptyVector.fromVector(newEvents.toVector)) match {
-        case Success(Some(v)) =>
-          val newEvents = this match {
-            case NoEvents => v
-            case HasEvents(e) => e.concatNev(v)
-          }
-          HasEvents(newEvents)
-        case Success(None) => this
-        case Failure(f) =>
-          logger.error("Failed processing metadata events. Events will be dropped and not be sent to the database.", f)
-          this
-      }
-    }
-  }
-
-  case object NoEvents extends WriteMetadataActorData
-  case class HasEvents(events: NonEmptyVector[MetadataEvent]) extends WriteMetadataActorData
+  case object CheckPendingWrites extends WriteMetadataActorMessage with MetadataServiceAction
+  case object HasPendingWrites extends WriteMetadataActorMessage
+  case object NoPendingWrites extends WriteMetadataActorMessage
 }
diff --git a/services/src/test/scala/cromwell/services/ServicesSpec.scala b/services/src/test/scala/cromwell/services/ServicesSpec.scala
index b2bf1e49f..71d56d39a 100644
--- a/services/src/test/scala/cromwell/services/ServicesSpec.scala
+++ b/services/src/test/scala/cromwell/services/ServicesSpec.scala
@@ -40,7 +40,7 @@ object ServicesSpec {
       |    }
       |
       |    # A dispatcher for engine actors
-      |    # Because backends behaviour is unpredictable (potentially blocking, slow) the engine runs
+      |    # Because backends behavior is unpredictable (potentially blocking, slow) the engine runs
       |    # on its own dispatcher to prevent backends from affecting its performance.
       |    engine-dispatcher {
       |      type = Dispatcher
diff --git a/services/src/test/scala/cromwell/services/ServicesStoreSpec.scala b/services/src/test/scala/cromwell/services/ServicesStoreSpec.scala
index 0e0cfbfd7..b62669e2e 100644
--- a/services/src/test/scala/cromwell/services/ServicesStoreSpec.scala
+++ b/services/src/test/scala/cromwell/services/ServicesStoreSpec.scala
@@ -1,12 +1,12 @@
 package cromwell.services
 
 import java.io.{ByteArrayOutputStream, PrintStream}
-import java.sql.{Connection, JDBCType}
+import java.sql.Connection
 import java.time.OffsetDateTime
 import javax.sql.rowset.serial.{SerialBlob, SerialClob, SerialException}
 
 import better.files._
-import com.typesafe.config.{Config, ConfigFactory}
+import com.typesafe.config.ConfigFactory
 import cromwell.core.Tags._
 import cromwell.core.WorkflowId
 import cromwell.database.migration.liquibase.LiquibaseUtils
@@ -23,11 +23,11 @@ import org.scalatest.concurrent.PatienceConfiguration.Timeout
 import org.scalatest.concurrent.ScalaFutures
 import org.scalatest.time.{Millis, Seconds, Span}
 import org.scalatest.{FlatSpec, Matchers}
-import slick.driver.JdbcProfile
+import slick.jdbc.JdbcProfile
 import slick.jdbc.meta._
 
 import scala.concurrent.duration._
-import scala.concurrent.{Await, ExecutionContext, Future, TimeoutException}
+import scala.concurrent.{Await, ExecutionContext, Future}
 import scala.util.Try
 import scala.xml._
 
@@ -41,53 +41,6 @@ class ServicesStoreSpec extends FlatSpec with Matchers with ScalaFutures with St
 
   behavior of "ServicesStore"
 
-  it should "deadlock and timeout after 10 seconds" in {
-    /*
-    Tests that we still need our deadlock workaround in `database.run` to deal with problems in Slick 3.1.
-    If/when this test fails, the workaround should be removed.
-    */
-
-    class DeadlockingSlickDatabase(config: Config) extends SlickDatabase(config) {
-
-      import dataAccess.driver.api._
-
-      override protected[this] def runTransaction[R](action: DBIO[R]): Future[R] = {
-        database.run(action.transactionally) //<-- https://github.com/slick/slick/issues/1274
-      }
-    }
-
-    // Test based on https://github.com/kwark/slick-deadlock/blob/82525fc/src/main/scala/SlickDeadlock.scala
-    val databaseConfig = ConfigFactory.parseString(
-      s"""|db.url = "jdbc:hsqldb:mem:$${uniqueSchema};shutdown=false;hsqldb.tx=mvcc"
-          |db.driver = "org.hsqldb.jdbcDriver"
-          |db.connectionTimeout = 3000
-          |db.numThreads = 2
-          |driver = "slick.driver.HsqldbDriver$$"
-          |""".stripMargin)
-    import ServicesStore.EnhancedSqlDatabase
-    for {
-      database <- new DeadlockingSlickDatabase(databaseConfig).initialized.autoClosed
-    } {
-      val futures = 1 to 20 map { _ =>
-        val workflowUuid = WorkflowId.randomId().toString
-        val callFqn = "call.fqn"
-        val jobIndex = 1
-        val jobAttempt = 1
-        val jobSuccessful = false
-        val jobStoreEntry = JobStoreEntry(workflowUuid, callFqn, jobIndex, jobAttempt, jobSuccessful, None, None, None)
-        val jobStoreJoins = Seq(JobStoreJoin(jobStoreEntry, Seq()))
-        // NOTE: This test just needs to repeatedly read/write from a table that acts as a PK for a FK.
-        for {
-          _ <- database.addJobStores(jobStoreJoins)
-          queried <- database.queryJobStores(workflowUuid, callFqn, jobIndex, jobAttempt)
-          _ = queried.get.jobStoreEntry.workflowExecutionUuid should be(workflowUuid)
-        } yield ()
-      }
-
-      intercept[TimeoutException](Await.result(Future.sequence(futures), 10.seconds))
-    }
-  }
-
   it should "not deadlock" in {
     // Test based on https://github.com/kwark/slick-deadlock/blob/82525fc/src/main/scala/SlickDeadlock.scala
     val databaseConfig = ConfigFactory.parseString(
@@ -95,7 +48,7 @@ class ServicesStoreSpec extends FlatSpec with Matchers with ScalaFutures with St
           |db.driver = "org.hsqldb.jdbcDriver"
           |db.connectionTimeout = 3000
           |db.numThreads = 2
-          |driver = "slick.driver.HsqldbDriver$$"
+          |profile = "slick.jdbc.HsqldbProfile$$"
           |""".stripMargin)
     import ServicesStore.EnhancedSqlDatabase
     for {
@@ -111,7 +64,7 @@ class ServicesStoreSpec extends FlatSpec with Matchers with ScalaFutures with St
         val jobStoreJoins = Seq(JobStoreJoin(jobStoreEntry, Seq()))
         // NOTE: This test just needs to repeatedly read/write from a table that acts as a PK for a FK.
         for {
-          _ <- database.addJobStores(jobStoreJoins)
+          _ <- database.addJobStores(jobStoreJoins, 10)
           queried <- database.queryJobStores(workflowUuid, callFqn, jobIndex, jobAttempt)
           _ = queried.get.jobStoreEntry.workflowExecutionUuid should be(workflowUuid)
         } yield ()
@@ -145,7 +98,6 @@ class ServicesStoreSpec extends FlatSpec with Matchers with ScalaFutures with St
           val diffFilters = StandardTypeFilters
           val filteredDiffResult = diffResult
             .filterLiquibaseObjects
-            .filterTableObjects(oldeTables)
             .filterChangedObjects(diffFilters)
 
           val totalChanged =
@@ -183,7 +135,6 @@ class ServicesStoreSpec extends FlatSpec with Matchers with ScalaFutures with St
             tables <- slickDatabase.database.run(MTable.getTables(Option("PUBLIC"), Option("PUBLIC"), None, None))
             workingTables = tables
               .filterNot(_.name.name.startsWith("DATABASECHANGELOG"))
-              .filterNot(table => oldeTables.contains(table.name.name))
               // NOTE: MetadataEntry column names are perma-busted due to the large size of the table.
               .filterNot(_.name.name == "METADATA_ENTRY")
             columns <- slickDatabase.database.run(DBIO.sequence(workingTables.map(_.getColumns)))
@@ -253,18 +204,7 @@ class ServicesStoreSpec extends FlatSpec with Matchers with ScalaFutures with St
         }
       }
 
-      var nullTyped = Seq.empty[String]
-
-      schemaMetadata.columnMetadata foreach { column =>
-        if (column.isNullable == Option(false)) {
-          val jdbcType = JDBCType.valueOf(column.sqlType)
-          if (jdbcType == JDBCType.BLOB || jdbcType == JDBCType.CLOB) {
-            nullTyped :+= s"  $jdbcType column ${column.table.name}.${column.name}"
-          }
-        }
-      }
-
-      if (missing.nonEmpty || misnamed.nonEmpty || nullTyped.nonEmpty) {
+      if (missing.nonEmpty || misnamed.nonEmpty) {
         var failMessage = ""
 
         if (misnamed.nonEmpty) {
@@ -277,11 +217,6 @@ class ServicesStoreSpec extends FlatSpec with Matchers with ScalaFutures with St
             "\n", "\n")
         }
 
-        if (nullTyped.nonEmpty) {
-          failMessage += nullTyped.mkString(
-            s"The following columns should not be nullable due to incompatibilities with MySQL:\n", "\n", "\n")
-        }
-
         fail(failMessage)
       }
     }
@@ -321,11 +256,8 @@ class ServicesStoreSpec extends FlatSpec with Matchers with ScalaFutures with St
       } yield ()).futureValue
     }
 
-    it should "fail to store and retrieve empty clobs in MySQL" taggedAs DbmsTest in {
-      /*
-      Tests that we still need our empty clob workaround in `SqlConverters.toClob`. The current mysql jdbc driver throws
-      an exception when serializing an empty clob. If/when this test fails, the workaround should be removed.
-      */
+    it should "fail to store and retrieve empty clobs" taggedAs DbmsTest in {
+      // See notes in StringToClobOption
       val emptyClob = new SerialClob(Array.empty[Char])
 
       val workflowUuid = WorkflowId.randomId().toString
@@ -339,44 +271,56 @@ class ServicesStoreSpec extends FlatSpec with Matchers with ScalaFutures with St
 
       val future = for {
         product <- dataAccess.database.run(getProduct)
-        _ = product match {
+        _ <- product match {
+          case "HSQL Database Engine" =>
+            // HSQLDB doesn't crash because it calls getCharacterStream instead of getSubString.
+            dataAccess.addJobStores(jobStoreJoins, 1)
           case "MySQL" =>
-            val exception = intercept[SerialException] {
-              Await.result(dataAccess.addJobStores(jobStoreJoins), Duration.Inf)
+            dataAccess.addJobStores(jobStoreJoins, 1).failed map { exception =>
+              exception should be(a[SerialException])
+              exception.getMessage should be("Invalid position in SerialClob object set")
             }
-            exception should be(a[SerialException])
-            exception.getMessage should be("Invalid position in SerialClob object set")
-          case _ => ()
         }
       } yield ()
 
       future.futureValue
     }
 
-    it should "fail to store and retrieve empty blobs in MySQL" taggedAs DbmsTest in {
-      /*
-      Tests that we still need our empty blob workaround in `SqlConverters.toBlob`. The current mysql jdbc driver throws
-      an exception when serializing an empty blob. If/when this test fails, the workaround should be removed.
-      */
+    it should "fail to store and retrieve empty blobs" taggedAs DbmsTest in {
+      // See notes in BytesToBlobOption
+      import eu.timepit.refined.auto._
+      import eu.timepit.refined.collection._
+      val clob = "".toClob(default = "{}")
+      val clobOption = "{}".toClobOption
       val emptyBlob = new SerialBlob(Array.empty[Byte])
 
       val workflowUuid = WorkflowId.randomId().toString
-      val workflowStoreEntry = WorkflowStoreEntry(workflowUuid, "{}".toClob, "{}".toClob, "{}".toClob,
-        "Testing", OffsetDateTime.now.toSystemTimestamp, Option(emptyBlob), "{}".toClob)
+      val workflowStoreEntry = WorkflowStoreEntry(
+        workflowExecutionUuid = workflowUuid,
+        workflowType = WdlWorkflowType,
+        workflowTypeVersion = None,
+        workflowDefinition = clobOption,
+        workflowInputs = clobOption,
+        workflowOptions = clobOption,
+        workflowState = "Testing",
+        submissionTime = OffsetDateTime.now.toSystemTimestamp,
+        importsZip = Option(emptyBlob),
+        customLabels = clob)
 
       val workflowStoreEntries = Seq(workflowStoreEntry)
 
       val future = for {
         product <- dataAccess.database.run(getProduct)
-        _ = product match {
+        _ <- product match {
+          case "HSQL Database Engine" =>
+            // HSQLDB doesn't crash because it calls getBinaryStream instead of getBytes.
+            dataAccess.addWorkflowStoreEntries(workflowStoreEntries)
           case "MySQL" =>
-            val exception = intercept[SerialException] {
-              Await.result(dataAccess.addWorkflowStoreEntries(workflowStoreEntries), Duration.Inf)
+            dataAccess.addWorkflowStoreEntries(workflowStoreEntries).failed map { exception =>
+              exception should be(a[SerialException])
+              exception.getMessage should
+                be("Invalid arguments: position cannot be less than 1 or greater than the length of the SerialBlob")
             }
-            exception should be(a[SerialException])
-            exception.getMessage should
-              be("Invalid arguments: position cannot be less than 1 or greater than the length of the SerialBlob")
-          case _ => ()
         }
       } yield ()
 
@@ -384,6 +328,7 @@ class ServicesStoreSpec extends FlatSpec with Matchers with ScalaFutures with St
     }
 
     it should "store and retrieve empty clobs" taggedAs DbmsTest in {
+      // See notes in StringToClobOption
       val workflowUuid = WorkflowId.randomId().toString
       val callFqn = "call.fqn"
       val jobIndex = 1
@@ -391,13 +336,13 @@ class ServicesStoreSpec extends FlatSpec with Matchers with ScalaFutures with St
       val jobSuccessful = false
       val jobStoreEntry = JobStoreEntry(workflowUuid, callFqn, jobIndex, jobAttempt, jobSuccessful, None, None, None)
       val jobStoreSimpletonEntries = Seq(
-        JobStoreSimpletonEntry("empty", "".toClob, "WdlString"),
-        JobStoreSimpletonEntry("aEntry", "a".toClob, "WdlString")
+        JobStoreSimpletonEntry("empty", "".toClobOption, "WdlString"),
+        JobStoreSimpletonEntry("aEntry", "a".toClobOption, "WdlString")
       )
       val jobStoreJoins = Seq(JobStoreJoin(jobStoreEntry, jobStoreSimpletonEntries))
 
       val future = for {
-        _ <- dataAccess.addJobStores(jobStoreJoins)
+        _ <- dataAccess.addJobStores(jobStoreJoins, 1)
         queried <- dataAccess.queryJobStores(workflowUuid, callFqn, jobIndex, jobAttempt)
         _ = {
           val jobStoreJoin = queried.get
@@ -415,20 +360,53 @@ class ServicesStoreSpec extends FlatSpec with Matchers with ScalaFutures with St
     }
 
     it should "store and retrieve empty blobs" taggedAs DbmsTest in {
+      // See notes in BytesToBlobOption
+      import eu.timepit.refined.auto._
+      import eu.timepit.refined.collection._
+
       val testWorkflowState = "Testing"
+      val clob = "".toClob(default = "{}")
+      val clobOption = "{}".toClobOption
 
       val emptyWorkflowUuid = WorkflowId.randomId().toString
-      val emptyWorkflowStoreEntry = WorkflowStoreEntry(emptyWorkflowUuid, "{}".toClob, "{}".toClob, "{}".toClob,
-        testWorkflowState, OffsetDateTime.now.toSystemTimestamp, Option(Array.empty[Byte]).toBlob, "{}".toClob)
+      val emptyWorkflowStoreEntry = WorkflowStoreEntry(
+        workflowExecutionUuid = emptyWorkflowUuid,
+        workflowType = WdlWorkflowType,
+        workflowTypeVersion = None,
+        workflowDefinition = clobOption,
+        workflowInputs = clobOption,
+        workflowOptions = clobOption,
+        workflowState = testWorkflowState,
+        submissionTime = OffsetDateTime.now.toSystemTimestamp,
+        importsZip = Option(Array.empty[Byte]).toBlobOption,
+        customLabels = clob)
 
       val noneWorkflowUuid = WorkflowId.randomId().toString
-      val noneWorkflowStoreEntry = WorkflowStoreEntry(noneWorkflowUuid, "{}".toClob, "{}".toClob, "{}".toClob,
-        testWorkflowState, OffsetDateTime.now.toSystemTimestamp, None, "{}".toClob)
+      val noneWorkflowStoreEntry = WorkflowStoreEntry(
+        workflowExecutionUuid = noneWorkflowUuid,
+        workflowType = WdlWorkflowType,
+        workflowTypeVersion = None,
+        workflowDefinition = clobOption,
+        workflowInputs = clobOption,
+        workflowOptions = clobOption,
+        workflowState = testWorkflowState,
+        submissionTime = OffsetDateTime.now.toSystemTimestamp,
+        importsZip = None,
+        customLabels = clob)
 
       val aByte = 'a'.toByte
       val aByteWorkflowUuid = WorkflowId.randomId().toString
-      val aByteWorkflowStoreEntry = WorkflowStoreEntry(aByteWorkflowUuid, "{}".toClob, "{}".toClob, "{}".toClob,
-        testWorkflowState, OffsetDateTime.now.toSystemTimestamp, Option(Array(aByte)).toBlob, "{}".toClob)
+      val aByteWorkflowStoreEntry = WorkflowStoreEntry(
+        workflowExecutionUuid = aByteWorkflowUuid,
+        workflowType = WdlWorkflowType,
+        workflowTypeVersion = None,
+        workflowDefinition = clobOption,
+        workflowInputs = clobOption,
+        workflowOptions = clobOption,
+        workflowState = testWorkflowState,
+        submissionTime = OffsetDateTime.now.toSystemTimestamp,
+        importsZip = Option(Array(aByte)).toBlobOption,
+        customLabels = clob)
 
       val workflowStoreEntries = Seq(emptyWorkflowStoreEntry, noneWorkflowStoreEntry, aByteWorkflowStoreEntry)
 
@@ -459,18 +437,6 @@ class ServicesStoreSpec extends FlatSpec with Matchers with ScalaFutures with St
 }
 
 object ServicesStoreSpec {
-  // TODO PBE get rid of this after the migration of #789 has run.
-  private val oldeTables = Seq(
-    "EXECUTION",
-    "EXECUTION_EVENT",
-    "EXECUTION_INFO",
-    "FAILURE_EVENT",
-    "RUNTIME_ATTRIBUTES",
-    "SYMBOL",
-    "WORKFLOW_EXECUTION",
-    "WORKFLOW_EXECUTION_AUX"
-  )
-
   // strip the namespace from elems and their children
   private def stripNodeScope(node: Node): Node = {
     node match {
@@ -485,7 +451,7 @@ object ServicesStoreSpec {
          |db.url = "jdbc:hsqldb:mem:$${uniqueSchema};shutdown=false;hsqldb.tx=mvcc"
          |db.driver = "org.hsqldb.jdbcDriver"
          |db.connectionTimeout = 3000
-         |driver = "slick.driver.HsqldbDriver$$"
+         |profile = "slick.jdbc.HsqldbProfile$$"
          |liquibase.updateSchema = false
          |""".stripMargin)
     val database = new SlickDatabase(databaseConfig)
@@ -502,9 +468,7 @@ object ServicesStoreSpec {
   (referenceProfile: ReferenceProfile,
    referenceDatabase: ReferenceProfile#Backend#Database,
    comparisonProfile: ComparisonProfile,
-   comparisonDatabase: ComparisonProfile#Backend#Database)(block: DiffResult => T)
-  (implicit executor: ExecutionContext): T = {
-
+   comparisonDatabase: ComparisonProfile#Backend#Database)(block: DiffResult => T): T = {
     withConnections(referenceProfile, referenceDatabase, comparisonProfile, comparisonDatabase) {
       LiquibaseUtils.compare(_, _)(block)
     }
@@ -640,4 +604,6 @@ object ServicesStoreSpec {
       tables.find(_.tableName == tableItem.tableName).exists(_.existsSlickMapping(tableItem.itemName))
     }
   }
+
+  private val WdlWorkflowType = Option("WDL")
 }
diff --git a/services/src/test/scala/cromwell/services/keyvalue/InMemoryKvServiceActor.scala b/services/src/test/scala/cromwell/services/keyvalue/InMemoryKvServiceActor.scala
new file mode 100644
index 000000000..96ebd30c3
--- /dev/null
+++ b/services/src/test/scala/cromwell/services/keyvalue/InMemoryKvServiceActor.scala
@@ -0,0 +1,21 @@
+package cromwell.services.keyvalue
+
+import cromwell.services.keyvalue.KeyValueServiceActor._
+
+import scala.concurrent.{ExecutionContextExecutor, Future}
+
+final class InMemoryKvServiceActor extends KeyValueServiceActor {
+  override implicit val ec: ExecutionContextExecutor = context.dispatcher
+
+  var kvStore = Map.empty[ScopedKey, Option[String]]
+
+  override def doGet(get: KvGet): Future[KvResponse] = kvStore.get(get.key).map(KvPair(get.key, _)) match {
+    case Some(kvPair) => Future.successful(kvPair)
+    case None => Future.successful(KvKeyLookupFailed(get))
+  }
+
+  override def doPut(put: KvPut): Future[KvResponse] = {
+    kvStore += (put.key -> put.pair.value)
+    Future.successful(KvPutSuccess(put))
+  }
+}
\ No newline at end of file
diff --git a/services/src/test/scala/cromwell/services/keyvalue/KvClientSpec.scala b/services/src/test/scala/cromwell/services/keyvalue/KvClientSpec.scala
new file mode 100644
index 000000000..2a38f46e1
--- /dev/null
+++ b/services/src/test/scala/cromwell/services/keyvalue/KvClientSpec.scala
@@ -0,0 +1,60 @@
+package cromwell.services.keyvalue
+
+import java.io.IOException
+
+import akka.actor.{Actor, ActorLogging, ActorRef, ActorSystem}
+import akka.testkit.{TestActorRef, TestKit, TestProbe}
+import cromwell.services.keyvalue.KeyValueServiceActor._
+import org.scalatest.{FlatSpecLike, Matchers}
+
+import scala.concurrent.Await
+import scala.concurrent.duration._
+import scala.language.postfixOps
+
+class KvClientSpec extends TestKit(ActorSystem("KvClientSpec")) with FlatSpecLike with Matchers {
+
+  implicit val ec = system.dispatcher
+
+  behavior of "KvClient"
+
+  it should "Correctly forward multiple requests and responses" in {
+    val serviceActorProbe = TestProbe()
+    val kvTestClient = TestActorRef(new KvTestClientActor(serviceActorProbe.ref))
+
+    val scopedKey1 = ScopedKey(null, null, "key1")
+    val scopedKey2 = ScopedKey(null, null, "key2")
+    val putRequest = KvPut(KvPair(scopedKey1, Some("value1")))
+    val getRequest = KvGet(scopedKey2)
+    val putResponse = KvFailure(putRequest, new IOException())
+    val getResponse = KvPair(scopedKey2, Some("value2"))
+
+    val requests = Seq(putRequest, getRequest)
+    val futureResult = kvTestClient.underlyingActor.makeKvRequest(requests)
+
+    serviceActorProbe.expectMsgAllOf(putRequest, getRequest)
+    serviceActorProbe.expectNoMsg(max = 50 milliseconds)
+
+    kvTestClient.underlyingActor.currentKvClientRequests.size should be(2)
+
+    kvTestClient.tell(getResponse, sender = serviceActorProbe.ref)
+    serviceActorProbe.expectNoMsg(max = 50 milliseconds)
+    futureResult.isCompleted should be(false)
+
+    kvTestClient.underlyingActor.currentKvClientRequests.size should be(1)
+
+    kvTestClient.tell(putResponse, sender = serviceActorProbe.ref)
+    serviceActorProbe.expectNoMsg(max = 50 milliseconds)
+
+    // Make sure the future completes promptly and the original order is preserved:
+    Await.result(futureResult, atMost = 100 milliseconds) should be(Seq(putResponse, getResponse))
+    serviceActorProbe.expectNoMsg(max = 50 milliseconds)
+
+    kvTestClient.underlyingActor.currentKvClientRequests.size should be(0)
+  }
+}
+
+class KvTestClientActor(val serviceRegistryActor: ActorRef) extends Actor with ActorLogging with KvClient {
+  override def receive = kvClientReceive orElse Actor.ignoringBehavior
+}
+
+
diff --git a/services/src/test/scala/cromwell/services/metadata/MetadataServiceSpec.scala b/services/src/test/scala/cromwell/services/metadata/MetadataServiceSpec.scala
index 0ef7d1976..a655234ff 100644
--- a/services/src/test/scala/cromwell/services/metadata/MetadataServiceSpec.scala
+++ b/services/src/test/scala/cromwell/services/metadata/MetadataServiceSpec.scala
@@ -1,11 +1,14 @@
 package cromwell.services.metadata
 
+import java.util.UUID
+
 import cromwell.core.WorkflowId
+import lenthall.exception.AggregatedException
 import org.scalactic.Equality
 import org.scalatest.prop.TableDrivenPropertyChecks
 import org.scalatest.{FlatSpec, Matchers}
-import wdl4s.types.{WdlArrayType, WdlMapType, WdlStringType}
-import wdl4s.values._
+import wdl4s.wdl.types.{WdlArrayType, WdlMapType, WdlStringType}
+import wdl4s.wdl.values._
 
 class MetadataServiceSpec extends FlatSpec with Matchers with TableDrivenPropertyChecks {
 
@@ -74,4 +77,135 @@ class MetadataServiceSpec extends FlatSpec with Matchers with TableDrivenPropert
     }
   }
 
+  // For the metadata event tests!
+  val failureMessageRegex = "([^\\[]*)\\[([0-9]+)\\](.*)\\:message".r
+  val pathToFailures = "path:to:failures"
+
+  it should "convert an exception into a failure event with an empty causedBy block" in {
+    import MetadataService.throwableToMetadataEvents
+
+    val workflowId = WorkflowId(UUID.randomUUID())
+    val mdkey = MetadataKey(workflowId, None, pathToFailures)
+
+    val tMsg = "The Oscars suck!"
+    val t = new RuntimeException(tMsg)
+
+    val events = throwableToMetadataEvents(mdkey, t)
+    events.size should be(2)
+    val (keyPrefix, causedBys, failureIndex) = validateExceptionMessage(events.head, workflowId, tMsg)
+    keyPrefix should be(pathToFailures)
+    causedBys should be("")
+    events(1).key.key should be(s"$keyPrefix[$failureIndex]:causedBy[]")
+    events(1).key.workflowId should be(workflowId)
+    events(1).value should be(None)
+
+  }
+
+  it should "convert nested exceptions into a sequence of failure events" in {
+    import MetadataService.throwableToMetadataEvents
+
+    val workflowId = WorkflowId(UUID.randomUUID())
+    val mdkey = MetadataKey(workflowId, None, pathToFailures)
+
+    val innerCauseMsg = "Envelope malfunctions"
+    val innerCause = new RuntimeException(innerCauseMsg)
+
+    val causeMsg = "Wrong recipients"
+    val cause = new RuntimeException(causeMsg, innerCause)
+
+    val tMsg = "The Oscars suck!"
+    val t = new RuntimeException(tMsg, cause)
+
+    val events = throwableToMetadataEvents(mdkey, t)
+    events.size should be(4)
+
+    val (outerPrefix, outerCausedBys, outerFailureId) = validateExceptionMessage(events.head, workflowId, tMsg)
+    val (cause1Prefix, cause1CausedBys, cause1FailureId) = validateExceptionMessage(events(1), workflowId, causeMsg)
+    val (cause2Prefix, cause2CausedBys, cause2FailureId) = validateExceptionMessage(events(2), workflowId, innerCauseMsg)
+    events(3).key.key should be(s"$cause2Prefix[$cause2FailureId]$cause2CausedBys:causedBy[]")
+
+    outerPrefix should be(pathToFailures)
+    cause1Prefix should be(pathToFailures)
+    cause2Prefix should be(pathToFailures)
+    outerCausedBys should be("")
+    cause1CausedBys should be(":causedBy[0]")
+    cause2CausedBys should be(":causedBy[0]:causedBy[0]")
+    cause1FailureId should be(outerFailureId)
+    cause2FailureId should be(cause1FailureId)
+  }
+
+  it should "convert aggregated exceptions into a sequence of failure events" in {
+    import MetadataService.throwableToMetadataEvents
+
+    val workflowId = WorkflowId(UUID.randomUUID())
+    val mdkey = MetadataKey(workflowId, None, "path:to:failures")
+
+    val innerCauseMsg = "Envelope malfunctions"
+    val innerCause = new RuntimeException(innerCauseMsg)
+
+    val cause1Msg = "Wrong recipients"
+    val cause1 = new RuntimeException(cause1Msg)
+    val cause2Msg = "Self congratulation"
+    val cause2 = new RuntimeException(cause2Msg, innerCause)
+    val cause3Msg = "The Globes are better anyway"
+    val cause3 = new RuntimeException(cause3Msg)
+
+    val causeContext = "Compound Entertainment Failure"
+    val cause = new AggregatedException(causeContext, List(cause1, cause2, cause3))
+
+    val tMsg = "The Oscars suck!"
+    val t = new RuntimeException(tMsg, cause)
+
+    val events = throwableToMetadataEvents(mdkey, t)
+    events.size should be(9)
+
+    // Outer runtime exception:
+    val (runtimePrefix, runtimeCausedBys, runtimeFailureId) = validateExceptionMessage(events.head, workflowId, tMsg)
+    runtimePrefix should be(pathToFailures)
+    runtimeCausedBys should be("")
+
+    // Aggregate exception:
+    val (aggregatePrefix, aggregateCausedBys, aggregateFailureId) = validateExceptionMessage(events(1), workflowId, causeContext)
+    aggregatePrefix should be(pathToFailures)
+    aggregateCausedBys should be(":causedBy[0]")
+    aggregateFailureId should be(runtimeFailureId)
+
+    // cause1, caused by []
+    val (cause1Prefix, cause1CausedBys, cause1FailureId) = validateExceptionMessage(events(2), workflowId, cause1Msg)
+    cause1Prefix should be(pathToFailures)
+    cause1CausedBys should be(":causedBy[0]:causedBy[0]")
+    cause1FailureId should be(runtimeFailureId)
+    events(3).key.key should be(s"$cause1Prefix[$runtimeFailureId]$cause1CausedBys:causedBy[]")
+
+    // cause2, caused by innerCause caused by []
+    val (cause2Prefix, cause2CausedBys, cause2FailureId) = validateExceptionMessage(events(4), workflowId, cause2Msg)
+    val (innerCausePrefix, innerCauseCausedBys, innerCauseFailureIds) = validateExceptionMessage(events(5), workflowId, innerCauseMsg)
+    cause2Prefix should be(pathToFailures)
+    cause2CausedBys should be(":causedBy[0]:causedBy[1]")
+    cause2FailureId should be(runtimeFailureId)
+    innerCausePrefix should be(pathToFailures)
+    innerCauseCausedBys should be(":causedBy[0]:causedBy[1]:causedBy[0]")
+    innerCauseFailureIds should be(runtimeFailureId)
+    events(6).key.key should be(s"$innerCausePrefix[$runtimeFailureId]$innerCauseCausedBys:causedBy[]")
+
+    // cause3, caused by []
+    val (cause3Prefix, cause3CausedBys, cause3FailureId) = validateExceptionMessage(events(7), workflowId, cause3Msg)
+    cause3Prefix should be(pathToFailures)
+    cause3CausedBys should be(":causedBy[0]:causedBy[2]")
+    cause3FailureId should be(runtimeFailureId)
+    events(8).key.key should be(s"$cause3Prefix[$cause3FailureId]$cause3CausedBys:causedBy[]")
+  }
+
+  def validateExceptionMessage(event: MetadataEvent, workflowId: WorkflowId, message: String) = event match {
+    case MetadataEvent(k, Some(MetadataValue(v, _)), _) =>
+      k.workflowId should be(workflowId)
+      v should be(message)
+
+      // Return the ID so that we can check for uniqueness later:
+      k.key match {
+        case failureMessageRegex(prefix, failureIndex, causedBys) => (prefix, causedBys, failureIndex)
+        case _ => fail("Unexpected failure key format: " + k.key)
+      }
+    case _ => fail("throwableToMetadataEvents generated a metadata event without a metadata value! Bad throwableToMetadataEvents! Very bad!")
+  }
 }
diff --git a/services/src/test/scala/cromwell/services/metadata/WorkflowQueryParametersSpec.scala b/services/src/test/scala/cromwell/services/metadata/WorkflowQueryParametersSpec.scala
index 161d5b198..3f503f075 100644
--- a/services/src/test/scala/cromwell/services/metadata/WorkflowQueryParametersSpec.scala
+++ b/services/src/test/scala/cromwell/services/metadata/WorkflowQueryParametersSpec.scala
@@ -3,6 +3,7 @@ package cromwell.services.metadata
 import java.time.OffsetDateTime
 
 import cats.data.Validated._
+import cromwell.core.labels.Label
 import cromwell.services.metadata.WorkflowQueryKey._
 import org.scalatest.{Matchers, WordSpec}
 
@@ -21,6 +22,7 @@ class WorkflowQueryParametersSpec extends WordSpec with Matchers {
           r.endDate should be('empty)
           r.names should be('empty)
           r.statuses should be('empty)
+          r.labels should be ('empty)
         case Invalid(fs) =>
           throw new RuntimeException(fs.toList.mkString(", "))
       }
@@ -32,6 +34,7 @@ class WorkflowQueryParametersSpec extends WordSpec with Matchers {
         Status.name -> "Succeeded",
         Name.name -> "my_other_workflow",
         Status.name -> "Running",
+        LabelKeyValue.name -> "label-key:label-value",
         StartDate.name -> StartDateString,
         EndDate.name -> EndDateString
       )
@@ -42,6 +45,7 @@ class WorkflowQueryParametersSpec extends WordSpec with Matchers {
           r.endDate.get.toInstant should equal(OffsetDateTime.parse(EndDateString).toInstant)
           r.names should be(Set("my_workflow", "my_other_workflow"))
           r.statuses should be(Set("Succeeded", "Running"))
+          r.labels should be(Set(Label("label-key", "label-value")))
         case Invalid(fs) =>
           throw new RuntimeException(fs.toList.mkString(", "))
       }
@@ -142,6 +146,38 @@ class WorkflowQueryParametersSpec extends WordSpec with Matchers {
       }
     }
 
+    "reject labels with invalid format" in {
+      val badLabelKey = "0-label-key"
+      val rawParameters = Seq(
+        LabelKeyValue.name -> "label-key:label-value",
+        LabelKeyValue.name -> s"$badLabelKey:label-value"
+      )
+      val result = WorkflowQueryParameters.runValidation(rawParameters)
+      result match {
+        case Valid(r) =>
+          throw new RuntimeException(s"Unexpected success: $r")
+        case Invalid(fs) =>
+          fs.toList should have size 1
+          fs.toList.head should include(s"Invalid label: `$badLabelKey` did not match the regex ${Label.LabelKeyRegex}")
+      }
+    }
+
+    "reject bad label syntax" in {
+      val badLabelSyntax = "label-keyLabel-value"
+      val rawParameters = Seq(
+        LabelKeyValue.name -> "label-key:label-value",
+        LabelKeyValue.name -> badLabelSyntax
+      )
+      val result = WorkflowQueryParameters.runValidation(rawParameters)
+      result match {
+        case Valid(r) =>
+          throw new RuntimeException(s"Unexpected success: $r")
+        case Invalid(fs) =>
+          fs.toList should have size 1
+          fs.toList.head should include("Label values do not match allowed pattern label-key:label-value")
+      }
+    }
+
     "collect multiple errors for a hot mess of a query" in {
       val rawParameters = Seq(
         Status.name -> "Moseying",
diff --git a/services/src/test/scala/cromwell/services/metadata/impl/MetadataDatabaseAccessSpec.scala b/services/src/test/scala/cromwell/services/metadata/impl/MetadataDatabaseAccessSpec.scala
index f9f555abf..85e077652 100644
--- a/services/src/test/scala/cromwell/services/metadata/impl/MetadataDatabaseAccessSpec.scala
+++ b/services/src/test/scala/cromwell/services/metadata/impl/MetadataDatabaseAccessSpec.scala
@@ -5,6 +5,7 @@ import java.time.OffsetDateTime
 import com.typesafe.config.ConfigFactory
 import cromwell.core.Tags.DbmsTest
 import cromwell.core._
+import cromwell.core.labels.Label
 import cromwell.database.slick.SlickDatabase
 import cromwell.services.ServicesStore
 import cromwell.services.metadata._
@@ -50,14 +51,18 @@ class MetadataDatabaseAccessSpec extends FlatSpec with Matchers with ScalaFuture
       dataAccess.addMetadataEvents(events)
     }
 
-    def baseWorkflowMetadata(name: String): Future[WorkflowId] = {
+    def baseWorkflowMetadata(name: String, labels: Set[Label] = Set.empty): Future[WorkflowId] = {
       val workflowId = WorkflowId.randomId()
+      val defaultLabels = Set(Label("cromwell-workflow-name", name))
+      val labelMetadata = (labels ++ defaultLabels).map(label => (s"${WorkflowMetadataKeys.Labels}:${label.key}", label.value)).toArray
+
       val workflowKey = MetadataKey(workflowId, jobKey = None, key = null)
       def keyAndValue(name: String) = Array(
         (WorkflowMetadataKeys.StartTime, OffsetDateTime.now.toString),
         (WorkflowMetadataKeys.Status, WorkflowSubmitted.toString),
         (WorkflowMetadataKeys.Name, name)
-      )
+      ) ++ labelMetadata
+
       publishMetadataEvents(workflowKey, keyAndValue(name)).map(_ => workflowId)
     }
 
@@ -66,27 +71,57 @@ class MetadataDatabaseAccessSpec extends FlatSpec with Matchers with ScalaFuture
         workflow1Id <- baseWorkflowMetadata(Workflow1Name)
         //get metadata when page and pagesize are specified
         _ <- dataAccess.queryWorkflowSummaries(WorkflowQueryParameters(Seq(
-          WorkflowQueryKey.Page.name -> "1", WorkflowQueryKey.PageSize.name -> "50"))) map { case (response, meta) =>
+          WorkflowQueryKey.Page.name -> "1", WorkflowQueryKey.PageSize.name -> "50"))) map { case (response @ _, meta) =>
           meta match {
-            case Some(metadata) =>
+            case Some(_) =>
             case None => fail("Should have metadata when page and pagesize are specified.")
           }
         }
         //don't get metadata when page and pagesize are not specified
         _ <- dataAccess.queryWorkflowSummaries(
-          WorkflowQueryParameters(Seq())) map { case(response, meta) =>
+          WorkflowQueryParameters(Seq())) map { case(response @ _, meta) =>
           meta match {
-            case Some(metadata) => fail("Should not have metadata when page and pagesize are not specified")
+            case Some(_) => fail("Should not have metadata when page and pagesize are not specified")
             case None =>
           }
         }
       } yield()).futureValue
     }
+    
+    it should "sort metadata events by timestamp from older to newer" taggedAs DbmsTest in {
+      def unorderedEvents(id: WorkflowId): Future[Vector[MetadataEvent]] = {
+        val workflowKey = MetadataKey(id, jobKey = None, key = null)
+        val now = OffsetDateTime.now()
+        val yesterday = now.minusDays(1)
+        val tomorrow = now.plusDays(1)
+
+        val yesterdayEvent = MetadataEvent(workflowKey.copy(key = WorkflowMetadataKeys.WorkflowRoot), Option(MetadataValue("A")), yesterday)
+        val nowEvent = MetadataEvent(workflowKey.copy(key = WorkflowMetadataKeys.WorkflowRoot), Option(MetadataValue("B")), now)
+        val tomorrowEvent = MetadataEvent(workflowKey.copy(key = WorkflowMetadataKeys.WorkflowRoot), Option(MetadataValue("C")), tomorrow)
+        
+        val events = Vector(tomorrowEvent, yesterdayEvent, nowEvent)
+        
+        val expectedEvents = Vector(yesterdayEvent, nowEvent, tomorrowEvent)
+        
+        dataAccess.addMetadataEvents(events) map { _ => expectedEvents }
+      }
+      
+      (for {
+        workflow1Id <- baseWorkflowMetadata(Workflow1Name)
+        expected <- unorderedEvents(workflow1Id)
+        response <- dataAccess.queryMetadataEvents(MetadataQuery(workflow1Id, None, Option(WorkflowMetadataKeys.WorkflowRoot), None, None, expandSubWorkflows = false))
+        _ = response shouldBe expected
+      } yield()).futureValue
+    }
 
     it should "create and query a workflow" taggedAs DbmsTest in {
 
       val randomIds = Seq.fill(10)(WorkflowId.randomId().toString)
 
+      val testLabel1 = Label("testing-key-1", "testing-value-1")
+      val testLabel2 = Label("testing-key-2", "testing-value-2")
+      val testLabel3 = Label("testing-key-3", "testing-value-3")
+
       def succeededWorkflowMetadata(id: WorkflowId): Future[Unit] = {
         val workflowKey = MetadataKey(id, jobKey = None, key = null)
         val keyAndValue = Array(
@@ -98,11 +133,11 @@ class MetadataDatabaseAccessSpec extends FlatSpec with Matchers with ScalaFuture
       }
 
       (for {
-        workflow1Id <- baseWorkflowMetadata(Workflow1Name)
+        workflow1Id <- baseWorkflowMetadata(Workflow1Name, Set(testLabel1, testLabel2))
         _ <- succeededWorkflowMetadata(workflow1Id)
         // Put a bit of space between the two workflows
         _ = Thread.sleep(50)
-        workflow2Id <- baseWorkflowMetadata(Workflow2Name)
+        workflow2Id <- baseWorkflowMetadata(Workflow2Name, Set(testLabel2, testLabel3))
 
         // refresh the metadata
         _ <- dataAccess.refreshWorkflowMetadataSummaries() map { max =>
@@ -111,7 +146,7 @@ class MetadataDatabaseAccessSpec extends FlatSpec with Matchers with ScalaFuture
 
         // Query with no filters
         (workflowQueryResult, workflowQueryResult2) <-
-        dataAccess.queryWorkflowSummaries(WorkflowQueryParameters(Seq.empty)) map { case (response, meta) =>
+        dataAccess.queryWorkflowSummaries(WorkflowQueryParameters(Seq.empty)) map { case (response, meta @ _) =>
           val result = response.results find { r => r.name.contains(Workflow1Name) && r.end.isDefined } getOrElse
             fail(s"$Workflow1Name with an end not found in ${response.results}")
           val result2 = response.results find {
@@ -120,14 +155,14 @@ class MetadataDatabaseAccessSpec extends FlatSpec with Matchers with ScalaFuture
           (result, result2)
         }
         // Filter by name
-        _ <- dataAccess.queryWorkflowSummaries(WorkflowQueryParameters(Seq(WorkflowQueryKey.Name.name -> Workflow1Name))) map { case (response, meta) =>
+        _ <- dataAccess.queryWorkflowSummaries(WorkflowQueryParameters(Seq(WorkflowQueryKey.Name.name -> Workflow1Name))) map { case (response, meta @ _) =>
           val resultsByName = response.results groupBy {
             _.name
           }
           resultsByName.keys.toSet.flatten should equal(Set(Workflow1Name))
         }
         // Filter by multiple names
-        _ <- dataAccess.queryWorkflowSummaries(WorkflowQueryParameters(Seq(WorkflowQueryKey.Name.name -> Workflow1Name, WorkflowQueryKey.Name.name -> Workflow2Name))) map { case (response, meta) =>
+        _ <- dataAccess.queryWorkflowSummaries(WorkflowQueryParameters(Seq(WorkflowQueryKey.Name.name -> Workflow1Name, WorkflowQueryKey.Name.name -> Workflow2Name))) map { case (response, meta @ _) =>
           val resultsByName = response.results groupBy {
             _.name
           }
@@ -135,7 +170,7 @@ class MetadataDatabaseAccessSpec extends FlatSpec with Matchers with ScalaFuture
         }
         // Filter by workflow id
         _ <- dataAccess.queryWorkflowSummaries(WorkflowQueryParameters(
-          Seq(WorkflowQueryKey.Id.name -> workflow1Id.toString))) map { case (response, meta) =>
+          Seq(WorkflowQueryKey.Id.name -> workflow1Id.toString))) map { case (response, meta @ _) =>
           val resultsById = response.results groupBy {
             _.name
           }
@@ -143,7 +178,7 @@ class MetadataDatabaseAccessSpec extends FlatSpec with Matchers with ScalaFuture
         }
         // Filter by multiple workflow ids
         _ <- dataAccess.queryWorkflowSummaries(WorkflowQueryParameters(
-          Seq(workflow1Id, workflow2Id).map(id => WorkflowQueryKey.Id.name -> id.toString))) map { case (response, meta) =>
+          Seq(workflow1Id, workflow2Id).map(id => WorkflowQueryKey.Id.name -> id.toString))) map { case (response, meta @ _) =>
           val resultsById = response.results groupBy {
             _.name
           }
@@ -151,25 +186,36 @@ class MetadataDatabaseAccessSpec extends FlatSpec with Matchers with ScalaFuture
         }
         // Filter by workflow id within random Ids
         _ <- dataAccess.queryWorkflowSummaries(WorkflowQueryParameters(
-          (randomIds :+ workflow1Id).map(id => WorkflowQueryKey.Id.name -> id.toString))) map { case (response, meta) =>
+          (randomIds :+ workflow1Id).map(id => WorkflowQueryKey.Id.name -> id.toString))) map { case (response, meta @ _) =>
           val resultsById = response.results groupBy {
             _.name
           }
           resultsById.keys.toSet.flatten should equal(Set(Workflow1Name))
         }
         // Filter by status
-        _ <- dataAccess.queryWorkflowSummaries(WorkflowQueryParameters(Seq(WorkflowQueryKey.Status.name -> "Submitted"))) map { case (response, meta) =>
+        _ <- dataAccess.queryWorkflowSummaries(WorkflowQueryParameters(Seq(WorkflowQueryKey.Status.name -> "Submitted"))) map { case (response, meta @ _) =>
           val resultsByStatus = response.results groupBy (_.status)
           resultsByStatus.keys.toSet.flatten should equal(Set("Submitted"))
         }
         // Filter by multiple statuses
-        _ <- dataAccess.queryWorkflowSummaries(WorkflowQueryParameters(Seq(WorkflowQueryKey.Status.name -> "Submitted", WorkflowQueryKey.Status.name -> "Succeeded"))) map { case (response, meta) =>
+        _ <- dataAccess.queryWorkflowSummaries(WorkflowQueryParameters(Seq(WorkflowQueryKey.Status.name -> "Submitted", WorkflowQueryKey.Status.name -> "Succeeded"))) map { case (response, meta @ _) =>
           val resultsByStatus = response.results groupBy (_.status)
           resultsByStatus.keys.toSet.flatten should equal(Set("Submitted", "Succeeded"))
         }
+        // Filter by label
+        _ <- dataAccess.queryWorkflowSummaries(WorkflowQueryParameters(Seq(WorkflowQueryKey.LabelKeyValue.name -> s"${testLabel2.key}:${testLabel2.value}"))) map { case (response, meta @ _) =>
+          val resultByName = response.results groupBy (_.name)
+          resultByName.keys.toSet.flatten should equal(Set(Workflow1Name, Workflow2Name))
+        }
+        // Filter by multiple labels
+        _ <- dataAccess.queryWorkflowSummaries(WorkflowQueryParameters(
+          Seq(testLabel2, testLabel3).map(label => WorkflowQueryKey.LabelKeyValue.name -> s"${label.key}:${label.value}"))) map { case (response, meta @ _) =>
+          val resultByName = response.results groupBy (_.name)
+          resultByName.keys.toSet.flatten should equal(Set(Workflow2Name))
+        }
         // Filter by start date
         _ <- dataAccess.queryWorkflowSummaries(WorkflowQueryParameters(Seq(
-          WorkflowQueryKey.StartDate.name -> workflowQueryResult2.start.get.toString))) map { case (response, meta) =>
+          WorkflowQueryKey.StartDate.name -> workflowQueryResult2.start.get.toString))) map { case (response, meta @ _) =>
           response.results partition { r => r.start.isDefined && r.start.get.compareTo(workflowQueryResult.start.get) >= 0 } match {
             case (y, n) if y.nonEmpty && n.isEmpty => // good
             case (y, n) => fail(s"Found ${y.size} later workflows and ${n.size} earlier")
@@ -177,7 +223,7 @@ class MetadataDatabaseAccessSpec extends FlatSpec with Matchers with ScalaFuture
         }
         // Filter by end date
         _ <- dataAccess.queryWorkflowSummaries(WorkflowQueryParameters(Seq(
-          WorkflowQueryKey.EndDate.name -> workflowQueryResult.end.get.toString))) map { case (response, meta) =>
+          WorkflowQueryKey.EndDate.name -> workflowQueryResult.end.get.toString))) map { case (response, meta @ _) =>
           response.results partition { r => r.end.isDefined && r.end.get.compareTo(workflowQueryResult.end.get) <= 0 } match {
             case (y, n) if y.nonEmpty && n.isEmpty => // good
             case (y, n) => fail(s"Found ${y.size} earlier workflows and ${n.size} later")
diff --git a/services/src/test/scala/cromwell/services/metadata/impl/MetadataServiceActorSpec.scala b/services/src/test/scala/cromwell/services/metadata/impl/MetadataServiceActorSpec.scala
index a08dfb9fa..a14dab597 100644
--- a/services/src/test/scala/cromwell/services/metadata/impl/MetadataServiceActorSpec.scala
+++ b/services/src/test/scala/cromwell/services/metadata/impl/MetadataServiceActorSpec.scala
@@ -41,6 +41,7 @@ class MetadataServiceActorSpec extends ServicesSpec("Metadata") {
       val putAction1 = PutMetadataAction(event1_1)
       val putAction2 = PutMetadataAction(event1_2)
       val putAction3 = PutMetadataAction(event2_1, event3_1, event3_2)
+
       actor ! putAction1
       actor ! putAction2
       actor ! putAction3
diff --git a/services/src/test/scala/cromwell/services/metadata/impl/WriteMetadataActorSpec.scala b/services/src/test/scala/cromwell/services/metadata/impl/WriteMetadataActorSpec.scala
index 577013aef..3b16acd72 100644
--- a/services/src/test/scala/cromwell/services/metadata/impl/WriteMetadataActorSpec.scala
+++ b/services/src/test/scala/cromwell/services/metadata/impl/WriteMetadataActorSpec.scala
@@ -5,35 +5,67 @@ import java.time.OffsetDateTime
 import akka.testkit.TestFSMRef
 import cats.data.NonEmptyVector
 import cromwell.core.WorkflowId
+import cromwell.core.actor.BatchingDbWriter
+import cromwell.core.actor.BatchingDbWriter._
 import cromwell.services.ServicesSpec
 import cromwell.services.metadata.MetadataService.PutMetadataAction
 import cromwell.services.metadata.{MetadataEvent, MetadataKey, MetadataValue}
-import cromwell.services.metadata.impl.WriteMetadataActor.{HasEvents, NoEvents, WaitingToWrite, WritingToDb}
+import org.scalatest.BeforeAndAfter
+import org.scalatest.concurrent.Eventually
 
 import scala.concurrent.duration._
+import scala.concurrent.{ExecutionContext, Future, Promise}
+import scala.util.Success
 
-class WriteMetadataActorSpec extends ServicesSpec("Metadata") {
+class WriteMetadataActorSpec extends ServicesSpec("Metadata") with Eventually with BeforeAndAfter {
   import WriteMetadataActorSpec.Action
 
-  val actor = TestFSMRef(WriteMetadataActor(10, 1.days)) // A WMA that won't (hopefully!) perform a time based flush during this test
+  var actor: TestFSMRef[BatchingDbWriterState, BatchingDbWriter.BatchingDbWriterData, DelayingWriteMetadataActor] = _
+
+  before {
+    actor = TestFSMRef(new DelayingWriteMetadataActor())
+  }
 
   "WriteMetadataActor" should {
     "start with no events and waiting to write" in {
-      assert(actor.stateName == WaitingToWrite)
-      assert(actor.stateData == NoEvents)
+      actor.stateName shouldBe WaitingToWrite
+      actor.stateData shouldBe NoData
     }
 
     "Have one event and be waiting after one event is sent" in {
       actor ! Action
-      assert(actor.stateName == WaitingToWrite)
-      assert(actor.stateData == HasEvents(NonEmptyVector.fromVectorUnsafe(Action.events.toVector)))
+      eventually {
+        actor.stateName shouldBe WaitingToWrite
+        actor.stateData shouldBe HasData(NonEmptyVector.fromVectorUnsafe(Action.events.toVector))
+      }
     }
 
     "Have one event after batch size + 1 is reached" in {
-      1 to 10 foreach { _ => actor ! Action }
-      assert(actor.stateName == WritingToDb)
+      1 to WriteMetadataActorSpec.BatchRate foreach { _ => actor ! Action }
+      actor.stateName shouldBe WaitingToWrite
+
+      eventually {
+        actor.stateData match {
+          case HasData(e) => e.toVector.size shouldBe WriteMetadataActorSpec.BatchRate
+          case _ => fail("Expecting the actor to have events queued up")
+        }
+      }
+      actor ! Action
+      eventually {
+        actor.stateName shouldBe WritingToDb
+        actor.underlyingActor.writeToDbInProgress shouldBe true
+        actor.stateData shouldBe NoData
+      }
       actor ! Action
-      assert(actor.stateData == HasEvents(NonEmptyVector.fromVectorUnsafe(Action.events.toVector)))
+      eventually {
+        actor.stateName shouldBe WritingToDb
+        actor.stateData shouldBe HasData(NonEmptyVector.fromVectorUnsafe(Action.events.toVector))
+      }
+      actor.underlyingActor.completeWritePromise()
+      eventually {
+        actor.stateName shouldBe WaitingToWrite
+        actor.stateData shouldBe HasData(NonEmptyVector.fromVectorUnsafe(Action.events.toVector))
+      }
     }
   }
 }
@@ -41,4 +73,30 @@ class WriteMetadataActorSpec extends ServicesSpec("Metadata") {
 object WriteMetadataActorSpec {
   val Event = MetadataEvent(MetadataKey(WorkflowId.randomId(), None, "key"), Option(MetadataValue("value")), OffsetDateTime.now)
   val Action = PutMetadataAction(Event)
+
+  val BatchRate: Int = 10
+  val FunctionallyForever: FiniteDuration = 100.days
+}
+
+// A WMA that won't (hopefully!) perform a time based flush during this test
+final class DelayingWriteMetadataActor extends WriteMetadataActor(WriteMetadataActorSpec.BatchRate, WriteMetadataActorSpec.FunctionallyForever) {
+
+  var writeToDbInProgress: Boolean = false
+  var writeToDbCompletionPromise: Option[Promise[Unit]] = None
+
+  override def addMetadataEvents(metadataEvents: Iterable[MetadataEvent])(implicit ec: ExecutionContext): Future[Unit] = {
+    writeToDbCompletionPromise = Option(Promise[Unit]())
+    writeToDbInProgress = true
+    writeToDbCompletionPromise.get.future
+  }
+
+  def completeWritePromise(): Unit = {
+    writeToDbCompletionPromise match {
+      case Some(promise) =>
+        promise.complete(Success(()))
+        writeToDbInProgress = false
+        writeToDbCompletionPromise = None
+      case None => throw new Exception("BAD TEST! Cannot complete the actor's write future if the actor hasn't requested it yet!")
+    }
+  }
 }
diff --git a/src/bin/travis/afterSuccess.sh b/src/bin/travis/afterSuccess.sh
index 4fac80d74..09e5be1ce 100755
--- a/src/bin/travis/afterSuccess.sh
+++ b/src/bin/travis/afterSuccess.sh
@@ -9,10 +9,11 @@ echo "TRAVIS_PULL_REQUEST='$TRAVIS_PULL_REQUEST'"
 if [ "$BUILD_TYPE" == "sbt" ] && [ "$TRAVIS_PULL_REQUEST" == "false" ]; then
 
     if [ "$TRAVIS_BRANCH" == "develop" ]; then
+        # Publish images for both the "cromwell develop branch" and the "cromwell dev environment".
         docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD"
         sbt \
           'set test in Test := {}' \
-          'set imageNames in docker := Seq(ImageName("broadinstitute/cromwell:'${TRAVIS_BRANCH}'"))' \
+          'set imageNames in docker := Seq("develop", "dev").map(tag => ImageName(s"broadinstitute/cromwell:$tag"))' \
           publish \
           dockerBuildAndPush
 
diff --git a/src/bin/travis/resources/centaur.inputs b/src/bin/travis/resources/centaur.inputs
index 544aa8524..67822dbd2 100644
--- a/src/bin/travis/resources/centaur.inputs
+++ b/src/bin/travis/resources/centaur.inputs
@@ -3,5 +3,6 @@
   "centaur_workflow.centaur.centaur_branch":"CENTAUR_BRANCH",
   "centaur_workflow.centaur.conf":"gs://cloud-cromwell-dev/travis-centaur/multiBackend.conf",
   "centaur_workflow.centaur.pem":"gs://cloud-cromwell-dev/travis-centaur/cromwell-account.pem",
+  "centaur_workflow.centaur.integration_tests_dir": "INTEGRATION_TESTS_DIR",
   "centaur_workflow.centaur.token": "gs://cloud-cromwell-dev/travis-centaur/token.txt"
 }
diff --git a/src/bin/travis/resources/centaur.wdl b/src/bin/travis/resources/centaur.wdl
index 8795c756b..c385d23b6 100644
--- a/src/bin/travis/resources/centaur.wdl
+++ b/src/bin/travis/resources/centaur.wdl
@@ -4,16 +4,36 @@ task centaur {
     File pem
     File cromwell_jar
     File token
-    String secret = read_string(token)
+    String? integration_tests_dir
 
     command<<<
+        # start mysql server
+        mysqld &
+        
+        # give time to server to start
+        git clone https://github.com/vishnubob/wait-for-it.git
+        cd wait-for-it
+        ./wait-for-it.sh -t 10 localhost:3306
+        if (($? != 0)); then
+            echo "Timed out waiting for mysql server. Exiting."
+            exit 1
+        fi
+        
+        cd ..
+        
+        # setup mysql
+        mysql -u root -e "SET GLOBAL sql_mode = 'STRICT_ALL_TABLES';"
+        mysql -u root -e "CREATE DATABASE IF NOT EXISTS cromwell_test;"
+        mysql -u root -e "CREATE USER 'travis'@'localhost' IDENTIFIED BY '';"
+        mysql -u root -e "GRANT ALL PRIVILEGES ON cromwell_test . * TO 'travis'@'localhost';"
+        
         mkdir -p /cromwell_root/tmp/ivy2
         export SBT_OPTS=-Dsbt.ivy.home=/cromwell_root/tmp/.ivy2
         git clone https://github.com/broadinstitute/centaur.git
         cd centaur
         git checkout ${centaur_branch}
         cd ..
-        centaur/test_cromwell.sh -j${cromwell_jar} -c${conf} -r/cromwell_root -t${secret} -elocaldockertest -p100
+        centaur/test_cromwell.sh -j${cromwell_jar} -c${conf} -r/cromwell_root -t${token} -elocaldockertest -p100 ${ "-i" + integration_tests_dir}
     >>>
 
     output {
@@ -22,7 +42,7 @@ task centaur {
     }
 
     runtime {
-        docker: "geoffjentry/centaur-cromwell:latest" 
+        docker: "us.gcr.io/broad-dsde-cromwell-dev/centaur:latest" 
         cpu: "8"
         zones: "us-central1-b"
         failOnStderr: false
diff --git a/src/bin/travis/resources/cromwell-service-account.json.ctmpl b/src/bin/travis/resources/cromwell-service-account.json.ctmpl
new file mode 100644
index 000000000..2cbdcea92
--- /dev/null
+++ b/src/bin/travis/resources/cromwell-service-account.json.ctmpl
@@ -0,0 +1,5 @@
+{{with $cromwellServiceAccount := vault (printf "secret/dsde/cromwell/common/cromwell-service-account.json")}}
+
+{{$cromwellServiceAccount.Data | toJSONPretty}}
+
+{{end}}
diff --git a/src/bin/travis/resources/tes.conf b/src/bin/travis/resources/funnel.conf
similarity index 67%
rename from src/bin/travis/resources/tes.conf
rename to src/bin/travis/resources/funnel.conf
index c32460328..08dbab623 100644
--- a/src/bin/travis/resources/tes.conf
+++ b/src/bin/travis/resources/funnel.conf
@@ -1,5 +1,4 @@
 HttpPort: 9000
-LogLevel: info
 Storage:
   - Local:
       AllowedDirs:
@@ -7,9 +6,5 @@ Storage:
         - /cromwell-executions
         - /tmp/
 DBPath: /tmp/tes_task.db
-Schedulers:
-  Local:
-    NumWorkers: 4
-Worker:
-  LogLevel: info
-  Timeout: 1
+Scheduler: local
+LogLevel: info
diff --git a/src/bin/travis/resources/jesConf.tar.enc b/src/bin/travis/resources/jesConf.tar.enc
deleted file mode 100644
index 85ec60f88..000000000
Binary files a/src/bin/travis/resources/jesConf.tar.enc and /dev/null differ
diff --git a/src/bin/travis/resources/jes_centaur.conf.ctmpl b/src/bin/travis/resources/jes_centaur.conf.ctmpl
new file mode 100644
index 000000000..0324162a7
--- /dev/null
+++ b/src/bin/travis/resources/jes_centaur.conf.ctmpl
@@ -0,0 +1,71 @@
+{{with $refreshToken := vault (printf "secret/dsde/cromwell/common/refresh-token")}}
+
+include "application.conf"
+
+backend {
+  default = "Jes"
+  enabled = ["Jes", "Jes-Refresh"]
+  providers {
+    Local.config.filesystems.gcs.auth = "service_account"
+    Jes {
+      actor-factory = "cromwell.backend.impl.jes.JesBackendLifecycleActorFactory"
+      config {
+        project = "broad-dsde-cromwell-dev"
+        root = "gs://cloud-cromwell-dev/cromwell_execution/travis"
+        maximum-polling-interval = 600
+        genomics {
+          auth = "service_account"
+          endpoint-url = "https://genomics.googleapis.com/"
+        }
+        filesystems {
+          gcs.auth = "service_account"
+        }
+      }
+    }
+    Jes-Refresh {
+      actor-factory = "cromwell.backend.impl.jes.JesBackendLifecycleActorFactory"
+      config {
+        project = "broad-dsde-cromwell-dev"
+        root = "gs://centaur-refresh-private/cromwell_execution/travis"
+        maximum-polling-interval = 600
+        genomics {
+          auth = "service_account"
+          endpoint-url = "https://genomics.googleapis.com/"
+        }
+        filesystems {
+          gcs.auth = "refresh_token"
+        }
+      }
+    }
+  }
+}
+
+google {
+  application-name = "cromwell"
+  auths = [
+    {
+      name = "service_account"
+      scheme = "service_account"
+      json-file = "cromwell-service-account.json"
+    }
+    {
+      name = "refresh_token"
+      scheme = "refresh_token"
+      client-id = "{{$refreshToken.Data.client_id}}"
+      client-secret = "{{$refreshToken.Data.client_secret}}"
+    }
+  ]
+}
+
+call-caching {
+  enabled = true
+  lookup-docker-hash = false
+}
+
+engine {
+  filesystems {
+    gcs.auth = "service_account"
+  }
+}
+
+{{end}}
diff --git a/src/bin/travis/resources/local_centaur.conf b/src/bin/travis/resources/local_centaur.conf
index 848780e82..86dcce84b 100644
--- a/src/bin/travis/resources/local_centaur.conf
+++ b/src/bin/travis/resources/local_centaur.conf
@@ -1,3 +1,5 @@
+include "application.conf"
+
 akka {
   loggers = ["akka.event.slf4j.Slf4jLogger"]
   logging-filter = "akka.event.slf4j.Slf4jLoggingFilter"
@@ -17,6 +19,52 @@ call-caching {
   enabled = true
 }
 
+system.graceful-server-shutdown = true
+
+backend.providers {
+  LocalNoDocker {
+    actor-factory = "cromwell.backend.impl.sfs.config.ConfigBackendLifecycleActorFactory"
+    config {
+      run-in-background = true
+      runtime-attributes = ""
+      submit = "/bin/bash ${script}"
+      root: "cromwell-executions"
+
+      filesystems {
+        local {
+          localization: [
+            "soft-link", "hard-link", "copy"
+          ]
+
+          caching {
+            duplication-strategy: [
+              "soft-link"
+            ]
+
+            # Possible values: file, path
+            # "file" will compute an md5 hash of the file content.
+            # "path" will compute an md5 hash of the file path. This strategy will only be effective if the duplication-strategy (above) is set to "soft-link",
+            # in order to allow for the original file path to be hashed.
+            hashing-strategy: "path"
+
+            # When true, will check if a sibling file with the same name and the .md5 extension exists, and if it does, use the content of this file as a hash.
+            # If false or the md5 does not exist, will proceed with the above-defined hashing strategy.
+            check-sibling-md5: false
+          }
+        }
+      }
+    }
+  }
+}
+
+database {
+  db.url = "jdbc:mysql://localhost/cromwell_test?rewriteBatchedStatements=true"
+  db.user = "travis"
+  db.password = ""
+  db.driver = "com.mysql.jdbc.Driver"
+  profile = "slick.jdbc.MySQLProfile$"
+}
+
 backend.providers.Local.config.filesystems.local.caching.duplication-strategy = ["copy"]
 backend.providers.Local.config.filesystems.local.localization = ["soft-link", "copy"]
 backend.providers.Local.config.concurrent-job-limit = 20
diff --git a/src/bin/travis/resources/tes_centaur.conf b/src/bin/travis/resources/tes_centaur.conf
index 4254c1cb4..fe7420017 100644
--- a/src/bin/travis/resources/tes_centaur.conf
+++ b/src/bin/travis/resources/tes_centaur.conf
@@ -1,3 +1,5 @@
+include "application.conf"
+
 akka {
   loggers = ["akka.event.slf4j.Slf4jLogger"]
   logging-filter = "akka.event.slf4j.Slf4jLoggingFilter"
@@ -17,6 +19,8 @@ call-caching {
   enabled = true
 }
 
+system.graceful-server-shutdown = true
+
 backend {
   default = "TES"
   providers {
@@ -25,8 +29,16 @@ backend {
       config {
         root = "cromwell-executions"
         dockerRoot = "/cromwell-executions"
-        endpoint = "http://127.0.0.1:9000/v1/jobs"
+        endpoint = "http://127.0.0.1:9000/v1/tasks"
       }
     }
   }
 }
+
+database {
+  db.url = "jdbc:mysql://localhost/cromwell_test?rewriteBatchedStatements=true"
+  db.user = "travis"
+  db.password = ""
+  db.driver = "com.mysql.jdbc.Driver"
+  profile = "slick.jdbc.MySQLProfile$"
+}
diff --git a/src/bin/travis/test.sh b/src/bin/travis/test.sh
index 0daa3001a..0cabb3e50 100755
--- a/src/bin/travis/test.sh
+++ b/src/bin/travis/test.sh
@@ -4,18 +4,48 @@ set -e
 
 SCRIPT_DIR=src/bin/travis
 
-# BUILD_TYPE is coming in from the Travis build matrix
-if [ "$BUILD_TYPE" = "centaurJes" ]; then
-    "${SCRIPT_DIR}"/testCentaurJes.sh
-elif [ "$BUILD_TYPE" = "centaurTes" ]; then
-    "${SCRIPT_DIR}"/testCentaurTes.sh
-elif [ "$BUILD_TYPE" = "centaurLocal" ]; then
-    "${SCRIPT_DIR}"/testCentaurLocal.sh
-elif [ "$BUILD_TYPE" = "sbt" ]; then
-    "${SCRIPT_DIR}"/testSbt.sh
-elif [ "$BUILD_TYPE" = "checkPublish" ]; then
-    "${SCRIPT_DIR}"/testCheckPublish.sh
-else
-    echo "Unknown BUILD_TYPE: '$BUILD_TYPE'"
-    exit 1
-fi
+# $TRAVIS_EVENT_TYPE will be cron if this build was initiated by a cron job
+case "$TRAVIS_EVENT_TYPE" in
+    push|pull_request|api)
+    # BUILD_TYPE is coming in from the Travis build matrix
+        case "$BUILD_TYPE" in
+        centaurJes)
+            "${SCRIPT_DIR}"/testCentaurJes.sh
+            ;;
+        centaurTes)
+            "${SCRIPT_DIR}"/testCentaurTes.sh
+            ;;
+        centaurLocal)
+            "${SCRIPT_DIR}"/testCentaurLocal.sh
+            ;;
+        sbt)
+            "${SCRIPT_DIR}"/testSbt.sh
+            ;;
+        checkPublish)
+            "${SCRIPT_DIR}"/testCheckPublish.sh
+            ;;
+        *)
+            echo "Unknown BUILD_TYPE: '$BUILD_TYPE'"
+            exit 1
+            ;;
+        esac
+        ;;
+    cron)
+        case "$BUILD_TYPE" in
+        centaurJes)
+            "${SCRIPT_DIR}"/testCentaurJes.sh -i
+            ;;
+        centaurTes|centaurLocal|sbt|checkPublish)
+            exit 0
+            ;;
+        *)
+            echo "Unknown BUILD_TYPE: '$BUILD_TYPE'"
+            exit 1
+            ;;
+        esac
+        ;;
+    *)
+        echo "Unknown TRAVIS_EVENT_TYPE: '$TRAVIS_EVENT_TYPE'"
+        exit 1
+        ;;
+    esac
diff --git a/src/bin/travis/testCentaurJes.sh b/src/bin/travis/testCentaurJes.sh
index 98cdbb33d..e112a8f04 100755
--- a/src/bin/travis/testCentaurJes.sh
+++ b/src/bin/travis/testCentaurJes.sh
@@ -42,8 +42,59 @@ printTravisHeartbeat
 
 set -x
 
-# Unpack our credentials and such
-tar xvf jesConf.tar
+PROGNAME="$(basename "$0")"
+RUN_INTEGRATION_TESTS=0
+
+usage="
+$PROGNAME [-i ]
+
+Builds and runs specified branch of Cromwell and runs Centaur against it.
+
+Arguments:
+    -i    Flag that if supplied, will run centaur integration tests instead of standardtests
+"
+
+while getopts ":hi" option; do
+    case "$option" in
+        h) echo "$usage"
+            exit
+            ;;
+        i) RUN_INTEGRATION_TESTS=1
+            ;;
+        :) printf "Missing argument for -%s\n" "$OPTARG" >&2
+            echo "$usage" >&2
+            exit 1
+            ;;
+        \?) printf "Illegal option: -%s\n" "$OPTARG" >&2
+            echo "$usage" >&2
+            exit 1
+            ;;
+        esac
+done
+
+# TURN OFF LOGGING WHILE WE TALK TO DOCKER/VAULT
+set +x
+
+# Login to docker to access the dsde-toolbox
+docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD"
+
+# Login to vault to access secrets
+docker run --rm \
+    -v $HOME:/root:rw \
+    broadinstitute/dsde-toolbox \
+    vault auth "$JES_TOKEN" < /dev/null > /dev/null && echo vault auth success
+
+set -x
+
+# Render secrets
+docker run --rm \
+    -v $HOME:/root:rw \
+    -v $PWD/src/bin/travis/resources:/working \
+    -v $PWD:/output \
+    -e ENVIRONMENT=not_used \
+    -e INPUT_PATH=/working \
+    -e OUT_PATH=/output \
+    broadinstitute/dsde-toolbox render-templates.sh
 
 # Do a bunch of crap to enable gsutil. It's possible this is overkill but it doesn't take long anyways
 sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 1397BC53640DB551
@@ -55,32 +106,52 @@ export PYTHONPATH="/usr/lib/python2.7/site-packages:/usr/local/lib/python2.7/sit
 export CONFIGURE_OPTS="--enable-unicode=ucs4"
 pyenv install 2.7.10
 pyenv global 2.7.10
-sudo pip install --upgrade pip
-sudo pip install pyopenssl ndg-httpsclient pyasn1 --upgrade
+sudo -H pip install --upgrade pip
+sudo -H pip install pyopenssl ndg-httpsclient pyasn1 --upgrade
 export CLOUDSDK_PYTHON_SITEPACKAGES=1
-gcloud auth activate-service-account --key-file=broad-dsde-cromwell-dev-d71ad10e17f4.json "$CROMWELL_SERVICE_ACCOUNT"
+
+# Use sed to redact the service account from the stderr
+gcloud -q \
+    auth \
+    activate-service-account \
+    --key-file=cromwell-service-account.json 2>&1 | \
+    sed 's/[A-Za-z0-9._-]*@[A-Za-z0-9._-]*/REDACTED/g'
 
 echo "RUNNING TRAVIS CENTAUR"
 sbt assembly
 
-# Update the inputs file with stuff specific to this run
+
+# Update the .inputs file with stuff specific to this run
 sed -i "s/CENTAUR_BRANCH/${CENTAUR_BRANCH}/g" src/bin/travis/resources/centaur.inputs
 CROMWELL_JAR=cromwell_${TRAVIS_BUILD_ID}.jar
 sed -i "s/CROMWELL_JAR/${CROMWELL_JAR}/g" src/bin/travis/resources/centaur.inputs
 
+# pass integration directory to the inputs json otherwise remove it from the inputs file
+if [ $RUN_INTEGRATION_TESTS -ne 1 ]; then
+    sed -i "/INTEGRATION_TESTS_DIR/d" src/bin/travis/resources/centaur.inputs
+else
+    sed -i "s|INTEGRATION_TESTS_DIR|$INTEGRATION_TESTS_DIR|g" src/bin/travis/resources/centaur.inputs
+fi
+
 # Upload the built Cromwell jar to GCS so we can use it in our centaur test. Set an exit trap to clean it up on failure
 JAR_GCS_PATH=gs://cloud-cromwell-dev/travis-centaur/${CROMWELL_JAR}
-gsutil cp target/scala-2.11/cromwell-*.jar "${JAR_GCS_PATH}"
-
-java -Dconfig.file=./jes.conf -jar target/scala-2.11/cromwell-*.jar run src/bin/travis/resources/centaur.wdl src/bin/travis/resources/centaur.inputs | tee log.txt
+gsutil cp target/scala-2.12/cromwell-*.jar "${JAR_GCS_PATH}"
+
+java \
+    -Dconfig.file=./jes_centaur.conf \
+    -jar target/scala-2.12/cromwell-*.jar \
+    run \
+    src/bin/travis/resources/centaur.wdl \
+    --inputs src/bin/travis/resources/centaur.inputs | \
+    tee log.txt
 EXIT_CODE="${PIPESTATUS[0]}"
 
 # The perl code below is to remove our lovely color highlighting
-export WORKFLOW_ID=`grep "SingleWorkflowRunnerActor: Workflow submitted " log.txt | perl -pe 's/\e\[?.*?[\@-~]//g' | cut -f7 -d" "`
-
-# Grab the Centaur log from GCS and cat it so we see it in the main travis log. 
+WORKFLOW_ID=$(grep "SingleWorkflowRunnerActor: Workflow submitted " log.txt | perl -pe 's/\e\[?.*?[\@-~]//g' | cut -f7 -d" ")
+export WORKFLOW_ID
+# Grab the Centaur log from GCS and cat it so we see it in the main travis log.
 export CENTAUR_LOG_PATH="gs://cloud-cromwell-dev/cromwell_execution/travis/centaur_workflow/${WORKFLOW_ID}/call-centaur/cromwell_root/logs/centaur.log"
-gsutil cp ${CENTAUR_LOG_PATH} centaur.log
+gsutil cp "${CENTAUR_LOG_PATH}" centaur.log
 cat centaur.log
 echo "More logs for this run are available at https://console.cloud.google.com/storage/browser/cloud-cromwell-dev/cromwell_execution/travis/centaur_workflow/${WORKFLOW_ID}/call-centaur/"
 
diff --git a/src/bin/travis/testCentaurLocal.sh b/src/bin/travis/testCentaurLocal.sh
index 3023f0a80..ba8138f2c 100755
--- a/src/bin/travis/testCentaurLocal.sh
+++ b/src/bin/travis/testCentaurLocal.sh
@@ -9,28 +9,85 @@ printTravisHeartbeat() {
     TRAVIS_HEARTBEAT_PID=$!
 }
 
+cromwellLogTail() {
+ (
+   while [ ! -f logs/cromwell.log ];
+   do
+     sleep 2
+     printf "(Cr)"
+   done
+   tail -f logs/cromwell.log &
+   CROMWELL_LOG_TAIL_PID=$!
+ ) &
+ CROMWELL_LOG_WAIT_PID=$!
+}
+
+centaurLogTail() {
+ (
+   while [ ! -f logs/centaur.log ];
+   do
+     sleep 2
+     printf "(Ce)"
+   done
+   tail -f logs/centaur.log &
+   CENTAUR_LOG_TAIL_PID=$!
+ ) &
+ CENTAUR_LOG_WAIT_PID=$!
+}
+
 killTravisHeartbeat() {
     if [ -n "${TRAVIS_HEARTBEAT_PID+set}" ]; then
         kill ${TRAVIS_HEARTBEAT_PID} || true
     fi
 }
 
+killCromwellLogTail() {
+    if [ -n "${CROMWELL_LOG_TAIL_PID+set}" ]; then
+        kill ${CROMWELL_LOG_TAIL_PID} || true
+    else
+        if [ -n "${CROMWELL_LOG_WAIT_PID+set}" ]; then
+            kill ${CROMWELL_LOG_WAIT_PID} || true
+        fi
+    fi
+}
+
+killCentaurLogTail() {
+    if [ -n "${CENTAUR_LOG_TAIL_PID+set}" ]; then
+        kill ${CENTAUR_LOG_TAIL_PID} || true
+    else
+        if [ -n "${CENTAUR_LOG_WAIT_PID+set}" ]; then
+            kill ${CENTAUR_LOG_WAIT_PID} || true
+        fi
+    fi
+}
+
 exitScript() {
-    echo "CROMWELL LOG"
-    cat logs/cromwell.log
     echo "CENTAUR LOG"
     cat logs/centaur.log
     killTravisHeartbeat
+    killCromwellLogTail
+    killCentaurLogTail
 }
 
 trap exitScript EXIT
+trap exitScript TERM
+cromwellLogTail
+centaurLogTail
 printTravisHeartbeat
 
 set -x
 set -e
 
+sudo apt-get update -qq
+sudo apt-get install -qq mysql-server-5.6 mysql-client-5.6 mysql-client-core-5.6
+docker pull ubuntu:latest
+mysql -u root -e "SET GLOBAL sql_mode = 'STRICT_ALL_TABLES';"
+mysql -u root -e "CREATE DATABASE IF NOT EXISTS cromwell_test;"
+mysql -u root -e "CREATE USER 'travis'@'localhost' IDENTIFIED BY '';"
+mysql -u root -e "GRANT ALL PRIVILEGES ON cromwell_test . * TO 'travis'@'localhost';"
+
 sbt assembly
-CROMWELL_JAR=$(find "$(pwd)/target/scala-2.11" -name "cromwell-*.jar")
+CROMWELL_JAR=$(find "$(pwd)/target/scala-2.12" -name "cromwell-*.jar")
 LOCAL_CONF="$(pwd)/src/bin/travis/resources/local_centaur.conf"
 git clone https://github.com/broadinstitute/centaur.git
 cd centaur
diff --git a/src/bin/travis/testCentaurTes.sh b/src/bin/travis/testCentaurTes.sh
index caff9cbe0..8fb829bac 100755
--- a/src/bin/travis/testCentaurTes.sh
+++ b/src/bin/travis/testCentaurTes.sh
@@ -16,8 +16,8 @@ killTravisHeartbeat() {
 }
 
 exitScript() {
-    echo "TES LOG"
-    cat logs/tes.log
+    echo "FUNNEL LOG"
+    cat logs/funnel.log
     echo "CROMWELL LOG"
     cat logs/cromwell.log
     echo "CENTAUR LOG"
@@ -31,25 +31,57 @@ printTravisHeartbeat
 set -x
 set -e
 
+sudo apt-get update -qq
+sudo apt-get install -qq mysql-server-5.6 mysql-client-5.6 mysql-client-core-5.6
+docker pull ubuntu:latest
+mysql -u root -e "SET GLOBAL sql_mode = 'STRICT_ALL_TABLES';"
+mysql -u root -e "CREATE DATABASE IF NOT EXISTS cromwell_test;"
+mysql -u root -e "CREATE USER 'travis'@'localhost' IDENTIFIED BY '';"
+mysql -u root -e "GRANT ALL PRIVILEGES ON cromwell_test . * TO 'travis'@'localhost';"
+
+WORKDIR=$(pwd)
+
 sbt assembly
-CROMWELL_JAR=$(find "$(pwd)/target/scala-2.11" -name "cromwell-*.jar")
+CROMWELL_JAR=$(find "$(pwd)/target/scala-2.12" -name "cromwell-*.jar")
 TES_CENTAUR_CONF="$(pwd)/src/bin/travis/resources/tes_centaur.conf"
 git clone https://github.com/broadinstitute/centaur.git
 cd centaur
 git checkout ${CENTAUR_BRANCH}
-cd ..
+cd $WORKDIR
 
-TES_CONF="$(pwd)/src/bin/travis/resources/tes.conf"
-git clone https://github.com/ohsu-comp-bio/funnel.git
-cd funnel
+
+FUNNEL_CONF="$(pwd)/src/bin/travis/resources/funnel.conf"
+wget https://storage.googleapis.com/golang/go1.8.1.linux-amd64.tar.gz
+tar xfz go1.8.1.linux-amd64.tar.gz
+export GOROOT=$WORKDIR/go
+mkdir go-lib
+export GOPATH=$WORKDIR/go-lib
+go get github.com/ohsu-comp-bio/funnel
+cd $GOPATH/src/github.com/ohsu-comp-bio/funnel
+git checkout c4d9134
 make
-cd ..
+cd $WORKDIR
 mkdir logs
-nohup funnel/bin/tes-server -config ${TES_CONF} > logs/tes.log 2>&1 &
+nohup $GOPATH/bin/funnel server --config ${FUNNEL_CONF} > logs/funnel.log 2>&1 &
 
 
 # All tests use ubuntu:latest - make sure it's there before starting the tests 
 # because pulling the image during some of the tests would cause them to fail 
 # (specifically output_redirection which expects a specific value in stderr)
 docker pull ubuntu:latest
-centaur/test_cromwell.sh -j"${CROMWELL_JAR}" -c${TES_CENTAUR_CONF} -elocaldockertest
+
+# The following tests are skipped:
+#
+# non_root_specified_user:   TES doesn't support switching users in the image
+# write_lines_files:         all inputs are read-only in TES
+# lots_of_inputs:            Funnel mounts in each input separately, this task surpasses the docker limit for volumes
+# call_cache_capoeira_local: fails on task 'read_files_without_docker' since the 'docker' runtime key is required for this backend
+#
+centaur/test_cromwell.sh \
+-j ${CROMWELL_JAR} \
+-c ${TES_CENTAUR_CONF} \
+-e non_root_specified_user \
+-e write_lines_files \
+-e lots_of_inputs \
+-e call_cache_capoeira_local \
+-e non_root_default_user
diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf
index a27527934..e397980ad 100644
--- a/src/main/resources/application.conf
+++ b/src/main/resources/application.conf
@@ -2,14 +2,21 @@ akka {
   loggers = ["akka.event.slf4j.Slf4jLogger"]
   logging-filter = "akka.event.slf4j.Slf4jLoggingFilter"
   actor.guardian-supervisor-strategy = "cromwell.core.CromwellUserGuardianStrategy"
-}
 
-spray.can {
-  server {
-    request-timeout = 40s
+  http {
+    server {
+      request-timeout = 40s
+      bind-timeout = 5s
+    }
+
+    client.connecting-timeout = 40s
   }
-  client {
-    request-timeout = 40s
-    connecting-timeout = 40s
+
+  coordinated-shutdown.phases {
+    # Adds stop-io-activity to the dependencies of before-actor-system-terminate
+    before-actor-system-terminate.depends-on = ["cluster-shutdown", "stop-io-activity"]
+
+    # Adds abort-all-workflows-activity to the dependencies of service-stop
+    service-stop.depends-on = ["service-requests-done", "abort-all-workflows"]
   }
 }
diff --git a/src/main/scala/cromwell/CommandLineParser.scala b/src/main/scala/cromwell/CommandLineParser.scala
new file mode 100644
index 000000000..da909505e
--- /dev/null
+++ b/src/main/scala/cromwell/CommandLineParser.scala
@@ -0,0 +1,107 @@
+package cromwell
+
+import com.typesafe.config.ConfigFactory
+import cromwell.core.path.{DefaultPathBuilder, Path}
+import scopt.OptionParser
+
+object CommandLineParser extends App {
+
+  sealed trait Command
+  case object Run extends Command
+  case object Server extends Command
+
+  case class CommandLineArguments(command: Option[Command] = None,
+                                  workflowSource: Option[Path] = None,
+                                  workflowInputs: Option[Path] = None,
+                                  workflowOptions: Option[Path] = None,
+                                  workflowType: Option[String] = Option("WDL"),
+                                  workflowTypeVersion: Option[String] = Option("v2.0-draft"),
+                                  workflowLabels: Option[Path] = None,
+                                  imports: Option[Path] = None,
+                                  metadataOutput: Option[Path] = None
+                                 )
+
+  lazy val cromwellVersion = ConfigFactory.load("cromwell-version.conf").getConfig("version").getString("cromwell")
+
+  case class ParserAndCommand(parser: OptionParser[CommandLineArguments], command: Option[Command])
+
+  //  cromwell 29
+  //  Usage: java -jar /path/to/cromwell.jar [server|run] [options] <args>...
+  //
+  //    --help                   Cromwell - Workflow Execution Engine
+  //    --version
+  //  Command: server
+  //  Starts a web server on port 8000.  See the web server documentation for more details about the API endpoints.
+  //  Command: run [options] workflow-source
+  //  Run the workflow and print out the outputs in JSON format.
+  //  workflow-source          Workflow source file.
+  //  -i, --inputs <value>     Workflow inputs file.
+  //  -o, --options <value>    Workflow options file.
+  //  -t, --type <value>       Workflow type.
+  //  -v, --type-version <value>
+  //                           Workflow type version.
+  //  -l, --labels <value>     Workflow labels file.
+  //  -p, --imports <value>    A directory or zipfile to search for workflow imports.
+  //  -m, --metadata-output <value>
+  //                           An optional directory path to output metadata.
+
+  def buildParser(): scopt.OptionParser[CommandLineArguments] = {
+    new scopt.OptionParser[CommandLineArguments]("java -jar /path/to/cromwell.jar") {
+      head("cromwell", cromwellVersion)
+
+      help("help").text("Cromwell - Workflow Execution Engine")
+
+      version("version")
+
+      cmd("server").action((_, c) => c.copy(command = Option(Server))).text(
+        "Starts a web server on port 8000.  See the web server documentation for more details about the API endpoints.")
+
+      cmd("run").
+        action((_, c) => c.copy(command = Option(Run))).
+        text("Run the workflow and print out the outputs in JSON format.").
+        children(
+          arg[String]("workflow-source").text("Workflow source file.").required().
+            action((s, c) => c.copy(workflowSource = Option(DefaultPathBuilder.get(s)))),
+          opt[String]('i', "inputs").text("Workflow inputs file.").
+            action((s, c) =>
+              c.copy(workflowInputs = Option(DefaultPathBuilder.get(s)))),
+          opt[String]('o', "options").text("Workflow options file.").
+            action((s, c) =>
+              c.copy(workflowOptions = Option(DefaultPathBuilder.get(s)))),
+          opt[String]('t', "type").text("Workflow type.").
+            action((s, c) =>
+              c.copy(workflowType = Option(s))),
+          opt[String]('v', "type-version").text("Workflow type version.").
+            action((s, c) =>
+              c.copy(workflowTypeVersion = Option(s))),
+          opt[String]('l', "labels").text("Workflow labels file.").
+            action((s, c) =>
+              c.copy(workflowLabels = Option(DefaultPathBuilder.get(s)))),
+          opt[String]('p', "imports").text(
+            "A directory or zipfile to search for workflow imports.").
+            action((s, c) =>
+              c.copy(imports = Option(DefaultPathBuilder.get(s)))),
+          opt[String]('m', "metadata-output").text(
+            "An optional directory path to output metadata.").
+            action((s, c) =>
+              c.copy(metadataOutput = Option(DefaultPathBuilder.get(s))))
+        )
+    }
+  }
+
+  def runCromwell(args: CommandLineArguments): Unit = {
+    args.command match {
+      case Some(Run) => CromwellEntryPoint.runSingle(args)
+      case Some(Server) => CromwellEntryPoint.runServer()
+      case None => parser.showUsage()
+    }
+  }
+
+  val parser = buildParser()
+
+  val parsedArgs = parser.parse(args, CommandLineArguments())
+  parsedArgs match {
+    case Some(pa) => runCromwell(pa)
+    case None => parser.showUsage()
+  }
+}
diff --git a/src/main/scala/cromwell/CromwellCommandLine.scala b/src/main/scala/cromwell/CromwellCommandLine.scala
deleted file mode 100644
index 73d4c2543..000000000
--- a/src/main/scala/cromwell/CromwellCommandLine.scala
+++ /dev/null
@@ -1,137 +0,0 @@
-package cromwell
-
-import cats.data.Validated._
-import cats.syntax.cartesian._
-import cats.syntax.validated._
-import cromwell.core.path.{DefaultPathBuilder, Path}
-import cromwell.core.{WorkflowSourceFilesCollection, WorkflowSourceFilesWithDependenciesZip, WorkflowSourceFilesWithoutImports}
-import lenthall.exception.MessageAggregation
-import lenthall.validation.ErrorOr._
-import org.slf4j.LoggerFactory
-
-import scala.util.{Failure, Success, Try}
-
-sealed abstract class CromwellCommandLine
-case object UsageAndExit extends CromwellCommandLine
-case object RunServer extends CromwellCommandLine
-case object VersionAndExit extends CromwellCommandLine
-
-object CromwellCommandLine {
-  def apply(args: Seq[String]): CromwellCommandLine = {
-    args.headOption match {
-      case Some("server") if args.size == 1 => RunServer
-      case Some("run") if args.size >= 2 && args.size <= 6 => RunSingle(args.tail)
-      case Some("-version") if args.size == 1 => VersionAndExit
-      case _ => UsageAndExit
-    }
-  }
-}
-
-// We cannot initialize the logging until after we parse the command line in Main.scala. So we have to bundle up and pass back this information, just for logging.
-case class SingleRunPathParameters(wdlPath: Path, inputsPath: Option[Path], optionsPath: Option[Path], metadataPath: Option[Path], importPath: Option[Path], labelsPath: Option[Path]) {
-  def logMe(log: org.slf4j.Logger) = {
-    log.info(s"  WDL file: $wdlPath")
-    inputsPath foreach { i => log.info(s"  Inputs: $i") }
-    optionsPath foreach { o => log.info(s"  Workflow Options: $o") }
-    metadataPath foreach { m => log.info(s"  Workflow Metadata Output: $m") }
-    importPath foreach { i => log.info(s"  WDL import bundle: $i") }
-    labelsPath foreach { o => log.info(s"  Custom labels: $o") }
-  }
-}
-
-final case class RunSingle(sourceFiles: WorkflowSourceFilesCollection, paths: SingleRunPathParameters) extends CromwellCommandLine
-
-object RunSingle {
-
-  lazy val Log = LoggerFactory.getLogger("cromwell")
-
-  def apply(args: Seq[String]): RunSingle = {
-    val pathParameters = SingleRunPathParameters(
-      wdlPath = DefaultPathBuilder.get(args.head).toAbsolutePath,
-      inputsPath = argPath(args, 1, Option("inputs"), checkDefaultExists = false),
-      optionsPath = argPath(args, 2, Option("options")),
-      metadataPath = argPath(args, 3, None),
-      importPath = argPath(args, 4, None),
-      labelsPath = argPath(args, 5, None)
-    )
-
-    val wdl = readContent("WDL file", pathParameters.wdlPath)
-    val inputsJson = readJson("Inputs", pathParameters.inputsPath)
-    val optionsJson = readJson("Workflow Options", pathParameters.optionsPath)
-    val labelsJson = readJson("Labels", pathParameters.labelsPath)
-
-    val sourceFileCollection = pathParameters.importPath match {
-      case Some(p) => (wdl |@| inputsJson |@| optionsJson |@| labelsJson) map { (w, i, o, l) =>
-        WorkflowSourceFilesWithDependenciesZip.apply(w, i, o, l, p.loadBytes) }
-      case None => (wdl |@| inputsJson |@| optionsJson |@| labelsJson) map WorkflowSourceFilesWithoutImports.apply
-    }
-
-    val runSingle: ErrorOr[RunSingle] = for {
-      sources <- sourceFileCollection
-      _ <- writeableMetadataPath(pathParameters.metadataPath)
-    } yield RunSingle(sources, pathParameters)
-
-    runSingle match {
-      case Valid(r) => r
-      case Invalid(nel) => throw new RuntimeException with MessageAggregation {
-        override def exceptionContext: String = "ERROR: Unable to run Cromwell:"
-        override def errorMessages: Traversable[String] = nel.toList
-      }
-    }
-  }
-
-  private def writeableMetadataPath(path: Option[Path]): ErrorOr[Unit] = {
-    path match {
-      case Some(p) if !metadataPathIsWriteable(p) => s"Unable to write to metadata directory: $p".invalidNel
-      case _ => ().validNel
-    }
-  }
-
-  /** Read the path to a string. */
-  private def readContent(inputDescription: String, path: Path): ErrorOr[String] = {
-    if (!path.exists) {
-      s"$inputDescription does not exist: $path".invalidNel
-    } else if (!path.isReadable) {
-      s"$inputDescription is not readable: $path".invalidNel
-    } else path.contentAsString.validNel
-  }
-
-  /** Read the path to a string, unless the path is None, in which case returns "{}". */
-  private def readJson(inputDescription: String, pathOption: Option[Path]): ErrorOr[String] = {
-    pathOption match {
-      case Some(path) => readContent(inputDescription, path)
-      case None => "{}".validNel
-    }
-  }
-
-  private def metadataPathIsWriteable(metadataPath: Path): Boolean = {
-    Try(metadataPath.createIfNotExists(createParents = true).append("")) match {
-      case Success(_) => true
-      case Failure(_) => false
-    }
-  }
-
-  /**
-    * Retrieve the arg at index as path, or return some default. Args specified as "-" will be returned as None.
-    *
-    * @param args The run command arguments, with the wdl path at arg.head.
-    * @param index The index of the path we're looking for.
-    * @param defaultExt The default extension to use if the argument was not specified at all.
-    * @param checkDefaultExists If true, verify that our computed default file exists before using it.
-    * @return The argument as a Path resolved as a sibling to the wdl path.
-    */
-  private def argPath(args: Seq[String], index: Int, defaultExt: Option[String],
-                      checkDefaultExists: Boolean = true): Option[Path] = {
-
-    // To return a default, swap the extension, and then maybe check if the file exists.
-    def defaultPath = defaultExt
-      .map(ext => DefaultPathBuilder.get(args.head).swapExt("wdl", ext))
-      .filter(path => !checkDefaultExists || path.exists)
-      .map(_.pathAsString)
-
-    // Return the path for the arg index, or the default, but remove "-" paths.
-    for {
-      path <- args.lift(index) orElse defaultPath filterNot (_ == "-")
-    } yield DefaultPathBuilder.get(path).toAbsolutePath
-  }
-}
diff --git a/src/main/scala/cromwell/CromwellEntryPoint.scala b/src/main/scala/cromwell/CromwellEntryPoint.scala
new file mode 100644
index 000000000..ab8926fbf
--- /dev/null
+++ b/src/main/scala/cromwell/CromwellEntryPoint.scala
@@ -0,0 +1,202 @@
+package cromwell
+
+import akka.pattern.GracefulStopSupport
+import cats.data.Validated._
+import cats.syntax.cartesian._
+import cats.syntax.validated._
+import com.typesafe.config.ConfigFactory
+import cromwell.CommandLineParser._
+import cromwell.core.path.Path
+import cromwell.core.{WorkflowSourceFilesCollection, WorkflowSourceFilesWithDependenciesZip, WorkflowSourceFilesWithoutImports}
+import cromwell.engine.workflow.SingleWorkflowRunnerActor
+import cromwell.engine.workflow.SingleWorkflowRunnerActor.RunWorkflow
+import cromwell.server.{CromwellServer, CromwellSystem}
+import lenthall.exception.MessageAggregation
+import lenthall.validation.ErrorOr._
+import net.ceedubs.ficus.Ficus._
+import org.slf4j.LoggerFactory
+
+import scala.collection.JavaConverters._
+import scala.concurrent.duration.{Duration, _}
+import scala.concurrent.{Await, Future, TimeoutException}
+import scala.language.postfixOps
+import scala.util.{Failure, Success, Try}
+
+object CromwellEntryPoint extends GracefulStopSupport {
+
+  lazy val EntryPointLogger = LoggerFactory.getLogger("Cromwell EntryPoint")
+  private lazy val config = ConfigFactory.load()
+
+  // Only abort jobs on SIGINT if the config explicitly sets system.abort-jobs-on-terminate = true.
+  val abortJobsOnTerminate = config.as[Boolean]("system.abort-jobs-on-terminate")
+
+  val gracefulShutdown = config.as[Boolean]("system.graceful-server-shutdown")
+
+  /**
+    * Run Cromwell in server mode.
+    */
+  def runServer() = {
+    val system = buildCromwellSystem(Server)
+    waitAndExit(CromwellServer.run(gracefulShutdown, abortJobsOnTerminate), system)
+  }
+
+  /**
+    * Run a single workflow using the successfully parsed but as yet not validated arguments.
+    */
+  def runSingle(args: CommandLineArguments): Unit = {
+    val cromwellSystem = buildCromwellSystem(Run)
+    implicit val actorSystem = cromwellSystem.actorSystem
+
+    val sources = validateRunArguments(args)
+    val runnerProps = SingleWorkflowRunnerActor.props(sources, args.metadataOutput, gracefulShutdown, abortJobsOnTerminate)(cromwellSystem.materializer)
+
+    val runner = cromwellSystem.actorSystem.actorOf(runnerProps, "SingleWorkflowRunnerActor")
+
+    import cromwell.util.PromiseActor.EnhancedActorRef
+    waitAndExit(_ => runner.askNoTimeout(RunWorkflow), cromwellSystem)
+  }
+
+  private def buildCromwellSystem(command: Command): CromwellSystem = {
+    initLogging(command)
+    lazy val Log = LoggerFactory.getLogger("cromwell")
+    Try {
+      new CromwellSystem {}
+    } recoverWith {
+      case t: Throwable =>
+        Log.error("Failed to instantiate Cromwell System. Shutting down Cromwell.")
+        Log.error(t.getMessage)
+        System.exit(1)
+        Failure(t)
+    } get
+  }
+
+  /**
+    * If a cromwell server is going to be run, makes adjustments to the default logback configuration.
+    * Overwrites LOG_MODE system property used in our logback.xml, _before_ the logback classes load.
+    * Restored from similar functionality in
+    *   https://github.com/broadinstitute/cromwell/commit/2e3f45b#diff-facc2160a82442932c41026c9a1e4b2bL28
+    * TODO: Logback is configurable programmatically. We don't have to overwrite system properties like this.
+    *
+    * Also copies variables from config/system/environment/defaults over to the system properties.
+    * Fixes issue where users are trying to specify Java properties as environment variables.
+    */
+  private def initLogging(command: Command): Unit = {
+    val logbackSetting = command match {
+      case Server => "STANDARD"
+      case Run => "PRETTY"
+    }
+
+    val defaultProps = Map(
+      "LOG_MODE" -> logbackSetting,
+      "LOG_LEVEL" -> "INFO"
+    )
+
+    val configWithFallbacks = config
+      .withFallback(ConfigFactory.systemEnvironment())
+      .withFallback(ConfigFactory.parseMap(defaultProps.asJava, "Defaults"))
+
+    val props = sys.props
+    defaultProps.keys foreach { key =>
+      props += key -> configWithFallbacks.getString(key)
+    }
+
+    /*
+    We've possibly copied values from the environment, or our defaults, into the system properties.
+    Make sure that the next time one uses the ConfigFactory that our updated system properties are loaded.
+     */
+    ConfigFactory.invalidateCaches()
+  }
+
+  private def waitAndExit(runner: CromwellSystem => Future[Any], workflowManagerSystem: CromwellSystem): Unit = {
+    val futureResult = runner(workflowManagerSystem)
+    Await.ready(futureResult, Duration.Inf)
+
+    try {
+      Await.ready(workflowManagerSystem.shutdownActorSystem(), 30 seconds)
+    } catch {
+      case _: TimeoutException => Console.err.println("Timed out trying to shutdown actor system")
+      case other: Exception => Console.err.println(s"Unexpected error trying to shutdown actor system: ${other.getMessage}")
+    }
+
+    val returnCode = futureResult.value.get match {
+      case Success(_) => 0
+      case Failure(e) =>
+        Console.err.println(e.getMessage)
+        1
+    }
+
+    sys.exit(returnCode)
+  }
+
+  def validateRunArguments(args: CommandLineArguments): WorkflowSourceFilesCollection = {
+
+    val workflowSource = readContent("Workflow source", args.workflowSource.get)
+    val inputsJson = readJson("Workflow inputs", args.workflowInputs)
+    val optionsJson = readJson("Workflow options", args.workflowOptions)
+    val labelsJson = readJson("Workflow labels", args.workflowLabels)
+
+    val sourceFileCollection = args.imports match {
+      case Some(p) => (workflowSource |@| inputsJson |@| optionsJson |@| labelsJson) map { (w, i, o, l) =>
+        WorkflowSourceFilesWithDependenciesZip.apply(
+          workflowSource = w,
+          workflowType = Option("WDL"),
+          workflowTypeVersion = None,
+          inputsJson = i,
+          workflowOptionsJson = o,
+          labelsJson = l,
+          importsZip = p.loadBytes)
+      }
+      case None => (workflowSource |@| inputsJson |@| optionsJson |@| labelsJson) map { (w, i, o, l) =>
+        WorkflowSourceFilesWithoutImports.apply(
+          workflowSource = w,
+          workflowType = Option("WDL"),
+          workflowTypeVersion = None,
+          inputsJson = i,
+          workflowOptionsJson = o,
+          labelsJson = l
+        )
+      }
+    }
+
+    val sourceFiles = for {
+      sources <- sourceFileCollection
+      _ <- writeableMetadataPath(args.metadataOutput)
+    } yield sources
+
+    sourceFiles match {
+      case Valid(r) => r
+      case Invalid(nel) => throw new RuntimeException with MessageAggregation {
+        override def exceptionContext: String = "ERROR: Unable to run Cromwell:"
+        override def errorMessages: Traversable[String] = nel.toList
+      }
+    }
+  }
+
+  private def writeableMetadataPath(path: Option[Path]): ErrorOr[Unit] = {
+    path match {
+      case Some(p) if !metadataPathIsWriteable(p) => s"Unable to write to metadata directory: $p".invalidNel
+      case _ => ().validNel
+    }
+  }
+
+  /** Read the path to a string. */
+  private def readContent(inputDescription: String, path: Path): ErrorOr[String] = {
+    if (!path.exists) {
+      s"$inputDescription does not exist: $path".invalidNel
+    } else if (!path.isReadable) {
+      s"$inputDescription is not readable: $path".invalidNel
+    } else path.contentAsString.validNel
+  }
+
+  /** Read the path to a string, unless the path is None, in which case returns "{}". */
+  private def readJson(inputDescription: String, pathOption: Option[Path]): ErrorOr[String] = {
+    pathOption match {
+      case Some(path) => readContent(inputDescription, path)
+      case None => "{}".validNel
+    }
+  }
+
+  private def metadataPathIsWriteable(metadataPath: Path): Boolean =
+    Try(metadataPath.createIfNotExists(createParents = true).append("")).isSuccess
+
+}
diff --git a/src/main/scala/cromwell/Main.scala b/src/main/scala/cromwell/Main.scala
deleted file mode 100644
index 445c227b7..000000000
--- a/src/main/scala/cromwell/Main.scala
+++ /dev/null
@@ -1,146 +0,0 @@
-package cromwell
-
-import com.typesafe.config.ConfigFactory
-import cromwell.engine.workflow.SingleWorkflowRunnerActor
-import cromwell.engine.workflow.SingleWorkflowRunnerActor.RunWorkflow
-import cromwell.server.{CromwellServer, CromwellSystem}
-import cromwell.util.PromiseActor
-import org.slf4j.LoggerFactory
-
-import scala.collection.JavaConverters._
-import scala.concurrent.duration._
-import scala.concurrent.{Await, Future}
-import scala.language.postfixOps
-import scala.util.{Failure, Success, Try}
-
-object Main extends App {
-  val CommandLine = CromwellCommandLine(args)
-  initLogging(CommandLine)
-
-  lazy val Log = LoggerFactory.getLogger("cromwell")
-  lazy val CromwellSystem: CromwellSystem = Try {
-    new CromwellSystem {}
-  } recoverWith {
-    case t: Throwable =>
-      Log.error("Failed to instantiate Cromwell System. Shutting down Cromwell.")
-      Log.error(t.getMessage)
-      System.exit(1)
-      Failure(t)
-  } get
-
-  CommandLine match {
-    case UsageAndExit => usageAndExit()
-    case VersionAndExit => versionAndExit()
-    case RunServer => waitAndExit(CromwellServer.run(CromwellSystem), CromwellSystem)
-    case r: RunSingle => runWorkflow(r)
-  }
-
-  /**
-    * If a cromwell server is going to be run, makes adjustments to the default logback configuration.
-    * Overwrites LOG_MODE system property used in our logback.xml, _before_ the logback classes load.
-    * Restored from similar functionality in
-    *   https://github.com/broadinstitute/cromwell/commit/2e3f45b#diff-facc2160a82442932c41026c9a1e4b2bL28
-    * TODO: Logback is configurable programmatically. We don't have to overwrite system properties like this.
-    *
-    * Also copies variables from config/system/environment/defaults over to the system properties.
-    * Fixes issue where users are trying to specify Java properties as environment variables.
-    */
-  private def initLogging(commandLine: CromwellCommandLine): Unit = {
-    val defaultLogMode = commandLine match {
-      case RunServer => "STANDARD"
-      case _ => "PRETTY"
-    }
-
-    val defaultProps = Map("LOG_MODE" -> defaultLogMode, "LOG_LEVEL" -> "INFO")
-
-    val config = ConfigFactory.load
-      .withFallback(ConfigFactory.systemEnvironment())
-      .withFallback(ConfigFactory.parseMap(defaultProps.asJava, "Defaults"))
-
-    val props = sys.props
-    defaultProps.keys foreach { key =>
-      props += key -> config.getString(key)
-    }
-
-    /*
-    We've possibly copied values from the environment, or our defaults, into the system properties.
-    Make sure that the next time one uses the ConfigFactory that our updated system properties are loaded.
-     */
-    ConfigFactory.invalidateCaches()
-  }
-
-  private def runWorkflow(commandLine: RunSingle): Unit = {
-    implicit val actorSystem = CromwellSystem.actorSystem
-
-    Log.info(s"RUN sub-command")
-    commandLine.paths.logMe(Log)
-    val runnerProps = SingleWorkflowRunnerActor.props(commandLine.sourceFiles, commandLine.paths.metadataPath)
-
-    val runner = CromwellSystem.actorSystem.actorOf(runnerProps, "SingleWorkflowRunnerActor")
-
-    import PromiseActor.EnhancedActorRef
-
-    waitAndExit(runner.askNoTimeout(RunWorkflow), CromwellSystem)
-  }
-
-  private def waitAndExit(futureResult: Future[Any], workflowManagerSystem: CromwellSystem): Unit = {
-    Await.ready(futureResult, Duration.Inf)
-
-    workflowManagerSystem.shutdownActorSystem()
-
-    val returnCode = futureResult.value.get match {
-      case Success(_) => 0
-      case Failure(e) =>
-        Console.err.println(e.getMessage)
-        1
-    }
-
-    sys.exit(returnCode)
-  }
-
-  def usageAndExit(): Unit = {
-    println(
-      """
-        |java -jar cromwell.jar <action> <parameters>
-        |
-        |Actions:
-        |run <WDL file> [<JSON inputs file>] [<JSON workflow options>]
-        |  [<OUTPUT workflow metadata>] [<Zip of WDL Files>] [<JSON labels file>]
-        |
-        |  Given a WDL file and JSON file containing the value of the
-        |  workflow inputs, this will run the workflow locally and
-        |  print out the outputs in JSON format.  The workflow
-        |  options file specifies some runtime configuration for the
-        |  workflow (see README for details).  The workflow metadata
-        |  output is an optional file path to output the metadata. The
-        |  directory of WDL files is optional. However, it is required
-        |  if the primary workflow imports workflows that are outside
-        |  of the root directory of the Cromwell project.
-        |
-        |  Use a single dash ("-") to skip optional files. Ex:
-        |    run noinputs.wdl - - metadata.json -
-        |
-        |server
-        |
-        |  Starts a web server on port 8000.  See the web server
-        |  documentation for more details about the API endpoints.
-        |
-        |-version
-        |
-        |   Returns the version of the Cromwell engine.
-        |
-      """.stripMargin)
-
-    System.exit(0)
-  }
-
-  def versionAndExit(): Unit = {
-    val versionConf = ConfigFactory.load("cromwell-version.conf").getConfig("version")
-    println(
-      s"""
-         |cromwell: ${versionConf.getString("cromwell")}
-       """.stripMargin
-    )
-    System.exit(0)
-  }
-}
diff --git a/src/test/scala/cromwell/CromwellCommandLineSpec.scala b/src/test/scala/cromwell/CromwellCommandLineSpec.scala
index bd6da383a..1f452175d 100644
--- a/src/test/scala/cromwell/CromwellCommandLineSpec.scala
+++ b/src/test/scala/cromwell/CromwellCommandLineSpec.scala
@@ -1,97 +1,119 @@
 package cromwell
 
+import cromwell.CommandLineParser.{CommandLineArguments, Run, Server}
+import cromwell.CromwellCommandLineSpec.WdlAndInputs
 import cromwell.core.path.{DefaultPathBuilder, Path}
 import cromwell.util.SampleWdl
 import cromwell.util.SampleWdl.{FileClobber, FilePassingWorkflow, ThreeStep}
-import org.scalatest.{FlatSpec, Matchers}
+import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers}
 
 import scala.util.Try
 
-class CromwellCommandLineSpec extends FlatSpec with Matchers {
-  import CromwellCommandLineSpec._
+class CromwellCommandLineSpec extends FlatSpec with Matchers with BeforeAndAfter {
+
+  var parser: scopt.OptionParser[CommandLineArguments] = _
 
   behavior of "CromwellCommandLine"
 
-  it should "UsageAndExit with no arguments" in {
-    CromwellCommandLine(List.empty[String]) shouldBe UsageAndExit
+  before {
+    parser = CommandLineParser.buildParser()
   }
 
-  it should "RunServer when specified" in {
-    CromwellCommandLine(List("server")) shouldBe RunServer
+  it should "fail to parse with no arguments" in {
+    parser.parse(Array.empty[String], CommandLineArguments()).get.command shouldBe None
   }
 
-  it should "UsageAndExit when supplying an argument to server" in {
-    CromwellCommandLine(List("server", "foo")) shouldBe UsageAndExit
+  it should "run server when specified" in {
+    parser.parse(Array("server"), CommandLineArguments()).get.command shouldBe Some(Server)
   }
 
-  it should "UsageAndExit with no arguments to run" in {
-    CromwellCommandLine(List("run")) shouldBe UsageAndExit
+  it should "fail to parse when supplying an argument to server" in {
+    parser.parse(Array("server", "foo"), CommandLineArguments()) shouldBe None
   }
 
-  it should "fail with too many arguments to run" in {
-    CromwellCommandLine(List("run", "bork", "bork", "bork", "bork", "bork", "blerg"))
+  it should "fail to parse with no arguments to run" in {
+    parser.parse(Array("run"), CommandLineArguments()) shouldBe None
   }
 
-  it should "VersionAndExit when the `-version` flag is passed" in {
-    CromwellCommandLine(List("-version")) shouldBe VersionAndExit
+  it should "fail to parse with too many arguments to run" in {
+    parser.parse(Array("run", "forrest", "run"), CommandLineArguments()) shouldBe None
   }
 
-  it should "RunSingle when supplying wdl and inputs" in {
-    CromwellCommandLine(List("run", ThreeStepWithoutOptions.wdl, ThreeStepWithoutOptions.inputs)) shouldBe a [RunSingle]
+  // --version exits the JVM which is not great in a test suite.  Haven't figure out a way to test this yet.
+  //  it should "handle version output when the `-version` flag is passed" in {
+  //    // I don't see a way to see that --version is printing just the version, but this at least confirms a `None`
+  //    // output that should generate a usage and version.
+  //    parser.parse(Array("--version"), CommandLineArguments()) shouldBe None
+  //  }
+
+  it should "run single when supplying wdl and inputs" in {
+    val optionsLast = parser.parse(Array("run", "3step.wdl", "--inputs", "3step.inputs"), CommandLineArguments()).get
+    optionsLast.command shouldBe Some(Run)
+    optionsLast.workflowSource.get.pathAsString shouldBe "3step.wdl"
+    optionsLast.workflowInputs.get.pathAsString shouldBe "3step.inputs"
+
+    val optionsFirst = parser.parse(Array("run", "--inputs", "3step.inputs", "3step.wdl"), CommandLineArguments()).get
+    optionsFirst.command shouldBe Some(Run)
+    optionsFirst.workflowSource.get.pathAsString shouldBe "3step.wdl"
+    optionsFirst.workflowInputs.get.pathAsString shouldBe "3step.inputs"
   }
 
-  it should "RunSingle with default inputs when only supplying wdl" in {
-    val ccl = CromwellCommandLine(List("run", ThreeStepWithoutOptions.wdl)).asInstanceOf[RunSingle]
-     ccl.sourceFiles.inputsJson shouldBe ThreeStepInputs
+  it should "run single when supplying wdl and inputs and options" in {
+    val optionsLast = parser.parse(Array("run", "3step.wdl", "--inputs", "3step.inputs", "--options", "3step.options"), CommandLineArguments()).get
+    optionsLast.command shouldBe Some(Run)
+    optionsLast.workflowSource.get.pathAsString shouldBe "3step.wdl"
+    optionsLast.workflowInputs.get.pathAsString shouldBe "3step.inputs"
+    optionsLast.workflowOptions.get.pathAsString shouldBe "3step.options"
+
+    val optionsFirst = parser.parse(Array("run", "--inputs", "3step.inputs", "--options", "3step.options", "3step.wdl"), CommandLineArguments()).get
+    optionsFirst.command shouldBe Some(Run)
+    optionsFirst.workflowSource.get.pathAsString shouldBe "3step.wdl"
+    optionsFirst.workflowInputs.get.pathAsString shouldBe "3step.inputs"
+    optionsFirst.workflowOptions.get.pathAsString shouldBe "3step.options"
   }
 
-  it should "RunSingle with defaults if you use dashes" in {
-    val ccl = CromwellCommandLine(List("run", ThreeStepWithoutOptions.wdl, "-", "-", "-")).asInstanceOf[RunSingle]
-    ccl.sourceFiles.inputsJson shouldBe "{}"
-    ccl.sourceFiles.workflowOptionsJson shouldBe "{}"
-  }
+  it should "fail if input files do not exist" in {
+    val parsedArgs = parser.parse(Array("run", "3step.wdl", "--inputs", "3step.inputs", "--options", "3step.options"), CommandLineArguments()).get
+    val validation = Try(CromwellEntryPoint.validateRunArguments(parsedArgs))
 
-  it should "RunSingle with options, if passed in" in {
-    val threeStep = WdlAndInputs(ThreeStep, optionsJson = """{ foobar bad json! }""")
-    val ccl = CromwellCommandLine(List("run", threeStep.wdl, threeStep.inputs, threeStep.options)).asInstanceOf[RunSingle]
-    ccl.sourceFiles.workflowOptionsJson shouldBe threeStep.optionsJson
-  }
-
-  it should "fail if inputs path does not exist" in {
-    val ccl = Try(CromwellCommandLine(List("run", ThreeStepWithoutOptions.wdl, "/some/path/that/doesnt/exit")))
-    ccl.isFailure shouldBe true
-    ccl.failed.get.getMessage should include("Inputs does not exist")
+    validation.isFailure shouldBe true
+    validation.failed.get.getMessage should include("Workflow source does not exist")
+    validation.failed.get.getMessage should include("Workflow inputs does not exist")
+    validation.failed.get.getMessage should include("Workflow options does not exist")
   }
 
-  it should "fail if inputs path is not writeable" in {
+  it should "fail if inputs path is not readable" in {
     val threeStep = WdlAndInputs(ThreeStep)
+    val parsedArgs = parser.parse(Array("run", threeStep.wdl, "--inputs", threeStep.inputs), CommandLineArguments()).get
     threeStep.inputsFile setPermissions Set.empty
-    val ccl = Try(CromwellCommandLine(List("run", threeStep.wdl, threeStep.inputs)))
+    val ccl = Try(CromwellEntryPoint.validateRunArguments(parsedArgs))
     ccl.isFailure shouldBe true
-    ccl.failed.get.getMessage should include("Inputs is not readable")
+    ccl.failed.get.getMessage should include("Workflow inputs is not readable")
   }
 
-  it should "fail if metadata path is not writeable" in {
+  it should "fail if metadata output path is not writeable" in {
     val threeStep = WdlAndInputs(ThreeStep)
+    val parsedArgs = parser.parse(Array("run", threeStep.wdl, "--inputs", threeStep.inputs, "--metadata-output", threeStep.metadata), CommandLineArguments()).get
     threeStep.metadataFile write "foo"
     threeStep.metadataFile setPermissions Set.empty
-    val ccl = Try(CromwellCommandLine(List("run", threeStep.wdl, threeStep.inputs, "-", threeStep.metadata)))
+    val ccl = Try(CromwellEntryPoint.validateRunArguments(parsedArgs))
     ccl.isFailure shouldBe true
     ccl.failed.get.getMessage should include("Unable to write to metadata directory:")
   }
 
-  it should "run if imports directory is a .zip file" in {
+  it should "run if the imports path is a .zip file" in {
     val wdlDir = DefaultPathBuilder.createTempDirectory("wdlDirectory")
 
     val filePassing = DefaultPathBuilder.createTempFile("filePassing", ".wdl", Option(wdlDir))
     val fileClobber = DefaultPathBuilder.createTempFile("fileClobber", ".wdl", Option(wdlDir))
-    filePassing write FilePassingWorkflow.wdlSource()
-    fileClobber write FileClobber.wdlSource()
+    filePassing write FilePassingWorkflow.workflowSource()
+    fileClobber write FileClobber.workflowSource()
 
     val zippedDir = wdlDir.zip()
     val zippedPath = zippedDir.pathAsString
 
-    val ccl = Try(CromwellCommandLine(List("run", filePassing.pathAsString, "-", "-", "-", zippedPath)))
+    val parsedArgs = parser.parse(Array("run", filePassing.pathAsString, "--imports", zippedPath), CommandLineArguments()).get
+    val ccl = Try(CromwellEntryPoint.validateRunArguments(parsedArgs))
     ccl.isFailure shouldBe false
 
     zippedDir.delete(swallowIOExceptions = true)
@@ -113,7 +135,7 @@ object CromwellCommandLineSpec {
     lazy val wdlFile = {
       val file = DefaultPathBuilder.createTempFile(s"${sampleWdl.name}.", ".wdl")
       tempFiles :+= file
-      file write sampleWdl.wdlSource()
+      file write sampleWdl.workflowSource()
     }
 
     lazy val wdl = wdlFile.pathAsString
@@ -121,7 +143,7 @@ object CromwellCommandLineSpec {
     lazy val inputsFile = {
       val file = wdlFile.swapExt("wdl", "inputs")
       tempFiles :+= file
-      file write sampleWdl.wdlJson
+      file write sampleWdl.workflowJson
     }
 
     lazy val inputs = inputsFile.pathAsString
diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorBackendFactory.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorBackendFactory.scala
deleted file mode 100644
index 3c8e48fd4..000000000
--- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorBackendFactory.scala
+++ /dev/null
@@ -1,72 +0,0 @@
-package cromwell.backend.impl.htcondor
-
-import akka.actor.{ActorRef, Props}
-import com.typesafe.config.Config
-import com.typesafe.scalalogging.StrictLogging
-import cromwell.backend._
-import cromwell.backend.impl.htcondor.caching.CacheActorFactory
-import cromwell.backend.io.JobPathsWithDocker
-import cromwell.backend.sfs.SharedFileSystemExpressionFunctions
-import cromwell.core.{CallContext, WorkflowOptions}
-import wdl4s.TaskCall
-import wdl4s.expression.WdlStandardLibraryFunctions
-
-import scala.util.{Failure, Success, Try}
-
-case class HtCondorBackendFactory(name: String, configurationDescriptor: BackendConfigurationDescriptor)
-  extends BackendLifecycleActorFactory with StrictLogging {
-
-  override def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor,
-                                                calls: Set[TaskCall],
-                                                serviceRegistryActor: ActorRef): Option[Props] = {
-    Option(HtCondorInitializationActor.props(workflowDescriptor, calls, configurationDescriptor, serviceRegistryActor))
-  }
-
-  override def jobExecutionActorProps(jobDescriptor: BackendJobDescriptor,
-                                      initializationData: Option[BackendInitializationData],
-                                      serviceRegistryActor: ActorRef,
-                                      backendSingletonActor: Option[ActorRef]): Props = {
-    HtCondorJobExecutionActor.props(jobDescriptor, configurationDescriptor, serviceRegistryActor, resolveCacheProviderProps(jobDescriptor.workflowDescriptor.workflowOptions))
-  }
-
-  override def expressionLanguageFunctions(workflowDescriptor: BackendWorkflowDescriptor,
-                                           jobKey: BackendJobDescriptorKey,
-                                           initializationData: Option[BackendInitializationData]): WdlStandardLibraryFunctions = {
-    val jobPaths = new JobPathsWithDocker(jobKey, workflowDescriptor, configurationDescriptor.backendConfig)
-    val callContext = CallContext(
-      jobPaths.callExecutionRoot,
-      jobPaths.stdout.toAbsolutePath.toString,
-      jobPaths.stderr.toAbsolutePath.toString
-    )
-
-    new SharedFileSystemExpressionFunctions(HtCondorJobExecutionActor.pathBuilders, callContext)
-  }
-
-  private def resolveCacheProviderProps(workflowOptions: WorkflowOptions) = {
-    val defaultCacheEnabled = configurationDescriptor.backendConfig.getBoolean("cache.enabled")
-    val cacheEnabled: Boolean = getBooleanFromWfOptions(workflowOptions, "cacheEnabled", defaultCacheEnabled)
-
-    if (cacheEnabled) {
-      val defaultForceRewrite = configurationDescriptor.backendConfig.getBoolean("cache.forceRewrite")
-      val cacheForceRewrite = getBooleanFromWfOptions(workflowOptions, "cacheForceRw", defaultForceRewrite)
-      val provider = configurationDescriptor.backendConfig.getString("cache.provider")
-      val cacheFactory = Class.forName(provider)
-        .getConstructor(classOf[Config])
-        .newInstance(configurationDescriptor.backendConfig)
-        .asInstanceOf[CacheActorFactory]
-      Option(cacheFactory.getCacheActorProps(cacheForceRewrite))
-    }
-    else None
-  }
-
-  private def getBooleanFromWfOptions(workflowOptions: WorkflowOptions, optionKey: String, defaultValue: Boolean) = {
-    workflowOptions.get(optionKey) match {
-      case Success(value) => Try(value.toBoolean).getOrElse {
-        logger.warn(s"Could not get '$optionKey' attribute from workflow options. Falling back to default value.")
-        defaultValue
-      }
-      case Failure(_) => defaultValue
-    }
-  }
-}
-
diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorInitializationActor.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorInitializationActor.scala
deleted file mode 100644
index 1212876b6..000000000
--- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorInitializationActor.scala
+++ /dev/null
@@ -1,69 +0,0 @@
-package cromwell.backend.impl.htcondor
-
-import akka.actor.{ActorRef, Props}
-import cromwell.backend.impl.htcondor.HtCondorInitializationActor._
-import cromwell.backend.impl.htcondor.HtCondorRuntimeAttributes._
-import cromwell.backend.validation.RuntimeAttributesDefault
-import cromwell.backend.validation.RuntimeAttributesKeys._
-import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendWorkflowDescriptor, BackendWorkflowInitializationActor}
-import cromwell.core.WorkflowOptions
-import wdl4s.TaskCall
-import wdl4s.types.{WdlBooleanType, WdlIntegerType, WdlStringType}
-import wdl4s.values.WdlValue
-
-import scala.concurrent.Future
-import scala.util.Try
-
-object HtCondorInitializationActor {
-  val SupportedKeys = Set(DockerKey, DockerWorkingDirKey, DockerOutputDirKey, FailOnStderrKey,
-    ContinueOnReturnCodeKey, CpuKey, MemoryKey, DiskKey)
-
-  def props(workflowDescriptor: BackendWorkflowDescriptor,
-            calls: Set[TaskCall],
-            configurationDescriptor: BackendConfigurationDescriptor,
-            serviceRegistryActor: ActorRef): Props =
-    Props(new HtCondorInitializationActor(workflowDescriptor, calls, configurationDescriptor, serviceRegistryActor))
-}
-
-class HtCondorInitializationActor(override val workflowDescriptor: BackendWorkflowDescriptor,
-                                  override val calls: Set[TaskCall],
-                                  override val configurationDescriptor: BackendConfigurationDescriptor,
-                                  override val serviceRegistryActor: ActorRef) extends BackendWorkflowInitializationActor {
-
-  override protected def runtimeAttributeValidators: Map[String, (Option[WdlValue]) => Boolean] = Map(
-    DockerKey -> wdlTypePredicate(valueRequired = false, WdlStringType.isCoerceableFrom),
-    DockerWorkingDirKey -> wdlTypePredicate(valueRequired = false, WdlStringType.isCoerceableFrom),
-    DockerOutputDirKey -> wdlTypePredicate(valueRequired = false, WdlStringType.isCoerceableFrom),
-    FailOnStderrKey -> wdlTypePredicate(valueRequired = false, WdlBooleanType.isCoerceableFrom),
-    ContinueOnReturnCodeKey -> continueOnReturnCodePredicate(valueRequired = false),
-    CpuKey -> wdlTypePredicate(valueRequired = false, WdlIntegerType.isCoerceableFrom),
-    MemoryKey -> wdlTypePredicate(valueRequired = false, WdlStringType.isCoerceableFrom),
-    DiskKey -> wdlTypePredicate(valueRequired = false, WdlStringType.isCoerceableFrom)
-  )
-
-  /**
-    * A call which happens before anything else runs
-    */
-  override def beforeAll(): Future[Option[BackendInitializationData]] = Future.successful(None)
-
-  /**
-    * Validate that this WorkflowBackendActor can run all of the calls that it's been assigned
-    */
-  override def validate(): Future[Unit] = {
-    Future {
-      calls foreach { call =>
-        val runtimeAttributes = call.task.runtimeAttributes.attrs
-        val notSupportedAttributes = runtimeAttributes filterKeys { !SupportedKeys.contains(_) }
-
-        if (notSupportedAttributes.nonEmpty) {
-          val notSupportedAttrString = notSupportedAttributes.keys mkString ", "
-          log.warning(s"Key/s [$notSupportedAttrString] is/are not supported by HtCondorBackend. Unsupported attributes will not be part of jobs executions.")
-        }
-      }
-    }
-  }
-
-  override protected def coerceDefaultRuntimeAttributes(options: WorkflowOptions): Try[Map[String, WdlValue]] = {
-    RuntimeAttributesDefault.workflowOptionsDefault(options, HtCondorRuntimeAttributes.coercionMap)
-  }
-}
diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala
deleted file mode 100644
index be5437ce4..000000000
--- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala
+++ /dev/null
@@ -1,379 +0,0 @@
-package cromwell.backend.impl.htcondor
-
-import java.nio.file.attribute.PosixFilePermission
-import java.util.UUID
-
-import akka.actor.{ActorRef, Props}
-import cromwell.backend.BackendJobExecutionActor.{BackendJobExecutionResponse, JobFailedNonRetryableResponse, JobSucceededResponse}
-import cromwell.backend._
-import cromwell.backend.impl.htcondor.caching.CacheActor._
-import cromwell.backend.impl.htcondor.caching.localization.CachedResultLocalization
-import cromwell.backend.io.JobPathsWithDocker
-import cromwell.backend.sfs.{SharedFileSystem, SharedFileSystemExpressionFunctions}
-import cromwell.backend.wdl.Command
-import cromwell.core.path.JavaWriterImplicits._
-import cromwell.core.path.Obsolete._
-import cromwell.core.path.{DefaultPathBuilder, PathBuilder}
-import cromwell.services.keyvalue.KeyValueServiceActor._
-import cromwell.services.metadata.CallMetadataKeys
-import lenthall.util.TryUtil
-import org.apache.commons.codec.digest.DigestUtils
-import wdl4s.EvaluatedTaskInputs
-import wdl4s.parser.MemoryUnit
-import wdl4s.types.{WdlArrayType, WdlFileType}
-import wdl4s.values.WdlArray
-
-import scala.concurrent.{Future, Promise}
-import scala.sys.process.ProcessLogger
-import scala.util.{Failure, Success, Try}
-
-object HtCondorJobExecutionActor {
-  val HtCondorJobIdKey = "htCondor_job_id"
-
-  val pathBuilders = List(DefaultPathBuilder)
-
-  def props(jobDescriptor: BackendJobDescriptor, configurationDescriptor: BackendConfigurationDescriptor, serviceRegistryActor: ActorRef, cacheActorProps: Option[Props]): Props =
-    Props(new HtCondorJobExecutionActor(jobDescriptor, configurationDescriptor, serviceRegistryActor, cacheActorProps))
-}
-
-class HtCondorJobExecutionActor(override val jobDescriptor: BackendJobDescriptor,
-                                override val configurationDescriptor: BackendConfigurationDescriptor,
-                                serviceRegistryActor: ActorRef,
-                                cacheActorProps: Option[Props])
-  extends BackendJobExecutionActor with CachedResultLocalization with SharedFileSystem {
-
-  import HtCondorJobExecutionActor._
-
-  private val tag = s"CondorJobExecutionActor-${jobDescriptor.call.fullyQualifiedName}:"
-  override val pathBuilders: List[PathBuilder] = HtCondorJobExecutionActor.pathBuilders
-
-  implicit val executionContext = context.dispatcher
-
-  lazy val cmds = new HtCondorCommands
-  lazy val extProcess = new HtCondorProcess
-
-  private val fileSystemsConfig = configurationDescriptor.backendConfig.getConfig("filesystems")
-  override val sharedFileSystemConfig = fileSystemsConfig.getConfig("local")
-  private val workflowDescriptor = jobDescriptor.workflowDescriptor
-  private val jobPaths = new JobPathsWithDocker(jobDescriptor.key, workflowDescriptor, configurationDescriptor.backendConfig)
-
-  // Files
-  private val executionDir = jobPaths.callExecutionRoot
-  private val returnCodePath = jobPaths.returnCode
-  private val stdoutPath = jobPaths.stdout
-  private val stderrPath = jobPaths.stderr
-  private val scriptPath = jobPaths.script
-
-  // stdout stderr writers for submit file logs
-  private val submitFilePath = executionDir.resolve("submitfile")
-  private val submitFileStderr = executionDir.resolve("submitfile.stderr")
-  private val submitFileStdout = executionDir.resolve("submitfile.stdout")
-  private val htCondorLog = executionDir.resolve(s"${jobDescriptor.call.unqualifiedName}.log")
-
-  private lazy val stdoutWriter = extProcess.untailedWriter(submitFileStdout)
-  private lazy val stderrWriter = extProcess.tailedWriter(100, submitFileStderr)
-
-  private val call = jobDescriptor.key.call
-  private val callEngineFunction = SharedFileSystemExpressionFunctions(jobPaths, pathBuilders)
-
-  private val lookup = jobDescriptor.fullyQualifiedInputs.apply _
-
-  private val runtimeAttributes = {
-    val evaluateAttrs = call.task.runtimeAttributes.attrs mapValues (_.evaluate(lookup, callEngineFunction))
-    // Fail the call if runtime attributes can't be evaluated
-    val runtimeMap = TryUtil.sequenceMap(evaluateAttrs, "Runtime attributes evaluation").get
-    HtCondorRuntimeAttributes(runtimeMap, jobDescriptor.workflowDescriptor.workflowOptions)
-  }
-
-  private val cacheActor = cacheActorProps match {
-    case Some(props) => Some(context.actorOf(props, s"CacheActor-${jobDescriptor.call.fullyQualifiedName}"))
-    case None => None
-  }
-
-  log.debug("{} Calculating hash for current job.", tag)
-  lazy private val jobHash = calculateHash
-
-  private val executionResponse = Promise[BackendJobExecutionResponse]()
-
-  // Message sent (by self, to self) wrapping over the response produced by HtCondor
-  private final case class JobExecutionResponse(resp: BackendJobExecutionResponse)
-
-  // Message sent (by self, to self) to trigger a status check to HtCondor
-  private final case class TrackTaskStatus(id: String)
-
-  private var condorJobId: Option[String] = None
-
-  private val pollingInterval = configurationDescriptor.backendConfig.getInt("poll-interval")
-
-  override def receive = super.receive orElse {
-    case JobExecutionResponse(resp) =>
-      log.debug("{}: Completing job [{}] with response: [{}]", tag, jobDescriptor.key, resp)
-      executionResponse trySuccess resp
-      ()
-    case TrackTaskStatus(id) =>
-      // Avoid the redundant status check if the response is already completed (e.g. in case of abort)
-      if (!executionResponse.isCompleted) trackTask(id)
-
-    // Messages received from Caching actor
-    case ExecutionResultFound(succeededResponse) =>
-      executionResponse trySuccess localizeCachedResponse(succeededResponse)
-      ()
-    case ExecutionResultNotFound => prepareAndExecute()
-    case ExecutionResultStored(hash) => log.debug("{} Cache entry was stored for Job with hash {}.", tag, hash)
-    case ExecutionResultAlreadyExist => log.warning("{} Cache entry for hash {} already exist.", tag, jobHash)
-
-    // Messages received from KV actor
-    case KvPair(scopedKey, Some(jobId)) if scopedKey.key == HtCondorJobIdKey =>
-      log.info("{} Found job id {}. Trying to recover job now.", tag, jobId)
-      self ! TrackTaskStatus(jobId)
-    case KvKeyLookupFailed(_) =>
-      log.debug("{} Job id not found. Falling back to execute.", tag)
-      execute
-      // -Ywarn-value-discard
-      ()
-    case KvFailure(_, e) =>
-      log.error("{} Failure attempting to look up HtCondor job id. Exception message: {}. Falling back to execute.", tag, e.getMessage)
-      execute
-      // -Ywarn-value-discard
-      ()
-  }
-
-  /**
-    * Restart or resume a previously-started job.
-    */
-  override def recover: Future[BackendJobExecutionResponse] = {
-    log.warning("{} Trying to recover job {}.", tag, jobDescriptor.key.call.fullyQualifiedName)
-    serviceRegistryActor ! KvGet(ScopedKey(jobDescriptor.workflowDescriptor.id,
-      KvJobKey(jobDescriptor.key.call.fullyQualifiedName, jobDescriptor.key.index, jobDescriptor.key.attempt),
-      HtCondorJobIdKey))
-    executionResponse.future
-  }
-
-  /**
-    * Execute a new job.
-    */
-  override def execute: Future[BackendJobExecutionResponse] = {
-    log.debug("{} Checking if hash {{}} is in the cache.", tag, jobHash)
-    cacheActor match {
-      case Some(actorRef) => actorRef ! ReadExecutionResult(jobHash)
-      case None => prepareAndExecute()
-    }
-    executionResponse.future
-  }
-
-  /**
-    * Abort a running job.
-    */
-  override def abort(): Unit = {
-    // Nothing to do in case `condorJobId` is not defined
-    condorJobId foreach { id =>
-      log.info("{}: Aborting job [{}:{}].", tag, jobDescriptor.key.tag, id)
-      val abortProcess = new HtCondorProcess
-      val argv = Seq(HtCondorCommands.Remove, id)
-      val process = abortProcess.externalProcess(argv)
-      val exitVal = process.exitValue()
-      if (exitVal == 0)
-        log.info("{}: Job {} successfully killed and removed from the queue.", tag, id)
-      else
-        log.error("{}: Failed to kill / remove job {}. Exit Code: {}, Stderr: {}", tag, id, exitVal, abortProcess.processStderr)
-    }
-  }
-
-  private def executeTask(): Unit = {
-    val argv = Seq(HtCondorCommands.Submit, submitFilePath.toString)
-    val process = extProcess.externalProcess(argv, ProcessLogger(stdoutWriter.writeWithNewline, stderrWriter.writeWithNewline))
-    val condorReturnCode = process.exitValue() // blocks until process (i.e. condor submission) finishes
-    log.debug("{} Return code of condor submit command: {}", tag, condorReturnCode)
-
-    List(stdoutWriter.writer, stderrWriter.writer).foreach(_.flushAndClose())
-
-    condorReturnCode match {
-      case 0 if File(submitFileStderr).lines.toList.isEmpty =>
-        log.info("{} {} submitted to HtCondor. Waiting for the job to complete via. RC file status.", tag, jobDescriptor.call.fullyQualifiedName)
-        val job = HtCondorCommands.SubmitOutputPattern.r
-        //Number of lines in stdout for submit job will be 3 at max therefore reading all lines at once.
-        log.debug(s"{} Output of submit process : {}", tag, File(submitFileStdout).lines.toList)
-        val line = File(submitFileStdout).lines.toList.last
-        line match {
-          case job(jobId, clusterId) =>
-            val overallJobIdentifier = s"$clusterId.${jobId.toInt - 1}" // Condor has 0 based indexing on the jobs, probably won't work on stuff like `queue 150`
-            log.info("{} {} mapped to HtCondor JobID: {}", tag, jobDescriptor.call.fullyQualifiedName, overallJobIdentifier)
-            serviceRegistryActor ! KvPut(KvPair(ScopedKey(jobDescriptor.workflowDescriptor.id,
-              KvJobKey(jobDescriptor.key.call.fullyQualifiedName, jobDescriptor.key.index, jobDescriptor.key.attempt),
-              HtCondorJobIdKey), Option(overallJobIdentifier)))
-            condorJobId = Option(overallJobIdentifier)
-            self ! TrackTaskStatus(overallJobIdentifier)
-
-          case _ => self ! JobExecutionResponse(JobFailedNonRetryableResponse(jobDescriptor.key,
-            new IllegalStateException("Failed to retrieve job(id) and cluster id"), Option(condorReturnCode)))
-        }
-
-      case 0 =>
-        log.error(s"Unexpected! Received return code for condor submission as 0, although stderr file is non-empty: {}", File(submitFileStderr).lines)
-        self ! JobExecutionResponse(JobFailedNonRetryableResponse(jobDescriptor.key,
-          new IllegalStateException(s"Execution process failed. HtCondor returned zero status code but non empty stderr file: $condorReturnCode"),
-          Option(condorReturnCode)))
-
-      case nonZeroExitCode: Int =>
-        self ! JobExecutionResponse(JobFailedNonRetryableResponse(jobDescriptor.key,
-          new IllegalStateException(s"Execution process failed. HtCondor returned non zero status code: $condorReturnCode"), Option(condorReturnCode)))
-    }
-  }
-
-  private def trackTask(jobIdentifier: String): Unit = {
-    val jobReturnCode = Try(extProcess.jobReturnCode(jobIdentifier, returnCodePath))
-    log.debug("{} Process complete. RC file now exists with value: {}", tag, jobReturnCode)
-
-    jobReturnCode match {
-      case Success(None) =>
-        import scala.concurrent.duration._
-        // Job is still running in HtCondor. Check back again after `pollingInterval` seconds
-        context.system.scheduler.scheduleOnce(pollingInterval.seconds, self, TrackTaskStatus(jobIdentifier))
-        ()
-      case Success(Some(rc)) if runtimeAttributes.continueOnReturnCode.continueFor(rc) => self ! JobExecutionResponse(processSuccess(rc))
-      case Success(Some(rc)) => self ! JobExecutionResponse(JobFailedNonRetryableResponse(jobDescriptor.key,
-        new IllegalStateException("Job exited with invalid return code: " + rc), Option(rc)))
-      case Failure(error) => self ! JobExecutionResponse(JobFailedNonRetryableResponse(jobDescriptor.key, error, None))
-    }
-  }
-
-  private def processSuccess(rc: Int): BackendJobExecutionResponse = {
-    evaluateOutputs(callEngineFunction, outputMapper(jobPaths)) match {
-      case Success(outputs) =>
-        val succeededResponse = JobSucceededResponse(jobDescriptor.key, Some(rc), outputs, None, Seq.empty)
-        log.debug("{} Storing data into cache for hash {}.", tag, jobHash)
-        // If cache fails to store data for any reason it should not stop the workflow/task execution but log the issue.
-        cacheActor foreach { _ ! StoreExecutionResult(jobHash, succeededResponse) }
-        succeededResponse
-      case Failure(e) =>
-        val message = Option(e.getMessage) map {
-          ": " + _
-        } getOrElse ""
-        JobFailedNonRetryableResponse(jobDescriptor.key, new Throwable("Failed post processing of outputs" + message, e), Option(rc))
-    }
-  }
-
-  private def calculateHash: String = {
-    val cmd = Command.instantiate(jobDescriptor, callEngineFunction) match {
-      case Success(command) => command
-      case Failure(ex) =>
-        val errMsg = s"$tag Cannot instantiate job command for caching purposes due to ${ex.getMessage}."
-        log.error(ex.getCause, errMsg)
-        throw new IllegalStateException(errMsg, ex.getCause)
-    }
-    val str = Seq[Any](cmd,
-      runtimeAttributes.failOnStderr,
-      runtimeAttributes.dockerImage.getOrElse(""),
-      runtimeAttributes.dockerWorkingDir.getOrElse(""),
-      runtimeAttributes.dockerOutputDir.getOrElse(""),
-      runtimeAttributes.cpu.toString,
-      runtimeAttributes.memory.toString,
-      runtimeAttributes.disk.toString).mkString
-    DigestUtils.md5Hex(str)
-  }
-
-  private def createExecutionFolderAndScript(): Unit = {
-    try {
-      log.debug("{} Creating execution folder: {}", tag, executionDir)
-      executionDir.toString.toFile.createIfNotExists(asDirectory = true, createParents = true)
-
-      log.debug("{} Resolving job command", tag)
-      val command = localizeInputs(jobPaths.callInputsRoot, runtimeAttributes.dockerImage.isDefined)(jobDescriptor.inputDeclarations) flatMap {
-        localizedInputs => resolveJobCommand(localizedInputs)
-      }
-
-      log.debug("{} Creating bash script for executing command: {}", tag, command)
-      cmds.writeScript(command.get, scriptPath.toAbsolutePath, executionDir.toAbsolutePath) // Writes the bash script for executing the command
-      File(scriptPath).addPermission(PosixFilePermission.OWNER_EXECUTE) // Add executable permissions to the script.
-      //TODO: Need to append other runtime attributes from Wdl to Condor submit file
-      val attributes: Map[String, Any] = Map(HtCondorRuntimeKeys.Executable -> scriptPath.toAbsolutePath,
-          HtCondorRuntimeKeys.InitialWorkingDir -> jobPaths.callExecutionRoot.toAbsolutePath,
-          HtCondorRuntimeKeys.Output -> stdoutPath.toAbsolutePath,
-          HtCondorRuntimeKeys.Error -> stderrPath.toAbsolutePath,
-          HtCondorRuntimeKeys.Log -> htCondorLog.toAbsolutePath,
-          HtCondorRuntimeKeys.LogXml -> true,
-          HtCondorRuntimeKeys.LeaveInQueue -> true,
-          HtCondorRuntimeKeys.Cpu -> runtimeAttributes.cpu,
-          HtCondorRuntimeKeys.Memory -> runtimeAttributes.memory.to(MemoryUnit.MB).amount.toLong,
-          HtCondorRuntimeKeys.Disk -> runtimeAttributes.disk.to(MemoryUnit.KB).amount.toLong
-        )
-
-      cmds.generateSubmitFile(submitFilePath, attributes, runtimeAttributes.nativeSpecs) // This writes the condor submit file
-      ()
-
-    } catch {
-      case ex: Exception =>
-        log.error(ex, "Failed to prepare task: " + ex.getMessage)
-        throw ex
-    }
-  }
-
-  private def resolveJobCommand(localizedInputs: EvaluatedTaskInputs): Try[String] = {
-    val command = if (runtimeAttributes.dockerImage.isDefined) {
-      modifyCommandForDocker(call.task.instantiateCommand(localizedInputs, callEngineFunction, identity), localizedInputs)
-    } else {
-      call.task.instantiateCommand(localizedInputs, callEngineFunction, identity)
-    }
-    command match {
-      case Success(cmd) => tellMetadata(Map("command" -> cmd))
-      case Failure(ex) =>
-        log.error("{} failed to resolve command due to exception:{}", tag, ex)
-        tellMetadata(Map(s"${CallMetadataKeys.Failures}[${UUID.randomUUID().toString}]" -> ex.getMessage))
-    }
-    command
-  }
-
-  /**
-    * Fire and forget data to the metadata service
-    */
-  private def tellMetadata(metadataKeyValues: Map[String, Any]): Unit = {
-    import cromwell.services.metadata.MetadataService.implicits.MetadataAutoPutter
-    serviceRegistryActor.putMetadata(jobDescriptor.workflowDescriptor.id, Option(jobDescriptor.key), metadataKeyValues)
-  }
-
-  private def modifyCommandForDocker(jobCmd: Try[String], localizedInputs: EvaluatedTaskInputs): Try[String] = {
-    Try {
-      val dockerInputDataVol = localizedInputs.values.collect {
-        case file if file.wdlType == WdlFileType =>
-          val limit = file.valueString.lastIndexOf("/")
-          Seq(file.valueString.substring(0, limit))
-        case files if files.wdlType == WdlArrayType(WdlFileType) => files.asInstanceOf[WdlArray].value map { file =>
-          val limit = file.valueString.lastIndexOf("/")
-          file.valueString.substring(0, limit)
-        }
-      }.flatten.toSeq
-
-      log.debug("{} List of input volumes: {}", tag, dockerInputDataVol.mkString(","))
-      val dockerCmd = configurationDescriptor.backendConfig.getString("docker.cmd")
-      val defaultWorkingDir = configurationDescriptor.backendConfig.getString("docker.defaultWorkingDir")
-      val defaultOutputDir = configurationDescriptor.backendConfig.getString("docker.defaultOutputDir")
-      val dockerVolume = "-v %s:%s"
-      val dockerVolumeInputs = s"$dockerVolume:ro"
-      // `v.get` is safe below since we filtered the list earlier with only defined elements
-      val inputVolumes = dockerInputDataVol.distinct.map(v => dockerVolumeInputs.format(v, v)).mkString(" ")
-      val outputVolume = dockerVolume.format(executionDir.toAbsolutePath.toString, runtimeAttributes.dockerOutputDir.getOrElse(defaultOutputDir))
-      val workingDir = dockerVolume.format(executionDir.toAbsolutePath.toString, runtimeAttributes.dockerWorkingDir.getOrElse(defaultWorkingDir))
-      val cmd = dockerCmd.format(runtimeAttributes.dockerWorkingDir.getOrElse(defaultWorkingDir), workingDir, inputVolumes, outputVolume, runtimeAttributes.dockerImage.get, jobCmd.get)
-      log.debug("{} Docker command line to be used for task execution: {}.", tag, cmd)
-      cmd
-    }
-  }
-
-  private def prepareAndExecute(): Unit = {
-    try {
-      createExecutionFolderAndScript()
-      executeTask()
-    } catch {
-      case e: Exception => self ! JobExecutionResponse(JobFailedNonRetryableResponse(jobDescriptor.key, e, None))
-    }
-  }
-
-  private def localizeCachedResponse(succeededResponse: JobSucceededResponse): BackendJobExecutionResponse = {
-    Try(localizeCachedOutputs(executionDir, succeededResponse.jobOutputs)) match {
-      case Success(outputs) =>
-        executionDir.toString.toFile.createIfNotExists(asDirectory = true, createParents = true)
-        JobSucceededResponse(jobDescriptor.key, succeededResponse.returnCode, outputs, None, Seq.empty)
-      case Failure(exception) => JobFailedNonRetryableResponse(jobDescriptor.key, exception, None)
-    }
-  }
-}
diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributes.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributes.scala
deleted file mode 100644
index c96fd2f0a..000000000
--- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributes.scala
+++ /dev/null
@@ -1,124 +0,0 @@
-package cromwell.backend.impl.htcondor
-
-
-import cats.data.Validated.{Invalid, Valid}
-import cats.syntax.cartesian._
-import cats.syntax.validated._
-import cromwell.backend.MemorySize
-import cromwell.backend.validation.ContinueOnReturnCode
-import cromwell.backend.validation.RuntimeAttributesDefault._
-import cromwell.backend.validation.RuntimeAttributesKeys._
-import cromwell.backend.validation.RuntimeAttributesValidation._
-import cromwell.core._
-import lenthall.validation.ErrorOr._
-import lenthall.exception.MessageAggregation
-import wdl4s.types._
-import wdl4s.values.{WdlArray, WdlBoolean, WdlInteger, WdlString, WdlValue}
-
-
-object HtCondorRuntimeAttributes {
-  private val FailOnStderrDefaultValue = false
-  private val ContinueOnRcDefaultValue = 0
-  private val CpuDefaultValue = 1
-  private val MemoryDefaultValue = "0.512 GB"
-  private val DisksDefaultValue = "1.024 GB"
-
-  val DockerWorkingDirKey = "dockerWorkingDir"
-  val DockerOutputDirKey = "dockerOutputDir"
-  val DiskKey = "disk"
-  val NativeSpecsKey = "nativeSpecs"
-
-  val staticDefaults = Map(
-    FailOnStderrKey -> WdlBoolean(FailOnStderrDefaultValue),
-    ContinueOnReturnCodeKey -> WdlInteger(ContinueOnRcDefaultValue),
-    CpuKey -> WdlInteger(CpuDefaultValue),
-    MemoryKey -> WdlString(MemoryDefaultValue),
-    DiskKey -> WdlString(DisksDefaultValue)
-  )
-
-  private[htcondor] val coercionMap: Map[String, Set[WdlType]] = Map (
-    FailOnStderrKey -> Set[WdlType](WdlBooleanType),
-    ContinueOnReturnCodeKey -> ContinueOnReturnCode.validWdlTypes,
-    DockerKey -> Set(WdlStringType),
-    DockerWorkingDirKey -> Set(WdlStringType),
-    DockerOutputDirKey -> Set(WdlStringType),
-    CpuKey -> Set(WdlIntegerType),
-    MemoryKey -> Set(WdlStringType),
-    DiskKey -> Set(WdlStringType),
-    NativeSpecsKey -> Set(WdlArrayType(WdlStringType))
-  )
-
-  def apply(attrs: Map[String, WdlValue], options: WorkflowOptions): HtCondorRuntimeAttributes = {
-    // Fail now if some workflow options are specified but can't be parsed correctly
-    val defaultFromOptions = workflowOptionsDefault(options, coercionMap).get
-    val withDefaultValues = withDefaults(attrs, List(defaultFromOptions, staticDefaults))
-
-    val docker = validateDocker(withDefaultValues.get(DockerKey), None.validNel)
-    val dockerWorkingDir = validateDockerWorkingDir(withDefaultValues.get(DockerWorkingDirKey), None.validNel)
-    val dockerOutputDir = validateDockerOutputDir(withDefaultValues.get(DockerOutputDirKey), None.validNel)
-    val failOnStderr = validateFailOnStderr(withDefaultValues.get(FailOnStderrKey), noValueFoundFor(FailOnStderrKey))
-    val continueOnReturnCode = validateContinueOnReturnCode(withDefaultValues.get(ContinueOnReturnCodeKey), noValueFoundFor(ContinueOnReturnCodeKey))
-    val cpu = validateCpu(withDefaultValues.get(CpuKey), noValueFoundFor(CpuKey))
-    val memory = validateMemory(withDefaultValues.get(MemoryKey), noValueFoundFor(MemoryKey))
-    val disk = validateDisk(withDefaultValues.get(DiskKey), noValueFoundFor(DiskKey))
-    val nativeSpecs = validateNativeSpecs(withDefaultValues.get(NativeSpecsKey), None.validNel)
-
-    (continueOnReturnCode |@| docker |@| dockerWorkingDir |@| dockerOutputDir |@| failOnStderr |@| cpu |@| memory |@| disk |@| nativeSpecs) map {
-      new HtCondorRuntimeAttributes(_, _, _, _, _, _, _, _, _)
-    } match {
-      case Valid(x) => x
-      case Invalid(nel) => throw new RuntimeException with MessageAggregation {
-        override def exceptionContext: String = "Runtime attribute validation failed"
-        override def errorMessages: Traversable[String] = nel.toList
-      }
-    }
-  }
-
-  private def validateDockerWorkingDir(dockerWorkingDir: Option[WdlValue], onMissingKey: => ErrorOr[Option[String]]): ErrorOr[Option[String]] = {
-    dockerWorkingDir match {
-      case Some(WdlString(s)) => Some(s).validNel
-      case None => onMissingKey
-      case _ => s"Expecting $DockerWorkingDirKey runtime attribute to be a String".invalidNel
-    }
-  }
-
-  private def validateDockerOutputDir(dockerOutputDir: Option[WdlValue], onMissingKey: => ErrorOr[Option[String]]): ErrorOr[Option[String]] = {
-    dockerOutputDir match {
-      case Some(WdlString(s)) => Some(s).validNel
-      case None => onMissingKey
-      case _ => s"Expecting $DockerOutputDirKey runtime attribute to be a String".invalidNel
-    }
-  }
-
-  private def validateDisk(value: Option[WdlValue], onMissingKey: => ErrorOr[MemorySize]): ErrorOr[MemorySize] = {
-    val diskWrongFormatMsg = s"Expecting $DiskKey runtime attribute to be an Integer or String with format '8 GB'. Exception: %s"
-
-    value match {
-      case Some(i: WdlInteger) => parseMemoryInteger(i)
-      case Some(s: WdlString) => parseMemoryString(s)
-      case Some(_) => String.format(diskWrongFormatMsg, "Not supported WDL type value").invalidNel
-      case None => onMissingKey
-    }
-  }
-
-  private def validateNativeSpecs(value: Option[WdlValue], onMissingKey: => ErrorOr[Option[Array[String]]]): ErrorOr[Option[Array[String]]] = {
-    val nativeSpecsWrongFormatMsg = s"Expecting $NativeSpecsKey runtime attribute to be an Array of Strings. Exception: %s"
-    value match {
-      case Some(ns: WdlArray) if ns.wdlType.memberType.equals(WdlStringType) =>
-        val nsa = ns.value.map { value => value.valueString }.toArray
-        Option(nsa).validNel
-      case Some(_) => String.format(nativeSpecsWrongFormatMsg, "Not supported WDL type value").invalidNel
-      case None => onMissingKey
-    }
-  }
-}
-
-case class HtCondorRuntimeAttributes(continueOnReturnCode: ContinueOnReturnCode,
-                                     dockerImage: Option[String],
-                                     dockerWorkingDir: Option[String],
-                                     dockerOutputDir: Option[String],
-                                     failOnStderr: Boolean,
-                                     cpu: Int,
-                                     memory: MemorySize,
-                                     disk: MemorySize,
-                                     nativeSpecs: Option[Array[String]])
diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorWrapper.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorWrapper.scala
deleted file mode 100644
index cbc9c24e9..000000000
--- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorWrapper.scala
+++ /dev/null
@@ -1,153 +0,0 @@
-package cromwell.backend.impl.htcondor
-
-import com.typesafe.scalalogging.StrictLogging
-import cromwell.backend.impl.htcondor
-import cromwell.core.path.JavaWriterImplicits._
-import cromwell.core.path.Obsolete._
-import cromwell.core.path.{Path, TailedWriter, UntailedWriter}
-
-import scala.sys.process._
-
-object JobStatus {
-  val MapOfStatuses = Map(
-    0 -> Created, // This is actually `unexpanded` in HtCondor, not sure what that actually means
-    1 -> Created, // Idle
-    2 -> Running,
-    3 -> Removed,
-    4 -> Completed,
-    5 -> Failed, // SystemOnHold
-    6 -> SubmissionError // Also the default
-  )
-
-  def fromCondorStatusCode(statusCode: Int): JobStatus = {
-    MapOfStatuses.getOrElse(statusCode, SubmissionError) // By default we return SubmissionError
-  }
-
-  def isTerminal(jobStatus: JobStatus): Boolean = jobStatus.isInstanceOf[TerminalJobStatus]
-}
-
-sealed trait JobStatus
-sealed trait TerminalJobStatus extends JobStatus
-case object Created extends JobStatus
-case object Running extends JobStatus
-case object Completed extends TerminalJobStatus
-case object Removed extends TerminalJobStatus
-case object Failed extends TerminalJobStatus
-case object Aborted extends TerminalJobStatus
-case object SubmissionError extends TerminalJobStatus
-
-
-object HtCondorCommands {
-  val SubmitOutputPattern = "(\\d*) job\\(s\\) submitted to cluster (\\d*)\\."
-  val Submit = "condor_submit"
-  val Remove = "condor_rm"
-  private val JobStatus = "condor_q %s -autoformat JobStatus"
-  def generateJobStatusCommand(jobId: String): String = HtCondorCommands.JobStatus.format(jobId)
-}
-
-class HtCondorCommands extends StrictLogging {
-
-  /**
-    * Writes the script file containing the user's command from the WDL as well
-    * as some extra shell code for monitoring jobs
-    */
-  def writeScript(instantiatedCommand: String, filePath: Path, containerRoot: Path): Unit = {
-    logger.debug(s"Writing bash script for execution. Command: $instantiatedCommand.")
-    val scriptBody = s"""
-
-#!/bin/sh
-cd $containerRoot
-$instantiatedCommand
-echo $$? > rc
-
-""".trim + "\n"
-    File(filePath).write(scriptBody)
-    ()
-  }
-
-  def generateSubmitFile(path: Path, attributes: Map[String, Any], nativeSpecs: Option[Array[String]]): String = {
-    def htCondorSubmitCommand(filePath: Path) = {
-      s"${HtCondorCommands.Submit} ${filePath.toString}"
-    }
-
-    val submitFileWriter = path.untailed
-    attributes.foreach { attribute => submitFileWriter.writeWithNewline(s"${attribute._1}=${attribute._2}") }
-    //Native specs is intended for attaching HtCondor native configuration such as 'requirements' and 'rank' definition
-    //directly to the submit file.
-    nativeSpecs foreach { _.foreach { submitFileWriter.writeWithNewline } }
-    submitFileWriter.writeWithNewline(HtCondorRuntimeKeys.Queue)
-    submitFileWriter.writer.flushAndClose()
-    logger.debug(s"submit file name is : $path")
-    logger.debug(s"content of file is : ${File(path).lines.toList}")
-    htCondorSubmitCommand(path)
-  }
-
-}
-
-class HtCondorProcess extends StrictLogging {
-  private val stdout = new StringBuilder
-  private val stderr = new StringBuilder
-
-  def processLogger: ProcessLogger = ProcessLogger(s => { stdout append s; () }, s => { stderr append s; () })
-  def processStdout: String = stdout.toString().trim
-  def processStderr: String = stderr.toString().trim
-  def commandList(command: String): Seq[String] = Seq("/bin/bash",command)
-  def untailedWriter(path: Path): UntailedWriter = path.untailed
-  def tailedWriter(limit: Int, path: Path): TailedWriter = path.tailed(limit)
-  def externalProcess(cmdList: Seq[String], processLogger: ProcessLogger = processLogger): Process = cmdList.run(processLogger)
-
-  /**
-    * Returns the RC of this job if it has finished.
-    */
-  def jobReturnCode(jobId: String, returnCodeFilePath: Path): Option[Int] = {
-
-    checkStatus(jobId) match {
-      case status if JobStatus.isTerminal(status) =>
-        Files.exists(returnCodeFilePath) match {
-          case true => Option(File(returnCodeFilePath).contentAsString.stripLineEnd.toInt)
-          case false =>
-            val msg = s"JobStatus from Condor is terminal ($status) and no RC file exists!"
-            logger.debug(msg)
-            throw new IllegalStateException(msg)
-        }
-      case nonTerminalStatus => None
-    }
-  }
-
-  private def checkStatus(jobId: String): JobStatus = {
-    val htCondorProcess = new HtCondorProcess
-    val commandArgv = HtCondorCommands.generateJobStatusCommand(jobId).split(" ").toSeq
-    val process = htCondorProcess.externalProcess(commandArgv)
-    val returnCode = process.exitValue()
-    returnCode match {
-      case 0 =>
-        val stdout = htCondorProcess.processStdout
-        // If stdout is empty, that means the job got removed from the queue. Return Completed in that case
-        val status = if (stdout.isEmpty) htcondor.Completed else JobStatus.fromCondorStatusCode(htCondorProcess.processStdout.toInt)
-        logger.info("Condor JobId {} current status: {}", jobId, status)
-        status
-      case errorCode =>
-        val msg = "Could not retreive status from the queue: " + htCondorProcess.processStderr
-        logger.error(msg)
-        throw new IllegalStateException(msg)
-    }
-  }
-
-}
-
-object HtCondorRuntimeKeys {
-  val Executable = "executable"
-  val Arguments = "arguments"
-  val Error = "error"
-  val Output = "output"
-  val Log = "log"
-  val Queue = "queue"
-  val Rank = "rank"
-  val Requirements = "requirements"
-  val Cpu = "request_cpus"
-  val Memory = "request_memory"
-  val Disk = "request_disk"
-  val LogXml = "log_xml"
-  val LeaveInQueue = "leave_in_queue"
-  val InitialWorkingDir = "Iwd"
-}
diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/CacheActor.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/CacheActor.scala
deleted file mode 100644
index 74fb3ade5..000000000
--- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/CacheActor.scala
+++ /dev/null
@@ -1,51 +0,0 @@
-package cromwell.backend.impl.htcondor.caching
-
-import akka.actor.{Actor, ActorLogging}
-import cromwell.backend.BackendJobExecutionActor.JobSucceededResponse
-import cromwell.backend.impl.htcondor.caching.CacheActor._
-import cromwell.backend.impl.htcondor.caching.exception.{CachedResultAlreadyExistException, CachedResultNotFoundException}
-import cromwell.backend.impl.htcondor.caching.model.CachedExecutionResult
-
-object CacheActor {
-
-  trait CacheActorCommand
-  case class ReadExecutionResult(hash: String) extends CacheActorCommand
-  case class StoreExecutionResult(hash: String, succeededResponse: JobSucceededResponse) extends CacheActorCommand
-
-  trait CacheActorResponse
-  case class ExecutionResultFound(succeededResponse: JobSucceededResponse) extends CacheActorResponse
-  case object ExecutionResultNotFound extends CacheActorResponse
-  case class ExecutionResultStored(hash: String) extends CacheActorResponse
-  case object ExecutionResultAlreadyExist extends CacheActorResponse
-
-}
-
-trait CacheActor extends Actor with ActorLogging {
-  def tag: String = "[CacheActor]"
-  def forceRewrite: Boolean = false
-
-  override def receive: Receive = {
-    case ReadExecutionResult(hash) =>
-      try {
-        val executionResult = readExecutionResult(hash)
-        log.info(s"{} Execution result found in cache for hash {}. Returning result: {}.", tag, hash, executionResult)
-        sender() ! ExecutionResultFound(executionResult.succeededResponse)
-      } catch {
-        case ex: CachedResultNotFoundException => sender() ! ExecutionResultNotFound
-      }
-
-    case StoreExecutionResult(hash, succeededResult) =>
-      try {
-        storeExecutionResult(CachedExecutionResult(hash, succeededResult))
-        log.info(s"{} Cache entry for job [{}] stored successfully.", tag, hash)
-        sender() ! ExecutionResultStored(hash)
-      } catch {
-        case ex: CachedResultAlreadyExistException => sender() ! ExecutionResultAlreadyExist
-      }
-  }
-
-  def readExecutionResult(hash: String): CachedExecutionResult
-
-  def storeExecutionResult(cachedExecutionResult: CachedExecutionResult): Unit
-
-}
diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/CacheActorFactory.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/CacheActorFactory.scala
deleted file mode 100644
index 9dd99cc24..000000000
--- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/CacheActorFactory.scala
+++ /dev/null
@@ -1,9 +0,0 @@
-package cromwell.backend.impl.htcondor.caching
-
-import akka.actor.Props
-
-trait CacheActorFactory {
-
-  def getCacheActorProps(forceRewrite: Boolean): Props
-
-}
diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/exception/CachedResultAlreadyExistException.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/exception/CachedResultAlreadyExistException.scala
deleted file mode 100644
index 387252809..000000000
--- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/exception/CachedResultAlreadyExistException.scala
+++ /dev/null
@@ -1,5 +0,0 @@
-package cromwell.backend.impl.htcondor.caching.exception
-
-class CachedResultNotFoundException(message: String) extends RuntimeException(message)
-
-class CachedResultAlreadyExistException(message: String) extends RuntimeException(message)
diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/localization/CachedResultLocalization.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/localization/CachedResultLocalization.scala
deleted file mode 100644
index 027b0b287..000000000
--- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/localization/CachedResultLocalization.scala
+++ /dev/null
@@ -1,38 +0,0 @@
-package cromwell.backend.impl.htcondor.caching.localization
-
-import cromwell.core._
-import cromwell.core.path.Obsolete._
-import cromwell.core.path.Path
-import wdl4s.types.{WdlArrayType, WdlFileType}
-import wdl4s.values.{WdlArray, WdlSingleFile, WdlValue}
-
-trait CachedResultLocalization {
-  private[localization] def localizePathViaSymbolicLink(originalPath: Path, executionPath: Path): Path = {
-    if (File(originalPath).isDirectory) throw new UnsupportedOperationException("Cannot localize directory with symbolic links.")
-    else {
-      File(executionPath).parent.createPermissionedDirectories()
-      Files.createSymbolicLink(executionPath, originalPath.toAbsolutePath)
-    }
-  }
-
-  private[localization] def localizeCachedFile(executionPath: Path, output: WdlValue): WdlSingleFile = {
-    val origPath = Paths.get(output.valueString)
-    val newPath = executionPath.toAbsolutePath.resolve(origPath.getFileName)
-    val slPath = localizePathViaSymbolicLink(origPath, newPath)
-    WdlSingleFile(slPath.toString)
-  }
-
-  def localizeCachedOutputs(executionPath: Path, outputs: CallOutputs): CallOutputs = {
-    outputs map { case (lqn, jobOutput) =>
-      jobOutput.wdlValue.wdlType match {
-        case WdlFileType => (lqn -> JobOutput(localizeCachedFile(executionPath, jobOutput.wdlValue)))
-        case WdlArrayType(WdlFileType) =>
-          val newArray: Seq[WdlSingleFile] = jobOutput.wdlValue.asInstanceOf[WdlArray].value map {
-            localizeCachedFile(executionPath, _)
-          }
-          (lqn -> JobOutput(WdlArray(WdlArrayType(WdlFileType), newArray)))
-        case _ => (lqn, jobOutput)
-      }
-    }
-  }
-}
diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/model/CachedExecutionResult.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/model/CachedExecutionResult.scala
deleted file mode 100644
index 1b023fd41..000000000
--- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/model/CachedExecutionResult.scala
+++ /dev/null
@@ -1,6 +0,0 @@
-package cromwell.backend.impl.htcondor.caching.model
-
-import cromwell.backend.BackendJobExecutionActor.JobSucceededResponse
-
-case class CachedExecutionResult(hash: String, succeededResponse: JobSucceededResponse)
-
diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/MongoCacheActor.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/MongoCacheActor.scala
deleted file mode 100644
index cf6cf1151..000000000
--- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/MongoCacheActor.scala
+++ /dev/null
@@ -1,88 +0,0 @@
-package cromwell.backend.impl.htcondor.caching.provider.mongodb
-
-import com.mongodb.DBObject
-import com.mongodb.casbah.MongoCollection
-import com.mongodb.casbah.commons.{MongoDBObject, TypeImports}
-import com.mongodb.util.JSON
-import cromwell.backend.BackendJobExecutionActor.JobSucceededResponse
-import cromwell.backend.impl.htcondor.caching.CacheActor
-import cromwell.backend.impl.htcondor.caching.exception.{CachedResultAlreadyExistException, CachedResultNotFoundException}
-import cromwell.backend.impl.htcondor.caching.model.CachedExecutionResult
-import cromwell.backend.impl.htcondor.caching.provider.mongodb.model.{KryoSerializedObject, MongoCachedExecutionResult}
-import cromwell.backend.impl.htcondor.caching.provider.mongodb.serialization.KryoSerDe
-
-class MongoCacheActor(collection: MongoCollection,
-                      override val forceRewrite: Boolean = false) extends CacheActor with KryoSerDe {
-
-  import cromwell.backend.impl.htcondor.caching.provider.mongodb.model.MongoCachedExecutionResultProtocol._
-  import spray.json._
-
-  val ErrMsg = "Got an exception when storing execution result for hash {}."
-  val HashIdentifier = "hash"
-  override val tag = s"[MongoCacheActor]"
-
-  override def readExecutionResult(hash: String): CachedExecutionResult = {
-    val query = MongoDBObject(HashIdentifier -> hash)
-    val result = collection.findOne(query)
-    result match {
-      case Some(mongoDbObject) =>
-        if (forceRewrite) throwCachedResultNotFoundException(hash)
-        else deserializeSucceededResponse(mongoDbObject)
-      case None => throwCachedResultNotFoundException(hash)
-    }
-  }
-
-  override def storeExecutionResult(cachedExecutionResult: CachedExecutionResult): Unit = {
-    try {
-      readExecutionResult(cachedExecutionResult.hash)
-      val warnMsg = s"$tag Execution result hash {${cachedExecutionResult.hash}} is already defined in database."
-      log.warning(warnMsg)
-      if (forceRewrite) {
-        removeExecutionResult(cachedExecutionResult.hash)
-        storeExecutionResultInMongoDb(cachedExecutionResult)
-      }
-      else throw new CachedResultAlreadyExistException(warnMsg)
-    } catch {
-      case e: CachedResultNotFoundException =>
-        storeExecutionResultInMongoDb(cachedExecutionResult)
-      case e: CachedResultAlreadyExistException =>
-        throw e
-      case e: Exception =>
-        log.error(e, "{} Got an unhandled exception when trying to store execution result for hash {}.", tag, cachedExecutionResult.hash)
-        throw e
-    }
-  }
-
-  private def deserializeSucceededResponse(mongoDbObject: TypeImports.DBObject): CachedExecutionResult = {
-    val cachedResult = JsonParser(mongoDbObject.toString).convertTo[MongoCachedExecutionResult]
-    val succeededResponse = deserialize(cachedResult.succeededResponse.byteArray, classOf[JobSucceededResponse])
-    CachedExecutionResult(cachedResult.hash, succeededResponse)
-  }
-
-  private def removeExecutionResult(hash: String): Unit = {
-    val query = MongoDBObject(HashIdentifier -> hash)
-    val result = collection.remove(query)
-    if (result.getN == 0) throwCachedResultNotFoundException(hash)
-    log.info("{} Removed execution result for hash: {}.", tag, hash)
-  }
-
-  private def storeExecutionResultInMongoDb(cachedExecutionResult: CachedExecutionResult): Unit = {
-    val cachedResult = MongoCachedExecutionResult(cachedExecutionResult.hash, KryoSerializedObject(serialize(cachedExecutionResult.succeededResponse)))
-    val result = collection.insert(constructDbObjectForCachedExecutionResult(cachedResult))
-    if (!result.wasAcknowledged()) {
-      log.error(ErrMsg, cachedExecutionResult)
-      throw new IllegalStateException(ErrMsg)
-    }
-  }
-
-  private def constructDbObjectForCachedExecutionResult(cachedExecutionResult: MongoCachedExecutionResult): DBObject = {
-    val resultAsJsonString = cachedExecutionResult.toJson.toString()
-    JSON.parse(resultAsJsonString).asInstanceOf[DBObject]
-  }
-
-  private def throwCachedResultNotFoundException(hash: String): Nothing = {
-    val warnMsg = s"$tag Execution result hash {$hash} does not exist in database."
-    log.warning(warnMsg)
-    throw new CachedResultNotFoundException(warnMsg)
-  }
-}
diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/MongoCacheActorFactory.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/MongoCacheActorFactory.scala
deleted file mode 100644
index bdb1354f9..000000000
--- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/MongoCacheActorFactory.scala
+++ /dev/null
@@ -1,21 +0,0 @@
-package cromwell.backend.impl.htcondor.caching.provider.mongodb
-
-import akka.actor.Props
-import com.mongodb.casbah.{MongoClient, MongoCollection}
-import com.typesafe.config.Config
-import cromwell.backend.impl.htcondor.caching.CacheActorFactory
-
-class MongoCacheActorFactory(config: Config) extends CacheActorFactory {
-  val dbHost = config.getString("cache.db.host")
-  val dbPort = config.getInt("cache.db.port")
-  val dbName = config.getString("cache.db.name")
-  val dbCollectionName = config.getString("cache.db.collection")
-  val dbInstance: MongoClient = MongoClient(dbHost, dbPort)
-  val db = dbInstance(dbName)
-  val collection: MongoCollection = db(dbCollectionName)
-
-  override def getCacheActorProps(forceRewrite: Boolean): Props = {
-    Props(new MongoCacheActor(collection, forceRewrite))
-  }
-
-}
diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/model/MongoCachedExecutionResult.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/model/MongoCachedExecutionResult.scala
deleted file mode 100644
index f13326257..000000000
--- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/model/MongoCachedExecutionResult.scala
+++ /dev/null
@@ -1,24 +0,0 @@
-package cromwell.backend.impl.htcondor.caching.provider.mongodb.model
-
-import spray.json
-
-/**
-  * Wrapper over the byte array that is stored in db
-  *
-  * @param byteArray Serialized data
-  */
-case class KryoSerializedObject(byteArray: Array[Byte])
-
-/**
-  * SucceededResponse to be stored in MongoDB.
-  *
-  * @param hash Calculated hash for the Job.
-  * @param succeededResponse Serialized succeeded response.
-  */
-case class MongoCachedExecutionResult(hash: String, succeededResponse: KryoSerializedObject)
-
-object MongoCachedExecutionResultProtocol extends json.DefaultJsonProtocol {
-  implicit val kryoSerializedObject = jsonFormat1(KryoSerializedObject)
-  implicit val cachedExecutionResultProtocol = jsonFormat2(MongoCachedExecutionResult)
-}
-
diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/serialization/KryoSerDe.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/serialization/KryoSerDe.scala
deleted file mode 100644
index d3f564c61..000000000
--- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/serialization/KryoSerDe.scala
+++ /dev/null
@@ -1,55 +0,0 @@
-package cromwell.backend.impl.htcondor.caching.provider.mongodb.serialization
-
-import java.io.{ByteArrayInputStream, ByteArrayOutputStream}
-
-import com.esotericsoftware.kryo.io.{Input, Output}
-import com.twitter.chill.ScalaKryoInstantiator
-
-/**
-  * This mixin provides access to `serialize` and `deserialize` methods that use Kryo underneath to
-  * perform the actual conversion to / from bytes
-  */
-trait KryoSerDe extends SerDe {
-
-  /**
-    *
-    * @param data Any Scala / Java object that needs to be serialized
-    * @return A serialized byte array that can be transported across Network moved around with
-    */
-  override def serialize[A <: AnyRef](data: A): Array[Byte] = {
-    try {
-      val instantiator = new ScalaKryoInstantiator
-      instantiator.setRegistrationRequired(false) // This makes it unnecessary to register all classes
-      val kryo = instantiator.newKryo()
-      val buffer = new ByteArrayOutputStream()
-      val output = new Output(buffer)
-      kryo.writeObject(output, data)
-      output.close()
-      buffer.toByteArray
-    } catch {
-      case exception: Exception => throw SerDeException("Failed to serialize data.", exception)
-    }
-  }
-
-  /**
-    *
-    * @param byteArray Kryo serialized data. Expects only results of `writeObject`
-    * @param toClass Class to which the `byteArray` will be deserialized
-    * @return The deserialized object as an instance of A
-    */
-  override def deserialize[A <: AnyRef](byteArray: Array[Byte], toClass: Class[A]): A = {
-    try {
-      val instantiator = new ScalaKryoInstantiator
-      instantiator.setRegistrationRequired(false)
-      val kryo = instantiator.newKryo()
-      val buffer = new ByteArrayInputStream(byteArray)
-      val input = new Input(buffer)
-      val result = kryo.readObject(input, toClass)
-      input.close()
-      result
-    } catch {
-      case exception: Exception => throw SerDeException("Failed to deserialize data.", exception)
-    }
-  }
-
-}
diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/serialization/SerDe.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/serialization/SerDe.scala
deleted file mode 100644
index 7ab8883d0..000000000
--- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/serialization/SerDe.scala
+++ /dev/null
@@ -1,25 +0,0 @@
-package cromwell.backend.impl.htcondor.caching.provider.mongodb.serialization
-
-/**
-  * A vanilla Serialization / Deserialization interface
-  */
-trait SerDe {
-
-  case class SerDeException(message: String, error: Throwable) extends IllegalStateException(message, error)
-
-  /**
-    * Serialize the given data
-    * @param data Any Scala / Java object that needs to be serialized
-    * @return A serialized byte array that can be transported across Network moved around with
-    */
-  def serialize[A <: AnyRef](data: A): Array[Byte]
-
-  /**
-    * Deserializes an array of Bytes to the given class
-    * @param byteArray Kryo serialized data. Expects only results of `writeObject`
-    * @param toClass Class to which the `byteArray` will be deserialized
-    * @return The deserialized object as an instance of A
-    */
-  def deserialize[A <: AnyRef](byteArray: Array[Byte], toClass: Class[A]): A
-
-}
diff --git a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorCommandSpec.scala b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorCommandSpec.scala
deleted file mode 100644
index d6778fadd..000000000
--- a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorCommandSpec.scala
+++ /dev/null
@@ -1,28 +0,0 @@
-package cromwell.backend.impl.htcondor
-
-import cromwell.core.path.Obsolete._
-import org.scalatest.{Matchers, WordSpecLike}
-
-class HtCondorCommandSpec extends WordSpecLike with Matchers {
-  private val attributes = Map("executable" -> "test.sh", "input" -> "/temp/test", "error"->"stderr")
-  private val resultAttributes = List("executable=test.sh","input=/temp/test","error=stderr", "spec1", "spec2", "queue")
-  private val htCondorCommands = new HtCondorCommands
-  private val nativeSpecs = Option(Array("spec1", "spec2"))
-
-  "submitCommand method" should {
-    "return submit file with content passed to it" in {
-      val file = File.newTemporaryFile()
-      val command = htCondorCommands.generateSubmitFile(file.path, attributes, nativeSpecs)
-      resultAttributes shouldEqual file.lines.toList
-      file.delete()
-      command shouldEqual s"condor_submit ${file.path}"
-    }
-  }
-
-  "statusCommand method" should {
-    "return status command" in {
-      val command = HtCondorCommands.generateJobStatusCommand("96.0")
-      command shouldEqual s"condor_q 96.0 -autoformat JobStatus"
-    }
-  }
-}
\ No newline at end of file
diff --git a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorInitializationActorSpec.scala b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorInitializationActorSpec.scala
deleted file mode 100644
index 29d19d117..000000000
--- a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorInitializationActorSpec.scala
+++ /dev/null
@@ -1,53 +0,0 @@
-package cromwell.backend.impl.htcondor
-
-import akka.testkit.{EventFilter, ImplicitSender, TestDuration}
-import cromwell.backend.BackendWorkflowInitializationActor.Initialize
-import cromwell.backend.{BackendConfigurationDescriptor, BackendSpec, BackendWorkflowDescriptor}
-import cromwell.core.TestKitSuite
-import org.scalatest.{Matchers, WordSpecLike}
-import wdl4s.TaskCall
-
-import scala.concurrent.duration._
-
-class HtCondorInitializationActorSpec extends TestKitSuite("HtCondorInitializationActorSpec") with WordSpecLike
-  with Matchers with ImplicitSender {
-  val Timeout = 5.second.dilated
-
-  import BackendSpec._
-
-  val HelloWorld =
-    s"""
-      |task hello {
-      |  String addressee = "you"
-      |  command {
-      |    echo "Hello $${addressee}!"
-      |  }
-      |  output {
-      |    String salutation = read_string(stdout())
-      |  }
-      |
-      |  RUNTIME
-      |}
-      |
-      |workflow wf_hello {
-      |  call hello
-      |}
-    """.stripMargin
-
-  private def getHtCondorBackend(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[TaskCall], conf: BackendConfigurationDescriptor) = {
-    system.actorOf(HtCondorInitializationActor.props(workflowDescriptor, calls, conf, emptyActor))
-  }
-
-  "HtCondorInitializationActor" should {
-    "log a warning message when there are unsupported runtime attributes" in {
-      within(Timeout) {
-        EventFilter.warning(message = s"Key/s [proc] is/are not supported by HtCondorBackend. Unsupported attributes will not be part of jobs executions.", occurrences = 1) intercept {
-          val workflowDescriptor = buildWorkflowDescriptor(HelloWorld, runtime = """runtime { proc: 1 }""")
-          val backend = getHtCondorBackend(workflowDescriptor, workflowDescriptor.workflow.taskCalls,
-            emptyBackendConfig)
-          backend ! Initialize
-        }
-      }
-    }
-  }
-}
diff --git a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActorSpec.scala b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActorSpec.scala
deleted file mode 100644
index 3081feba7..000000000
--- a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActorSpec.scala
+++ /dev/null
@@ -1,472 +0,0 @@
-package cromwell.backend.impl.htcondor
-
-import java.io.Writer
-
-import akka.actor.{Actor, Props}
-import akka.testkit.{ImplicitSender, TestActorRef}
-import com.typesafe.config.ConfigFactory
-import cromwell.backend.BackendJobExecutionActor.{JobFailedNonRetryableResponse, JobSucceededResponse}
-import cromwell.backend.impl.htcondor.caching.CacheActor
-import cromwell.backend.impl.htcondor.caching.exception.CachedResultNotFoundException
-import cromwell.backend.impl.htcondor.caching.model.CachedExecutionResult
-import cromwell.backend.io.JobPathsWithDocker
-import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptor, BackendSpec}
-import cromwell.core._
-import cromwell.core.path.Obsolete._
-import cromwell.core.path.{Path, PathWriter, TailedWriter, UntailedWriter}
-import cromwell.services.keyvalue.KeyValueServiceActor.{KvGet, KvPair, KvPut}
-import org.mockito.Mockito
-import org.mockito.Mockito._
-import org.scalatest.concurrent.PatienceConfiguration.Timeout
-import org.scalatest.mockito.MockitoSugar
-import org.scalatest.{BeforeAndAfter, Matchers, WordSpecLike}
-import wdl4s.types.{WdlArrayType, WdlFileType}
-import wdl4s.values.{WdlArray, WdlFile, WdlValue}
-
-import scala.concurrent.duration._
-import scala.io.Source
-import scala.sys.process.{Process, ProcessLogger}
-
-class HtCondorJobExecutionActorSpec extends TestKitSuite("HtCondorJobExecutionActorSpec")
-  with WordSpecLike
-  with Matchers
-  with MockitoSugar
-  with BeforeAndAfter
-  with ImplicitSender {
-
-  import BackendSpec._
-
-  private val htCondorCommands: HtCondorCommands = new HtCondorCommands
-  private val htCondorProcess: HtCondorProcess = mock[HtCondorProcess]
-  private val cacheActorMockProps = Props(new CacheActorMock())
-
-  private val helloWorldWdl =
-    """
-      |task hello {
-      |
-      |  command {
-      |    echo "Hello World!"
-      |  }
-      |  output {
-      |    String salutation = read_string(stdout())
-      |  }
-      |  RUNTIME
-      |}
-      |
-      |workflow wf_hello {
-      |  call hello
-      |}
-    """.stripMargin
-
-  private val helloWorldWdlWithFileInput =
-    s"""
-      |task hello {
-      |  File inputFile
-      |
-      |  command {
-      |    echo $${inputFile}
-      |  }
-      |  output {
-      |    String salutation = read_string(stdout())
-      |  }
-      |  RUNTIME
-      |}
-      |
-      |workflow wf_hello {
-      |  call hello
-      |}
-    """.stripMargin
-
-  private val helloWorldWdlWithFileArrayInput =
-    s"""
-      |task hello {
-      |  Array[File] inputFiles
-      |
-      |  command {
-      |    echo $${sep=' ' inputFiles}
-      |  }
-      |  output {
-      |    String salutation = read_string(stdout())
-      |  }
-      |  RUNTIME
-      |}
-      |
-      |workflow wf_hello {
-      |  call hello
-      |}
-    """.stripMargin
-
-  private val backendConfig = ConfigFactory.parseString(
-    s"""{
-        |  root = "local-cromwell-executions"
-        |
-        |  docker {
-        |    cmd = "docker run -w %s %s %s %s --rm %s /bin/bash -c \\"%s\\""
-        |    defaultWorkingDir = "/workingDir/"
-        |    defaultOutputDir = "/output/"
-        |  }
-        |
-        |  filesystems {
-        |    local {
-        |      localization = [
-        |        "hard-link", "soft-link", "copy"
-        |      ]
-        |    }
-        |  }
-        |  poll-interval = 3
-        |}
-        """.stripMargin)
-
-  private val timeout = Timeout(1.seconds)
-
-  after {
-    Mockito.reset(htCondorProcess)
-  }
-
-  "executeTask method" should {
-    "return succeeded task status with stdout" in {
-      val jobDescriptor = prepareJob()
-      val (job, jobPaths, backendConfigDesc) = (jobDescriptor.jobDescriptor, jobDescriptor.jobPaths, jobDescriptor.backendConfigurationDescriptor)
-      val stubProcess = mock[Process]
-      val stubUntailed = new UntailedWriter(jobPaths.stdout) with MockPathWriter
-      val stubTailed = new TailedWriter(jobPaths.stderr, 100) with MockPathWriter
-      val stderrResult = ""
-
-      when(htCondorProcess.commandList(any[String])).thenReturn(Seq.empty[String])
-      when(htCondorProcess.externalProcess(any[Seq[String]], any[ProcessLogger])).thenReturn(stubProcess)
-      when(stubProcess.exitValue()).thenReturn(0)
-      when(htCondorProcess.tailedWriter(any[Int], any[Path])).thenReturn(stubTailed)
-      when(htCondorProcess.untailedWriter(any[Path])).thenReturn(stubUntailed)
-      when(htCondorProcess.processStderr).thenReturn(stderrResult)
-      when(htCondorProcess.jobReturnCode(any[String], any[Path])).thenReturn(Option(0))
-
-      val backend = TestActorRef(new HtCondorJobExecutionActor(job, backendConfigDesc, system.deadLetters, None) {
-        override lazy val cmds = htCondorCommands
-        override lazy val extProcess = htCondorProcess
-      }).underlyingActor
-
-      whenReady(backend.execute, timeout) { response =>
-        response shouldBe a[JobSucceededResponse]
-        verify(htCondorProcess, times(1)).externalProcess(any[Seq[String]], any[ProcessLogger])
-        verify(htCondorProcess, times(1)).tailedWriter(any[Int], any[Path])
-        verify(htCondorProcess, times(1)).untailedWriter(any[Path])
-      }
-
-      cleanUpJob(jobPaths)
-    }
-
-    "return succeeded task status when it recovers from a shutdown" in {
-      val jobDescriptor = prepareJob()
-      val (job, jobPaths, backendConfigDesc) = (jobDescriptor.jobDescriptor, jobDescriptor.jobPaths, jobDescriptor.backendConfigurationDescriptor)
-      val stubProcess = mock[Process]
-      val stubUntailed = new UntailedWriter(jobPaths.stdout) with MockPathWriter
-      val stubTailed = new TailedWriter(jobPaths.stderr, 100) with MockPathWriter
-      val stderrResult = ""
-      val kVServiceActor = system.actorOf(Props(new KVServiceActor()))
-
-      when(htCondorProcess.commandList(any[String])).thenReturn(Seq.empty[String])
-      when(htCondorProcess.externalProcess(any[Seq[String]], any[ProcessLogger])).thenReturn(stubProcess)
-      when(stubProcess.exitValue()).thenReturn(0)
-      when(htCondorProcess.tailedWriter(any[Int], any[Path])).thenReturn(stubTailed)
-      when(htCondorProcess.untailedWriter(any[Path])).thenReturn(stubUntailed)
-      when(htCondorProcess.processStderr).thenReturn(stderrResult)
-      when(htCondorProcess.jobReturnCode(any[String], any[Path])).thenReturn(Option(0))
-
-      val backend = TestActorRef(new HtCondorJobExecutionActor(job, backendConfigDesc, kVServiceActor, None) {
-        override lazy val cmds = htCondorCommands
-        override lazy val extProcess = htCondorProcess
-      }).underlyingActor
-
-      whenReady(backend.recover, timeout) { response =>
-        response shouldBe a[JobSucceededResponse]
-      }
-
-      cleanUpJob(jobPaths)
-    }
-
-    "return succeeded task status with stdout when cache is enabled" in {
-      val jobDescriptor = prepareJob()
-      val (job, jobPaths, backendConfigDesc) = (jobDescriptor.jobDescriptor, jobDescriptor.jobPaths, jobDescriptor.backendConfigurationDescriptor)
-      val stubProcess = mock[Process]
-      val stubUntailed = new UntailedWriter(jobPaths.stdout) with MockPathWriter
-      val stubTailed = new TailedWriter(jobPaths.stderr, 100) with MockPathWriter
-      val stderrResult = ""
-
-      when(htCondorProcess.commandList(any[String])).thenReturn(Seq.empty[String])
-      when(htCondorProcess.externalProcess(any[Seq[String]], any[ProcessLogger])).thenReturn(stubProcess)
-      when(stubProcess.exitValue()).thenReturn(0)
-      when(htCondorProcess.tailedWriter(any[Int], any[Path])).thenReturn(stubTailed)
-      when(htCondorProcess.untailedWriter(any[Path])).thenReturn(stubUntailed)
-      when(htCondorProcess.processStderr).thenReturn(stderrResult)
-      when(htCondorProcess.jobReturnCode(any[String], any[Path])).thenReturn(Option(0))
-
-      val backend = TestActorRef(new HtCondorJobExecutionActor(job, backendConfigDesc, system.deadLetters, Some(cacheActorMockProps)) {
-        override lazy val cmds = htCondorCommands
-        override lazy val extProcess = htCondorProcess
-      }).underlyingActor
-
-      whenReady(backend.execute, timeout) { response =>
-        response shouldBe a[JobSucceededResponse]
-        verify(htCondorProcess, times(1)).externalProcess(any[Seq[String]], any[ProcessLogger])
-        verify(htCondorProcess, times(1)).tailedWriter(any[Int], any[Path])
-        verify(htCondorProcess, times(1)).untailedWriter(any[Path])
-      }
-
-      cleanUpJob(jobPaths)
-    }
-
-    "return failed task status with stderr on non-zero process exit" in {
-      val jobDescriptor = prepareJob()
-      val (job, jobPaths, backendConfigDesc) = (jobDescriptor.jobDescriptor, jobDescriptor.jobPaths, jobDescriptor.backendConfigurationDescriptor)
-
-      val backend = TestActorRef(new HtCondorJobExecutionActor(job, backendConfigDesc, system.deadLetters, Some(cacheActorMockProps)) {
-        override lazy val cmds = htCondorCommands
-        override lazy val extProcess = htCondorProcess
-      }).underlyingActor
-      val stubProcess = mock[Process]
-      val stubUntailed = new UntailedWriter(jobPaths.stdout) with MockPathWriter
-      val stubTailed = new TailedWriter(jobPaths.stderr, 100) with MockPathWriter
-      val stderrResult = ""
-
-      when(htCondorProcess.externalProcess(any[Seq[String]], any[ProcessLogger])).thenReturn(stubProcess)
-      when(stubProcess.exitValue()).thenReturn(0)
-      when(htCondorProcess.tailedWriter(any[Int], any[Path])).thenReturn(stubTailed)
-      when(htCondorProcess.untailedWriter(any[Path])).thenReturn(stubUntailed)
-      when(htCondorProcess.processStderr).thenReturn(stderrResult)
-      when(htCondorProcess.jobReturnCode(any[String], any[Path])).thenReturn(Option(-1))
-
-      whenReady(backend.execute, timeout) { response =>
-        response shouldBe a[JobFailedNonRetryableResponse]
-        assert(response.asInstanceOf[JobFailedNonRetryableResponse].throwable.getMessage.contains("Job exited with invalid return code"))
-      }
-
-      cleanUpJob(jobPaths)
-    }
-
-    "return a successful task status even with a non-zero process exit" in {
-      val runtime =
-        """
-          |runtime {
-          | continueOnReturnCode: [911]
-          |}
-        """.stripMargin
-      val jobDescriptor = prepareJob(runtimeString = runtime)
-      val (job, jobPaths, backendConfigDesc) = (jobDescriptor.jobDescriptor, jobDescriptor.jobPaths, jobDescriptor.backendConfigurationDescriptor)
-
-      val backend = TestActorRef(new HtCondorJobExecutionActor(job, backendConfigDesc, system.deadLetters, Some(cacheActorMockProps)) {
-        override lazy val cmds = htCondorCommands
-        override lazy val extProcess = htCondorProcess
-      }).underlyingActor
-      val stubProcess = mock[Process]
-      val stubUntailed = new UntailedWriter(jobPaths.stdout) with MockPathWriter
-      val stubTailed = new TailedWriter(jobPaths.stderr, 100) with MockPathWriter
-      val stderrResult = ""
-
-      when(htCondorProcess.externalProcess(any[Seq[String]], any[ProcessLogger])).thenReturn(stubProcess)
-      when(stubProcess.exitValue()).thenReturn(0)
-      when(htCondorProcess.tailedWriter(any[Int], any[Path])).thenReturn(stubTailed)
-      when(htCondorProcess.untailedWriter(any[Path])).thenReturn(stubUntailed)
-      when(htCondorProcess.processStderr).thenReturn(stderrResult)
-      when(htCondorProcess.jobReturnCode(any[String], any[Path])).thenReturn(Option(911))
-
-      whenReady(backend.execute, timeout) { response =>
-        response shouldBe a[JobSucceededResponse]
-      }
-
-      cleanUpJob(jobPaths)
-    }
-
-    "return a successful task status when it runs a docker command with working and output directory" in {
-      val runtime =
-        """
-          |runtime {
-          | docker: "ubuntu/latest"
-          | dockerWorkingDir: "/workingDir/"
-          | dockerOutputDir: "/outputDir/"
-          |}
-        """.stripMargin
-      val jsonInputFile = createCannedFile("testFile", "some content").pathAsString
-      val inputs = Map(
-        "wf_hello.hello.inputFile" -> WdlFile(jsonInputFile)
-      )
-      val jobDescriptor = prepareJob(helloWorldWdlWithFileInput, runtime, Option(inputs))
-      val (job, jobPaths, backendConfigDesc) = (jobDescriptor.jobDescriptor, jobDescriptor.jobPaths, jobDescriptor.backendConfigurationDescriptor)
-
-      val backend = TestActorRef(new HtCondorJobExecutionActor(job, backendConfigDesc, system.deadLetters, Some(cacheActorMockProps)) {
-        override lazy val cmds = htCondorCommands
-        override lazy val extProcess = htCondorProcess
-      }).underlyingActor
-      val stubProcess = mock[Process]
-      val stubUntailed = new UntailedWriter(jobPaths.stdout) with MockPathWriter
-      val stubTailed = new TailedWriter(jobPaths.stderr, 100) with MockPathWriter
-      val stderrResult = ""
-
-      when(htCondorProcess.externalProcess(any[Seq[String]], any[ProcessLogger])).thenReturn(stubProcess)
-      when(stubProcess.exitValue()).thenReturn(0)
-      when(htCondorProcess.tailedWriter(any[Int], any[Path])).thenReturn(stubTailed)
-      when(htCondorProcess.untailedWriter(any[Path])).thenReturn(stubUntailed)
-      when(htCondorProcess.processStderr).thenReturn(stderrResult)
-      when(htCondorProcess.jobReturnCode(any[String], any[Path])).thenReturn(Option(0))
-
-      whenReady(backend.execute) { response =>
-        response shouldBe a[JobSucceededResponse]
-      }
-
-      val bashScript = Source.fromFile(jobPaths.script.toFile).getLines.mkString
-
-      assert(bashScript.contains("docker run -w /workingDir/ -v"))
-      assert(bashScript.contains(":/workingDir/"))
-      assert(bashScript.contains(":ro"))
-      assert(bashScript.contains("/call-hello/execution:/outputDir/ --rm ubuntu/latest /bin/bash -c \"echo"))
-
-      cleanUpJob(jobPaths)
-    }
-
-    "return failed when cmds fails to write script" in {
-      val htCondorCommandsMock: HtCondorCommands = mock[HtCondorCommands]
-      val jobDescriptor = prepareJob()
-      val (job, jobPaths, backendConfigDesc) = (jobDescriptor.jobDescriptor, jobDescriptor.jobPaths, jobDescriptor.backendConfigurationDescriptor)
-
-      val backend = TestActorRef(new HtCondorJobExecutionActor(job, backendConfigDesc, system.deadLetters, Some(cacheActorMockProps)) {
-        override lazy val cmds = htCondorCommandsMock
-        override lazy val extProcess = htCondorProcess
-      }).underlyingActor
-      val stubProcess = mock[Process]
-      val stubUntailed = new UntailedWriter(jobPaths.stdout) with MockPathWriter
-      val stubTailed = new TailedWriter(jobPaths.stderr, 100) with MockPathWriter
-      val stderrResult = ""
-
-      when(htCondorCommandsMock.writeScript(any[String], any[Path], any[Path])).thenThrow(new IllegalStateException("Could not write the file."))
-      when(htCondorProcess.externalProcess(any[Seq[String]], any[ProcessLogger])).thenReturn(stubProcess)
-      when(stubProcess.exitValue()).thenReturn(0)
-      when(htCondorProcess.tailedWriter(any[Int], any[Path])).thenReturn(stubTailed)
-      when(htCondorProcess.untailedWriter(any[Path])).thenReturn(stubUntailed)
-      when(htCondorProcess.processStderr).thenReturn(stderrResult)
-      when(htCondorProcess.jobReturnCode(any[String], any[Path])).thenReturn(Option(-1))
-
-      whenReady(backend.execute, timeout) { response =>
-        response shouldBe a[JobFailedNonRetryableResponse]
-        assert(response.asInstanceOf[JobFailedNonRetryableResponse].throwable.getMessage.contains("Could not write the file."))
-      }
-
-      cleanUpJob(jobPaths)
-    }
-  }
-
-  "return a successful task status when it tries to run a docker command containing file data from a WDL file array" in {
-    val runtime =
-      """
-        |runtime {
-        | docker: "ubuntu/latest"
-        | dockerWorkingDir: "/workingDir/"
-        | dockerOutputDir: "/outputDir/"
-        |}
-      """.stripMargin
-
-    val tempDir1 = Files.createTempDirectory("dir1")
-    val tempDir2 = Files.createTempDirectory("dir2")
-    val jsonInputFile =
-      createCannedFile(prefix = "testFile", contents = "some content", dir = Some(tempDir1)).pathAsString
-    val jsonInputFile2 =
-      createCannedFile(prefix = "testFile2", contents = "some other content", dir = Some(tempDir2)).pathAsString
-
-    val inputs = Map(
-      "wf_hello.hello.inputFiles" -> WdlArray(WdlArrayType(WdlFileType), Seq(WdlFile(jsonInputFile), WdlFile(jsonInputFile2)))
-    )
-    val jobDescriptor = prepareJob(helloWorldWdlWithFileArrayInput, runtime, Option(inputs))
-    val (job, jobPaths, backendConfigDesc) = (jobDescriptor.jobDescriptor, jobDescriptor.jobPaths, jobDescriptor.backendConfigurationDescriptor)
-
-    val backend = TestActorRef(new HtCondorJobExecutionActor(job, backendConfigDesc, system.deadLetters, Some(cacheActorMockProps)) {
-      override lazy val cmds = htCondorCommands
-      override lazy val extProcess = htCondorProcess
-    }).underlyingActor
-    val stubProcess = mock[Process]
-    val stubUntailed = new UntailedWriter(jobPaths.stdout) with MockPathWriter
-    val stubTailed = new TailedWriter(jobPaths.stderr, 100) with MockPathWriter
-    val stderrResult = ""
-
-    when(htCondorProcess.externalProcess(any[Seq[String]], any[ProcessLogger])).thenReturn(stubProcess)
-    when(stubProcess.exitValue()).thenReturn(0)
-    when(htCondorProcess.tailedWriter(any[Int], any[Path])).thenReturn(stubTailed)
-    when(htCondorProcess.untailedWriter(any[Path])).thenReturn(stubUntailed)
-    when(htCondorProcess.processStderr).thenReturn(stderrResult)
-    when(htCondorProcess.jobReturnCode(any[String], any[Path])).thenReturn(Option(0))
-
-    whenReady(backend.execute) { response =>
-      response shouldBe a[JobSucceededResponse]
-    }
-
-    val bashScript = Source.fromFile(jobPaths.script.toFile).getLines.mkString
-
-    assert(bashScript.contains("docker run -w /workingDir/ -v"))
-    assert(bashScript.contains(":/workingDir/"))
-    assert(bashScript.contains(tempDir1.toAbsolutePath.toString))
-    assert(bashScript.contains(tempDir2.toAbsolutePath.toString))
-    assert(bashScript.contains("/call-hello/execution:/outputDir/ --rm ubuntu/latest /bin/bash -c \"echo"))
-
-    cleanUpJob(jobPaths)
-  }
-
-  private def cleanUpJob(jobPaths: JobPathsWithDocker): Unit = {
-    File(jobPaths.workflowRoot).delete(true)
-    ()
-  }
-
-  private def createCannedFile(prefix: String, contents: String, dir: Option[Path] = None): File = {
-    val suffix = ".out"
-    val file = File.newTemporaryFile(prefix, suffix, dir.map(File.apply))
-    file.write(contents)
-  }
-
-  val emptyWorkflowOptions = WorkflowOptions.fromMap(Map.empty).get
-
-  private def prepareJob(source: String = helloWorldWdl, runtimeString: String = "", inputFiles: Option[Map[String, WdlValue]] = None): TestJobDescriptor = {
-    val backendWorkflowDescriptor = buildWorkflowDescriptor(wdl = source, inputs = inputFiles.getOrElse(Map.empty), runtime = runtimeString)
-    val backendConfigurationDescriptor = BackendConfigurationDescriptor(backendConfig, ConfigFactory.load)
-    val jobDesc = jobDescriptorFromSingleCallWorkflow(backendWorkflowDescriptor, inputFiles.getOrElse(Map.empty), emptyWorkflowOptions, Set.empty)
-    val jobPaths = new JobPathsWithDocker(jobDesc.key, backendWorkflowDescriptor, backendConfig)
-    val executionDir = File(jobPaths.callExecutionRoot)
-    val stdout = File(executionDir.pathAsString, "stdout")
-    stdout.createIfNotExists(asDirectory = false, createParents = true)
-    val submitFileStderr = executionDir./("submitfile.stderr")
-    val submitFileStdout = executionDir./("submitfile.stdout")
-    submitFileStdout.createIfNotExists(asDirectory = false, createParents = true)
-    submitFileStdout <<
-      """Submitting job(s)..
-        |1 job(s) submitted to cluster 88.
-      """.stripMargin.trim
-    submitFileStderr.createIfNotExists(asDirectory = false, createParents = true)
-    TestJobDescriptor(jobDesc, jobPaths, backendConfigurationDescriptor)
-  }
-
-  private case class TestJobDescriptor(jobDescriptor: BackendJobDescriptor, jobPaths: JobPathsWithDocker, backendConfigurationDescriptor: BackendConfigurationDescriptor)
-
-  trait MockWriter extends Writer {
-    var closed = false
-
-    override def close() = closed = true
-
-    override def flush() = {}
-
-    override def write(a: Array[Char], b: Int, c: Int) = {}
-  }
-
-  trait MockPathWriter extends PathWriter {
-    override lazy val writer: Writer = new MockWriter {}
-    override val path: Path = mock[Path]
-  }
-
-  class CacheActorMock extends CacheActor {
-    override def readExecutionResult(hash: String): CachedExecutionResult = throw new CachedResultNotFoundException("Entry not found.")
-
-    override def storeExecutionResult(cachedExecutionResult: CachedExecutionResult): Unit = ()
-  }
-
-  class KVServiceActor extends Actor {
-    override def receive: Receive = {
-      case KvPut => // Do nothing
-      case KvGet(kvKey) => sender ! KvPair(kvKey, Option("123"))
-    }
-  }
-
-}
diff --git a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributesSpec.scala b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributesSpec.scala
deleted file mode 100644
index 9812c44e9..000000000
--- a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributesSpec.scala
+++ /dev/null
@@ -1,304 +0,0 @@
-package cromwell.backend.impl.htcondor
-
-import cromwell.backend.{BackendSpec, MemorySize}
-import cromwell.backend.validation.ContinueOnReturnCodeSet
-import cromwell.backend.validation.RuntimeAttributesKeys._
-import cromwell.core.WorkflowOptions
-import lenthall.util.TryUtil
-import org.scalatest.{Matchers, WordSpecLike}
-import spray.json._
-import wdl4s.WdlExpression._
-import wdl4s._
-import wdl4s.expression.NoFunctions
-import wdl4s.values.WdlValue
-
-class HtCondorRuntimeAttributesSpec extends WordSpecLike with Matchers {
-
-  import BackendSpec._
-
-  val HelloWorld =
-    s"""
-      |task hello {
-      |  String addressee = "you"
-      |  command {
-      |    echo "Hello $${addressee}!"
-      |  }
-      |  output {
-      |    String salutation = read_string(stdout())
-      |  }
-      |
-      |  RUNTIME
-      |}
-      |
-      |workflow wf_hello {
-      |  call hello
-      |}
-    """.stripMargin
-
-  val emptyWorkflowOptions = WorkflowOptions(JsObject(Map.empty[String, JsValue]))
-
-  val memorySize = MemorySize.parse("0.512 GB").get
-  val diskSize = MemorySize.parse("1.024 GB").get
-  val staticDefaults = new HtCondorRuntimeAttributes(ContinueOnReturnCodeSet(Set(0)), None, None, None, false, 1,
-    memorySize, diskSize, None)
-
-  def workflowOptionsWithDefaultRA(defaults: Map[String, JsValue]) = {
-    WorkflowOptions(JsObject(Map(
-      "default_runtime_attributes" -> JsObject(defaults)
-    )))
-  }
-
-  "HtCondorRuntimeAttributes" should {
-    "return an instance of itself when there are no runtime attributes defined." in {
-      val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { }""").head
-      assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, emptyWorkflowOptions, staticDefaults)
-    }
-
-    "return an instance of itself when tries to validate a valid Docker entry" in {
-      val expectedRuntimeAttributes = staticDefaults.copy(dockerImage = Option("ubuntu:latest"))
-      val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { docker: "ubuntu:latest" }""").head
-      assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, emptyWorkflowOptions, expectedRuntimeAttributes)
-    }
-
-    "return an instance of itself when tries to validate a valid Docker entry based on input" in {
-      val expectedRuntimeAttributes = staticDefaults.copy(dockerImage = Option("you"))
-      val runtimeAttributes = createRuntimeAttributes(HelloWorld, s"""runtime { docker: "\\$${addressee}" }""").head
-      assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, emptyWorkflowOptions, expectedRuntimeAttributes)
-    }
-
-    "use workflow options as default if docker key is missing" in {
-      val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { }""").head
-      val workflowOptions = workflowOptionsWithDefaultRA(Map(DockerKey -> JsString("ubuntu:latest")))
-      assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, workflowOptions, staticDefaults.copy(dockerImage = Some("ubuntu:latest")))
-    }
-
-    "throw an exception when tries to validate an invalid Docker entry" in {
-      val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { docker: 1 }""").head
-      assertHtCondorRuntimeAttributesFailedCreation(runtimeAttributes, "Expecting docker runtime attribute to be a String")
-    }
-
-    "return an instance of itself when tries to validate a valid docker working directory entry" in {
-      val expectedRuntimeAttributes = staticDefaults.copy(dockerWorkingDir = Option("/workingDir"))
-      val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { dockerWorkingDir: "/workingDir" }""").head
-      assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, emptyWorkflowOptions, expectedRuntimeAttributes)
-    }
-
-    "return an instance of itself when tries to validate a valid docker working directory entry based on input" in {
-      val expectedRuntimeAttributes = staticDefaults.copy(dockerWorkingDir = Option("you"))
-      val runtimeAttributes = createRuntimeAttributes(HelloWorld, s"""runtime { dockerWorkingDir: "\\$${addressee}" }""").head
-      assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, emptyWorkflowOptions, expectedRuntimeAttributes)
-    }
-
-    "use workflow options as default if docker working directory key is missing" in {
-      val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { }""").head
-      val workflowOptions = workflowOptionsWithDefaultRA(Map("dockerWorkingDir" -> JsString("/workingDir")))
-      assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, workflowOptions, staticDefaults.copy(dockerWorkingDir = Some("/workingDir")))
-    }
-
-    "throw an exception when tries to validate an invalid docker working directory entry" in {
-      val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { dockerWorkingDir: 1 }""").head
-      assertHtCondorRuntimeAttributesFailedCreation(runtimeAttributes, "Expecting dockerWorkingDir runtime attribute to be a String")
-    }
-
-    "return an instance of itself when tries to validate a valid docker output directory entry" in {
-      val expectedRuntimeAttributes = staticDefaults.copy(dockerOutputDir = Option("/outputDir"))
-      val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { dockerOutputDir: "/outputDir" }""").head
-      assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, emptyWorkflowOptions, expectedRuntimeAttributes)
-    }
-
-    "return an instance of itself when tries to validate a valid docker output directory entry based on input" in {
-      val expectedRuntimeAttributes = staticDefaults.copy(dockerOutputDir = Option("you"))
-      val runtimeAttributes = createRuntimeAttributes(HelloWorld, s"""runtime { dockerOutputDir: "\\$${addressee}" }""").head
-      assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, emptyWorkflowOptions, expectedRuntimeAttributes)
-    }
-
-    "use workflow options as default if docker output directory key is missing" in {
-      val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { }""").head
-      val workflowOptions = workflowOptionsWithDefaultRA(Map("dockerOutputDir" -> JsString("/outputDir")))
-      assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, workflowOptions, staticDefaults.copy(dockerOutputDir = Some("/outputDir")))
-    }
-
-    "throw an exception when tries to validate an invalid docker output directory entry" in {
-      val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { dockerOutputDir: 1 }""").head
-      assertHtCondorRuntimeAttributesFailedCreation(runtimeAttributes, "Expecting dockerOutputDir runtime attribute to be a String")
-    }
-
-    "return an instance of itself when tries to validate a valid failOnStderr entry" in {
-      val expectedRuntimeAttributes = staticDefaults.copy(failOnStderr = true)
-      val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { failOnStderr: "true" }""").head
-      val shouldBeIgnored = workflowOptionsWithDefaultRA(Map(FailOnStderrKey -> JsBoolean(false)))
-      assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, shouldBeIgnored, expectedRuntimeAttributes)
-    }
-
-    "throw an exception when tries to validate an invalid failOnStderr entry" in {
-      val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { failOnStderr: "yes" }""").head
-      assertHtCondorRuntimeAttributesFailedCreation(runtimeAttributes, "Expecting failOnStderr runtime attribute to be a Boolean or a String with values of 'true' or 'false'")
-    }
-
-    "use workflow options as default if failOnStdErr key is missing" in {
-      val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { }""").head
-      val workflowOptions = workflowOptionsWithDefaultRA(Map(FailOnStderrKey -> JsBoolean(true)))
-      assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, workflowOptions, staticDefaults.copy(failOnStderr = true))
-    }
-
-    "return an instance of itself when tries to validate a valid continueOnReturnCode entry" in {
-      val expectedRuntimeAttributes = staticDefaults.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1)))
-      val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { continueOnReturnCode: 1 }""").head
-      val shouldBeIgnored = workflowOptionsWithDefaultRA(Map(ContinueOnReturnCodeKey -> JsBoolean(false)))
-      assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, shouldBeIgnored, expectedRuntimeAttributes)
-    }
-
-    "throw an exception when tries to validate an invalid continueOnReturnCode entry" in {
-      val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { continueOnReturnCode: "value" }""").head
-      assertHtCondorRuntimeAttributesFailedCreation(runtimeAttributes, "Expecting continueOnReturnCode runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]")
-    }
-
-    "use workflow options as default if continueOnReturnCode key is missing" in {
-      val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { }""").head
-      val workflowOptions = workflowOptionsWithDefaultRA(Map(ContinueOnReturnCodeKey -> JsArray(Vector(JsNumber(1), JsNumber(2)))))
-      assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, workflowOptions, staticDefaults.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1, 2))))
-    }
-
-    "return an instance of itself when tries to validate a valid cpu entry" in {
-      val expectedRuntimeAttributes = staticDefaults.copy(cpu = 2)
-      val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { cpu: 2 }""").head
-      val shouldBeIgnored = workflowOptionsWithDefaultRA(Map(CpuKey -> JsString("6")))
-      assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, shouldBeIgnored, expectedRuntimeAttributes)
-    }
-
-    "throw an exception when tries to validate an invalid cpu entry" in {
-      val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { cpu: "value" }""").head
-      assertHtCondorRuntimeAttributesFailedCreation(runtimeAttributes, "Expecting cpu runtime attribute to be an Integer")
-    }
-
-    "use workflow options as default if cpu key is missing" in {
-      val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { }""").head
-      val expectedRuntimeAttributes = staticDefaults.copy(cpu = 6)
-      val workflowOptions = workflowOptionsWithDefaultRA(Map(CpuKey -> JsString("6")))
-      val workflowOptions2 = workflowOptionsWithDefaultRA(Map(CpuKey -> JsNumber("6")))
-      assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, workflowOptions, expectedRuntimeAttributes)
-      assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, workflowOptions2, expectedRuntimeAttributes)
-    }
-
-    "use default cpu value when there is no cpu key entry" in {
-      val expectedRuntimeAttributes = staticDefaults.copy(cpu = 1)
-      val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { }""").head
-      val shouldBeIgnored = workflowOptionsWithDefaultRA(Map())
-      assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, shouldBeIgnored, expectedRuntimeAttributes)
-    }
-
-    "return an instance of itself when tries to validate a valid memory entry" in {
-      val expectedRuntimeAttributes = staticDefaults.copy(memory = MemorySize.parse("1 GB").get)
-      val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { memory: "1 GB" }""").head
-      val shouldBeIgnored = workflowOptionsWithDefaultRA(Map(MemoryKey -> JsString("blahaha")))
-      assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, shouldBeIgnored, expectedRuntimeAttributes)
-    }
-
-    "throw an exception when tries to validate an invalid memory entry" in {
-      val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { docker: "ubuntu:latest" memory: "value" }""").head
-      assertHtCondorRuntimeAttributesFailedCreation(runtimeAttributes, "Expecting memory runtime attribute to be an Integer or String with format '8 GB'")
-    }
-
-    "use workflow options as default if memory key is missing" in {
-      val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { }""").head
-      val expectedRuntimeAttributes = staticDefaults.copy(memory = MemorySize.parse("65 GB").get)
-      val workflowOptions = workflowOptionsWithDefaultRA(Map(MemoryKey -> JsString("65 GB")))
-      assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, workflowOptions, expectedRuntimeAttributes)
-    }
-
-    "use default memory value when there is no memory key entry" in {
-      val expectedRuntimeAttributes = staticDefaults.copy(memory = MemorySize.parse("0.512 GB").get)
-      val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { }""").head
-      val shouldBeIgnored = workflowOptionsWithDefaultRA(Map())
-      assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, shouldBeIgnored, expectedRuntimeAttributes)
-    }
-
-    "return an instance of itself when tries to validate a valid disk entry" in {
-      val expectedRuntimeAttributes = staticDefaults.copy(disk = MemorySize.parse("1 GB").get)
-      val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { disk: "1 GB" }""").head
-      val shouldBeIgnored = workflowOptionsWithDefaultRA(Map("disk" -> JsString("blahaha")))
-      assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, shouldBeIgnored, expectedRuntimeAttributes)
-    }
-
-    "throw an exception when tries to validate an invalid String disk entry" in {
-      val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { docker: "ubuntu:latest" disk: "value" }""").head
-      assertHtCondorRuntimeAttributesFailedCreation(runtimeAttributes, "Expecting memory runtime attribute to be an Integer or String with format '8 GB'")
-    }
-
-    "throw an exception when tries to validate an invalid Integer array disk entry" in {
-      val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { docker: "ubuntu:latest" disk: [1] }""").head
-      assertHtCondorRuntimeAttributesFailedCreation(runtimeAttributes, "Expecting disk runtime attribute to be an Integer or String with format '8 GB'")
-    }
-
-    "use workflow options as default if disk key is missing" in {
-      val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { }""").head
-      val expectedRuntimeAttributes = staticDefaults.copy(disk = MemorySize.parse("65 GB").get)
-      val workflowOptions = workflowOptionsWithDefaultRA(Map("disk" -> JsString("65 GB")))
-      assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, workflowOptions, expectedRuntimeAttributes)
-    }
-
-    "use default disk value when there is no disk key entry" in {
-      val expectedRuntimeAttributes = staticDefaults.copy(disk = MemorySize.parse("1.024 GB").get)
-      val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { }""").head
-      val shouldBeIgnored = workflowOptionsWithDefaultRA(Map())
-      assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, shouldBeIgnored, expectedRuntimeAttributes)
-    }
-
-    "return an instance of itself when tries to validate a valid native specs entry" in {
-      val expectedRuntimeAttributes = staticDefaults.copy(nativeSpecs = Option(Array("spec1", "spec2")))
-      val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { nativeSpecs: ["spec1", "spec2"] }""").head
-      assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, emptyWorkflowOptions, expectedRuntimeAttributes)
-    }
-
-    "throw an exception when tries to validate an invalid native specs entry" in {
-      val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { nativeSpecs: [1, 2] }""").head
-      assertHtCondorRuntimeAttributesFailedCreation(runtimeAttributes, "Expecting nativeSpecs runtime attribute to be an Array of Strings.")
-    }
-  }
-
-  private def assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes: Map[String, WdlValue],
-                                                                workflowOptions: WorkflowOptions,
-                                                                expectedRuntimeAttributes: HtCondorRuntimeAttributes) = {
-    try {
-      val actualRuntimeAttr = HtCondorRuntimeAttributes(runtimeAttributes, workflowOptions)
-      assert(actualRuntimeAttr.cpu == expectedRuntimeAttributes.cpu)
-      assert(actualRuntimeAttr.disk == expectedRuntimeAttributes.disk)
-      assert(actualRuntimeAttr.memory == expectedRuntimeAttributes.memory)
-      assert(actualRuntimeAttr.continueOnReturnCode == expectedRuntimeAttributes.continueOnReturnCode)
-      assert(actualRuntimeAttr.failOnStderr == expectedRuntimeAttributes.failOnStderr)
-      assert(actualRuntimeAttr.dockerWorkingDir == expectedRuntimeAttributes.dockerWorkingDir)
-      assert(actualRuntimeAttr.dockerImage == expectedRuntimeAttributes.dockerImage)
-      assert(actualRuntimeAttr.dockerOutputDir == expectedRuntimeAttributes.dockerOutputDir)
-      expectedRuntimeAttributes.nativeSpecs match {
-        case Some(ns) => assert(ns.deep == expectedRuntimeAttributes.nativeSpecs.get.deep)
-        case None => assert(expectedRuntimeAttributes.nativeSpecs.isEmpty)
-      }
-    } catch {
-      case ex: RuntimeException => fail(s"Exception was not expected but received: ${ex.getMessage}")
-    }
-  }
-
-  private def assertHtCondorRuntimeAttributesFailedCreation(runtimeAttributes: Map[String, WdlValue], exMsg: String) = {
-    try {
-      HtCondorRuntimeAttributes(runtimeAttributes, emptyWorkflowOptions)
-      fail("A RuntimeException was expected.")
-    } catch {
-      case ex: RuntimeException => assert(ex.getMessage.contains(exMsg))
-    }
-  }
-
-  private def createRuntimeAttributes(wdlSource: WdlSource, runtimeAttributes: String): Seq[Map[String, WdlValue]] = {
-    val workflowDescriptor = buildWorkflowDescriptor(wdlSource, runtime = runtimeAttributes)
-
-    def createLookup(call: Call): ScopedLookupFunction = {
-      val knownInputs = workflowDescriptor.knownValues
-      call.lookupFunction(knownInputs, NoFunctions)
-    }
-
-    workflowDescriptor.workflow.taskCalls.toSeq map {
-      call =>
-        val ra = call.task.runtimeAttributes.attrs mapValues { _.evaluate(createLookup(call), NoFunctions) }
-        TryUtil.sequenceMap(ra, "Runtime attributes evaluation").get
-    }
-  }
-}
diff --git a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/caching/localization/CachedResultLocalizationSpec.scala b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/caching/localization/CachedResultLocalizationSpec.scala
deleted file mode 100644
index 2e3b66656..000000000
--- a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/caching/localization/CachedResultLocalizationSpec.scala
+++ /dev/null
@@ -1,63 +0,0 @@
-package cromwell.backend.impl.htcondor.caching.localization
-
-import cromwell.core.path.Obsolete._
-import cromwell.core.{CallOutputs, JobOutput}
-import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpecLike}
-import wdl4s.types.{WdlArrayType, WdlFileType}
-import wdl4s.values.{WdlArray, WdlSingleFile, WdlString}
-
-class CachedResultLocalizationSpec extends WordSpecLike with Matchers with BeforeAndAfterAll {
-  private class CachedResultLocalizationMock extends CachedResultLocalization
-  private val defaultTmpDir = Files.createTempDirectory("cachedFiles").toAbsolutePath
-  private val defaultCachedFile = defaultTmpDir.resolve("input.txt")
-  private val newTmpDir = Files.createTempDirectory("newFiles").toAbsolutePath
-  private val newTmpFile = newTmpDir.resolve(defaultCachedFile.getFileName())
-  private val cachedResults = new CachedResultLocalizationMock()
-  private val defaultFileArray = Seq("arrInput1.txt", "arrInput2.txt", "arrInput3.txt").map(defaultTmpDir.resolve(_).toAbsolutePath)
-  private val newFileArray = Seq("arrInput1.txt", "arrInput2.txt", "arrInput3.txt").map(newTmpDir.resolve(_).toAbsolutePath)
-
-  override def afterAll() = {
-    Seq(defaultCachedFile, newTmpFile) ++ newFileArray ++ Seq(defaultTmpDir, newTmpDir) foreach { _.toFile.delete() }
-  }
-
-  "CachedResultLocalization" should {
-    "localize file path via symbolic link" in {
-      val slPath = cachedResults.localizePathViaSymbolicLink(defaultCachedFile, newTmpFile)
-      assert(Files.isSymbolicLink(slPath))
-      Files.delete(newTmpFile)
-    }
-
-    "not localize dir path via symbolic link" in {
-      assertThrows[UnsupportedOperationException](cachedResults.localizePathViaSymbolicLink(defaultTmpDir, newTmpFile))
-    }
-
-    "localize cached job outputs which are WDL files using symbolic link" in {
-      val outputs: CallOutputs = Map("File1" -> JobOutput(WdlSingleFile(defaultCachedFile.toAbsolutePath.toString)))
-      val newJobOutputs = cachedResults.localizeCachedOutputs(newTmpDir, outputs)
-      newJobOutputs foreach { case (lqn, jobOutput) =>
-        assert(jobOutput.wdlValue.valueString == newTmpFile.toString)
-      }
-    }
-
-    "localize cached job outputs which are WDL File Array using symbolic link" in {
-      val wdlArray = WdlArray(WdlArrayType(WdlFileType), defaultFileArray.map(file => WdlSingleFile(file.toString())))
-      val outputs = Map("File1" -> JobOutput(wdlArray))
-      val newJobOutputs = cachedResults.localizeCachedOutputs(newTmpDir, outputs)
-      newJobOutputs foreach { case (lqn, jobOutput) =>
-        val wdlArray = jobOutput.wdlValue.asInstanceOf[WdlArray].value
-        wdlArray foreach { entry =>
-          assert(!entry.valueString.contains(defaultTmpDir.toString))
-          assert(entry.valueString.contains(newTmpDir.toString))
-        }
-      }
-    }
-
-    "not localize cached job outputs which are not WDL files" in {
-      val outputs = Map("String1" -> JobOutput(WdlString(defaultCachedFile.toAbsolutePath.toString)))
-      val newJobOutputs = cachedResults.localizeCachedOutputs(newTmpDir, outputs)
-      newJobOutputs foreach { case (lqn, jobOutput) =>
-        assert(jobOutput.wdlValue.valueString == defaultCachedFile.toString)
-      }
-    }
-  }
-}
diff --git a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/MongoCacheActorSpec.scala b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/MongoCacheActorSpec.scala
deleted file mode 100644
index d638b82d9..000000000
--- a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/MongoCacheActorSpec.scala
+++ /dev/null
@@ -1,104 +0,0 @@
-package cromwell.backend.impl.htcondor.caching.provider.mongodb
-
-import akka.actor.ActorSystem
-import akka.testkit.{ImplicitSender, TestActorRef, TestKit}
-import com.mongodb.casbah.MongoCollection
-import com.mongodb.casbah.commons.MongoDBObject
-import com.mongodb.util.JSON
-import com.mongodb.{DBObject, WriteResult}
-import com.typesafe.config.{Config, ConfigFactory}
-import cromwell.backend.{MemorySize, BackendJobDescriptorKey}
-import cromwell.backend.BackendJobExecutionActor.JobSucceededResponse
-import cromwell.backend.impl.htcondor.HtCondorRuntimeAttributes
-import cromwell.backend.impl.htcondor.caching.CacheActor._
-import cromwell.backend.impl.htcondor.caching.exception.CachedResultNotFoundException
-import cromwell.backend.impl.htcondor.caching.provider.mongodb.model.{KryoSerializedObject, MongoCachedExecutionResult}
-import cromwell.backend.impl.htcondor.caching.provider.mongodb.serialization.KryoSerDe
-import cromwell.backend.validation.ContinueOnReturnCodeSet
-import cromwell.core.JobOutput
-import org.mockito.Mockito
-import org.mockito.Mockito._
-import org.scalatest.mockito.MockitoSugar
-import org.scalatest.{BeforeAndAfter, BeforeAndAfterAll, MustMatchers, WordSpecLike}
-import wdl4s.TaskCall
-import wdl4s.values.WdlString
-
-class MongoCacheActorSpec extends TestKit(ActorSystem("MongoCacheProviderActorSpecSystem")) with WordSpecLike with MustMatchers
-  with BeforeAndAfter with BeforeAndAfterAll with ImplicitSender with MockitoSugar with KryoSerDe {
-
-  import spray.json._
-  import cromwell.backend.impl.htcondor.caching.provider.mongodb.model.MongoCachedExecutionResultProtocol._
-
-  val config: Config = ConfigFactory.load()
-  val mongoDbCollectionMock = mock[MongoCollection]
-  val memorySize = MemorySize.parse("0.512 GB").get
-  val diskSize = MemorySize.parse("1.024 GB").get
-  val runtimeConfig = HtCondorRuntimeAttributes(ContinueOnReturnCodeSet(Set(0)), Some("tool-name"), Some("/workingDir"), Some("/outputDir"), true, 1, memorySize, diskSize, None)
-  val jobHash = "88dde49db10f1551299fb9937f313c10"
-  val taskStatus = "done"
-  val succeededResponseMock = JobSucceededResponse(BackendJobDescriptorKey(TaskCall(Option("taskName"), null, null, null), None, 0), None, Map("test" -> JobOutput(WdlString("Test"))), None, Seq.empty)
-  val serSucceededRespMock = KryoSerializedObject(serialize(succeededResponseMock))
-  val cachedExecutionResult = MongoCachedExecutionResult(jobHash, serSucceededRespMock)
-  val cachedExecutionResultDbObject = JSON.parse(cachedExecutionResult.toJson.toString).asInstanceOf[DBObject]
-  val query = MongoDBObject("hash" -> jobHash)
-
-  after {
-    Mockito.reset(mongoDbCollectionMock)
-  }
-
-  override def afterAll = shutdown()
-
-  "A CacheActor" should {
-    "return an ExecutionResultFound when read an execution result from cache" in {
-      when(mongoDbCollectionMock.findOne(query)) thenReturn Some(cachedExecutionResultDbObject)
-      val cacheActor = TestActorRef(new MongoCacheActor(mongoDbCollectionMock))
-      cacheActor ! ReadExecutionResult(jobHash)
-      expectMsg(ExecutionResultFound(succeededResponseMock))
-      verify(mongoDbCollectionMock, atLeastOnce).findOne(query)
-    }
-
-    "return an ExecutionResultNotFound when it can't find an execution result in cache" in {
-      when(mongoDbCollectionMock.findOne(query)) thenReturn None
-      val cacheActor = TestActorRef(new MongoCacheActor(mongoDbCollectionMock))
-      cacheActor ! ReadExecutionResult(jobHash)
-      expectMsg(ExecutionResultNotFound)
-      verify(mongoDbCollectionMock, atLeastOnce).findOne(query)
-    }
-
-    "return ExecutionResultStored when it stores an execution result" in {
-      when(mongoDbCollectionMock.findOne(query)) thenThrow new CachedResultNotFoundException("")
-      when(mongoDbCollectionMock.insert(cachedExecutionResultDbObject)) thenReturn new WriteResult(0, true, "")
-      val cacheActor = TestActorRef(new MongoCacheActor(mongoDbCollectionMock))
-      cacheActor ! StoreExecutionResult(jobHash, succeededResponseMock)
-      expectMsg(ExecutionResultStored("88dde49db10f1551299fb9937f313c10"))
-      verify(mongoDbCollectionMock, atLeastOnce).insert(cachedExecutionResultDbObject)
-    }
-
-    "return ExecutionResultAlreadyExist when it tries to store an existing execution result" in {
-      when(mongoDbCollectionMock.findOne(query)) thenReturn Some(cachedExecutionResultDbObject)
-      val cacheActor = TestActorRef(new MongoCacheActor(mongoDbCollectionMock))
-      cacheActor ! StoreExecutionResult(jobHash, succeededResponseMock)
-      expectMsg(ExecutionResultAlreadyExist)
-      verify(mongoDbCollectionMock, atLeastOnce).findOne(query)
-    }
-
-    "return ExecutionResultNotFound when try to read and force re-write flag is enabled" in {
-      when(mongoDbCollectionMock.findOne(query)) thenReturn Some(cachedExecutionResultDbObject)
-      val cacheActor = TestActorRef(new MongoCacheActor(mongoDbCollectionMock, true))
-      cacheActor ! ReadExecutionResult(jobHash)
-      expectMsg(ExecutionResultNotFound)
-      verify(mongoDbCollectionMock, atLeastOnce).findOne(query)
-    }
-
-    "return ExecutionResultStored when try to store and force re-write flag is enabled" in {
-      when(mongoDbCollectionMock.findOne(query)) thenThrow new CachedResultNotFoundException("")
-      when(mongoDbCollectionMock.insert(cachedExecutionResultDbObject)) thenReturn new WriteResult(0, true, "")
-      when(mongoDbCollectionMock.remove(query)) thenReturn new WriteResult(0, true, "")
-      val cacheActor = TestActorRef(new MongoCacheActor(mongoDbCollectionMock))
-      cacheActor ! StoreExecutionResult(jobHash, succeededResponseMock)
-      expectMsg(ExecutionResultStored("88dde49db10f1551299fb9937f313c10"))
-      verify(mongoDbCollectionMock, atLeastOnce).insert(cachedExecutionResultDbObject)
-      verify(mongoDbCollectionMock, atLeastOnce).remove(query)
-    }
-  }
-}
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/GenomicsFactory.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/GenomicsFactory.scala
index 574e58153..edbdac0f6 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/GenomicsFactory.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/GenomicsFactory.scala
@@ -2,20 +2,29 @@ package cromwell.backend.impl.jes
 
 import java.net.URL
 
+import com.google.api.client.http.{HttpRequest, HttpRequestInitializer}
 import com.google.api.services.genomics.Genomics
-import cromwell.core.WorkflowOptions
+import com.google.auth.Credentials
+import com.google.auth.http.HttpCredentialsAdapter
 import cromwell.filesystems.gcs.auth.GoogleAuthMode
 
 
 case class GenomicsFactory(applicationName: String, authMode: GoogleAuthMode, endpointUrl: URL) {
 
-  def withOptions(options: WorkflowOptions) = {
-    val credential = authMode.credential(options)
+  def fromCredentials(credentials: Credentials) = {
+    val httpRequestInitializer = {
+      val delegate = new HttpCredentialsAdapter(credentials)
+      new HttpRequestInitializer() {
+        def initialize(httpRequest: HttpRequest) = {
+          delegate.initialize(httpRequest)
+        }
+      }
+    }
 
     new Genomics.Builder(
-      credential.getTransport,
-      credential.getJsonFactory,
-      credential)
+      GoogleAuthMode.httpTransport,
+      GoogleAuthMode.jsonFactory,
+      httpRequestInitializer)
       .setApplicationName(applicationName)
       .setRootUrl(endpointUrl.toString)
       .build
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/GoogleLabels.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/GoogleLabels.scala
new file mode 100644
index 000000000..a2bcbddf8
--- /dev/null
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/GoogleLabels.scala
@@ -0,0 +1,55 @@
+package cromwell.backend.impl.jes
+
+import cats.data.Validated.Valid
+import cromwell.core.labels.{Label, Labels}
+
+object GoogleLabels {
+
+  val MaxLabelLength = Label.MaxLabelLength
+  val GoogleLabelsRegexPattern = Label.LabelKeyRegex
+
+  // This function is used to coerce a string into one that meets the requirements for a label submission to JES.
+  // See 'labels' in https://cloud.google.com/genomics/reference/rpc/google.genomics.v1alpha2#google.genomics.v1alpha2.RunPipelineArgs
+  def safeGoogleName(mainText: String, emptyAllowed: Boolean = false): String = {
+
+    Label.validateLabelRegex(mainText, GoogleLabelsRegexPattern.r) match {
+      case Valid(labelText) => labelText
+      case invalid @ _ if mainText.equals("") && emptyAllowed => mainText
+      case invalid @ _ =>
+        def appendSafe(current: String, nextChar: Char): String = {
+          nextChar match {
+            case c if c.isLetterOrDigit || c == '-' => current + c.toLower
+            case _ => current + '-'
+          }
+        }
+
+        val foldResult = mainText.toCharArray.foldLeft("")(appendSafe)
+
+        val startsValid = foldResult.headOption.exists(_.isLetter)
+        val endsValid = foldResult.lastOption.exists(_.isLetterOrDigit)
+
+        val validStart = if (startsValid) foldResult else "x--" + foldResult
+        val validStartAndEnd = if (endsValid) validStart else validStart + "--x"
+
+        val length = validStartAndEnd.length
+        val tooLong = length > MaxLabelLength
+
+        if (tooLong) {
+          val middleSeparator = "---"
+          val subSectionLength = (MaxLabelLength - middleSeparator.length) / 2
+          validStartAndEnd.substring(0, subSectionLength) + middleSeparator + validStartAndEnd.substring(length - subSectionLength, length)
+        } else {
+          validStartAndEnd
+        }
+    }
+  }
+
+  def toLabels(values: (String, String)*): Labels = {
+
+    def safeGoogleLabel(key: String, value: String): Label = {
+      Label(safeGoogleName(key), safeGoogleName(value, emptyAllowed = true))
+    }
+
+    Labels(values.toVector map (safeGoogleLabel _ ).tupled)
+  }
+}
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala
index 65050a67c..9d229b3ee 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala
@@ -3,11 +3,14 @@ package cromwell.backend.impl.jes
 import java.net.SocketTimeoutException
 
 import akka.actor.ActorRef
+import cats.data.Validated.{Invalid, Valid}
 import com.google.api.client.googleapis.json.GoogleJsonResponseException
 import com.google.api.services.genomics.model.RunPipelineRequest
+import com.google.cloud.storage.contrib.nio.CloudStorageOptions
 import cromwell.backend._
-import cromwell.backend.async.{AbortedExecutionHandle, ExecutionHandle, FailedNonRetryableExecutionHandle, FailedRetryableExecutionHandle, PendingExecutionHandle}
 import cromwell.backend.impl.jes.RunStatus.TerminalRunStatus
+import cromwell.backend.async.{AbortedExecutionHandle, ExecutionHandle, FailedNonRetryableExecutionHandle, FailedRetryableExecutionHandle, PendingExecutionHandle}
+import cromwell.backend.impl.jes.errors.FailedToDelocalizeFailure
 import cromwell.backend.impl.jes.io._
 import cromwell.backend.impl.jes.statuspolling.{JesRunCreationClient, JesStatusRequestClient}
 import cromwell.backend.standard.{StandardAsyncExecutionActor, StandardAsyncExecutionActorParams, StandardAsyncJob}
@@ -15,17 +18,21 @@ import cromwell.core._
 import cromwell.core.logging.JobLogger
 import cromwell.core.path.{DefaultPathBuilder, Path}
 import cromwell.core.retry.SimpleExponentialBackoff
+import cromwell.services.keyvalue.KeyValueServiceActor._
 import cromwell.filesystems.gcs.GcsPath
+import lenthall.validation.ErrorOr.ErrorOr
+import cromwell.filesystems.gcs.batch.GcsBatchCommandBuilder
+import cromwell.services.keyvalue.KvClient
 import org.slf4j.LoggerFactory
-import wdl4s._
-import wdl4s.expression.NoFunctions
-import wdl4s.values._
+import wdl4s.wdl._
+import wdl4s.wdl.expression.PureStandardLibraryFunctions
+import wdl4s.wdl.values._
 
+import scala.collection.JavaConverters._
 import scala.concurrent.duration._
-import scala.concurrent.{ExecutionContext, Future}
+import scala.concurrent.Future
 import scala.language.postfixOps
 import scala.util.{Success, Try}
-import scala.collection.JavaConverters._
 
 object JesAsyncBackendJobExecutionActor {
   val JesOperationIdKey = "__jes_operation_id"
@@ -36,16 +43,29 @@ object JesAsyncBackendJobExecutionActor {
     val GoogleComputeServiceAccount = "google_compute_service_account"
   }
 
-  type JesPendingExecutionHandle =
-    PendingExecutionHandle[StandardAsyncJob, Run, RunStatus]
+  type JesPendingExecutionHandle = PendingExecutionHandle[StandardAsyncJob, Run, RunStatus]
 
   private val ExtraConfigParamName = "__extra_config_gcs_path"
+
   private def stringifyMap(m: Map[String, String]): String = m map { case(k, v) => s"  $k -> $v"} mkString "\n"
+
+  val maxUnexpectedRetries = 2
+
+  val GoogleCancelledRpc = 1
+  val GoogleNotFoundRpc = 5
+  val GoogleAbortedRpc = 10 // Note "Aborted" here is not the same as our "abort"
+  val JesFailedToDelocalize = 5
+  val JesUnexpectedTermination = 13
+  val JesPreemption = 14
+
+  def StandardException(errorCode: Int, message: String, jobTag: String) = {
+    new Exception(s"Task $jobTag failed. JES error code $errorCode. $message")
+  }
 }
 
 class JesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyncExecutionActorParams)
   extends BackendJobLifecycleActor with StandardAsyncExecutionActor with JesJobCachingActorHelper
-    with JesStatusRequestClient with JesRunCreationClient  {
+    with JesStatusRequestClient with JesRunCreationClient with GcsBatchCommandBuilder with KvClient {
 
   import JesAsyncBackendJobExecutionActor._
 
@@ -68,7 +88,6 @@ class JesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn
   override lazy val executeOrRecoverBackOff = SimpleExponentialBackoff(
     initialInterval = 3 seconds, maxInterval = 20 seconds, multiplier = 1.1)
 
-  override lazy val retryable: Boolean = jobDescriptor.key.attempt <= runtimeAttributes.preemptible
   private lazy val cmdInput =
     JesFileInput(ExecParamName, jesCallPaths.script.pathAsString, DefaultPathBuilder.get(jesCallPaths.scriptFilename), workingDisk)
   private lazy val jesCommandLine = s"/bin/bash ${cmdInput.containerPath}"
@@ -78,15 +97,28 @@ class JesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn
 
   private lazy val dockerConfiguration = jesConfiguration.dockerCredentials
 
+  private val previousRetryReasons: ErrorOr[PreviousRetryReasons] = PreviousRetryReasons.tryApply(jobDescriptor.prefetchedKvStoreEntries, jobDescriptor.key.attempt)
+
+  private lazy val jobDockerImage = jobDescriptor.maybeCallCachingEligible.dockerHash.getOrElse(runtimeAttributes.dockerImage)
+  
+  override lazy val dockerImageUsed: Option[String] = Option(jobDockerImage)
+  
+  override val preemptible: Boolean = previousRetryReasons match {
+    case Valid(PreviousRetryReasons(p, _)) => p < maxPreemption
+    case _ => false
+  }
+
   override def tryAbort(job: StandardAsyncJob): Unit = {
     Run(job, initializationData.genomics).abort()
   }
 
-  override def receive: Receive = pollingActorClientReceive orElse runCreationClientReceive orElse super.receive
+  override def requestsAbortAndDiesImmediately: Boolean = true
+
+  override def receive: Receive = pollingActorClientReceive orElse runCreationClientReceive orElse ioReceive orElse kvClientReceive orElse super.receive
 
   private def gcsAuthParameter: Option[JesInput] = {
     if (jesAttributes.auths.gcs.requiresAuthFile || dockerConfiguration.isDefined)
-      Option(JesLiteralInput(ExtraConfigParamName, jesCallPaths.gcsAuthFilePath.pathAsString))
+      Option(JesLiteralInput(ExtraConfigParamName, jesCallPaths.workflowPaths.gcsAuthFilePath.pathAsString))
     else None
   }
 
@@ -118,17 +150,19 @@ class JesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn
 
   private[jes] def generateJesInputs(jobDescriptor: BackendJobDescriptor): Set[JesInput] = {
 
-    val writeFunctionFiles = call.task.evaluateFilesFromCommand(jobDescriptor.fullyQualifiedInputs, backendEngineFunctions) map {
-      case (expression, file) =>  expression.toWdlString.md5SumShort -> Seq(file)
+    val fullyQualifiedPreprocessedInputs = jobDescriptor.inputDeclarations map { case (declaration, value) => declaration.fullyQualifiedName -> commandLineValueMapper(value) }
+    val writeFunctionFiles = call.task.evaluateFilesFromCommand(fullyQualifiedPreprocessedInputs, backendEngineFunctions) map {
+      case (expression, file) => expression.toWdlString.md5SumShort -> Seq(file)
     }
 
     /* Collect all WdlFiles from inputs to the call */
-    val callInputFiles: Map[FullyQualifiedName, Seq[WdlFile]] = jobDescriptor.fullyQualifiedInputs mapValues { _.collectAsSeq { case w: WdlFile => w } }
+    val callInputFiles: Map[FullyQualifiedName, Seq[WdlFile]] = jobDescriptor.fullyQualifiedInputs mapValues {
+      _.collectAsSeq { case w: WdlFile => w }
+    }
 
     val inputs = (callInputFiles ++ writeFunctionFiles) flatMap {
       case (name, files) => jesInputsFromWdlFiles(name, files, files.map(relativeLocalizationPath), jobDescriptor)
     }
-
     inputs.toSet
   }
 
@@ -160,7 +194,7 @@ class JesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn
   }
 
   private[jes] def generateJesOutputs(jobDescriptor: BackendJobDescriptor): Set[JesFileOutput] = {
-    val wdlFileOutputs = call.task.findOutputFiles(jobDescriptor.fullyQualifiedInputs, NoFunctions) map relativeLocalizationPath
+    val wdlFileOutputs = call.task.findOutputFiles(jobDescriptor.fullyQualifiedInputs, PureStandardLibraryFunctions) map relativeLocalizationPath
 
     val outputs = wdlFileOutputs.distinct flatMap { wdlFile =>
       wdlFile match {
@@ -230,13 +264,15 @@ class JesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn
     val runPipelineParameters = Run.makeRunPipelineRequest(
       jobDescriptor = jobDescriptor,
       runtimeAttributes = runtimeAttributes,
+      dockerImage = jobDockerImage,
       callRootPath = callRootPath.pathAsString,
       commandLine = jesCommandLine,
       logFileName = jesLogFilename,
       jesParameters,
       googleProject(jobDescriptor.workflowDescriptor),
       computeServiceAccount(jobDescriptor.workflowDescriptor),
-      retryable,
+      backendLabels,
+      preemptible,
       initializationData.genomics
     )
     logger.debug(s"Inputs:\n${stringifyMap(runPipelineParameters.getPipelineArgs.getInputs.asScala.toMap)}")
@@ -248,26 +284,29 @@ class JesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn
 
   override def isTransient(throwable: Throwable): Boolean = isTransientJesException(throwable)
 
-  override def executeAsync()(implicit ec: ExecutionContext): Future[ExecutionHandle] = {
-    runWithJes(None)
-  }
+  override def executeAsync(): Future[ExecutionHandle] = runWithJes(None)
 
-  override def recoverAsync(jobId: StandardAsyncJob)(implicit ec: ExecutionContext): Future[ExecutionHandle] = {
-    runWithJes(Option(jobId))
-  }
+  val futureKvJobKey = KvJobKey(jobDescriptor.key.call.fullyQualifiedName, jobDescriptor.key.index, jobDescriptor.key.attempt + 1)
 
-  private def runWithJes(jobForResumption: Option[StandardAsyncJob]): Future[ExecutionHandle] = {
+  override def recoverAsync(jobId: StandardAsyncJob): Future[ExecutionHandle] = runWithJes(Option(jobId))
 
+  private def runWithJes(jobForResumption: Option[StandardAsyncJob]): Future[ExecutionHandle] = {
     // Want to force runtimeAttributes to evaluate so we can fail quickly now if we need to:
-    def makeRpr: Try[RunPipelineRequest] = Try(runtimeAttributes) flatMap { _ => Try {
-      val jesInputs: Set[JesInput] = generateJesInputs(jobDescriptor) ++ monitoringScript + cmdInput
+    def evaluateRuntimeAttributes = Future.fromTry(Try(runtimeAttributes))
+
+    def generateJesParameters = Future.fromTry( Try {
+      val generatedJesInputs = generateJesInputs(jobDescriptor)
+      val jesInputs: Set[JesInput] = generatedJesInputs ++ monitoringScript + cmdInput
       val jesOutputs: Set[JesFileOutput] = generateJesOutputs(jobDescriptor) ++ monitoringOutput
 
-      val jesParameters = standardParameters ++ gcsAuthParameter ++ jesInputs ++ jesOutputs
+      standardParameters ++ gcsAuthParameter ++ jesInputs ++ jesOutputs
+    })
+
+    def uploadScriptFile = writeAsync(jobPaths.script, commandScriptContents, Seq(CloudStorageOptions.withMimeType("text/plain")))
 
-      jobPaths.script.writeAsText(commandScriptContents)
+    def makeRpr(jesParameters: Seq[JesParameter]) = Future.fromTry(Try {
       createJesRunPipelineRequest(jesParameters)
-    }}
+    })
 
     jobForResumption match {
       case Some(job) =>
@@ -275,18 +314,17 @@ class JesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn
         Future.successful(PendingExecutionHandle(jobDescriptor, job, Option(run), previousStatus = None))
       case None =>
         for {
-          rpr <- Future.fromTry(makeRpr)
+          _ <- evaluateRuntimeAttributes
+          jesParameters <- generateJesParameters
+          _ <- uploadScriptFile
+          rpr <- makeRpr(jesParameters)
           runId <- runPipeline(initializationData.genomics, rpr)
           run = Run(runId, initializationData.genomics)
         } yield PendingExecutionHandle(jobDescriptor, runId, Option(run), previousStatus = None)
     }
   }
 
-  override def pollStatusAsync(handle: JesPendingExecutionHandle)
-                              (implicit ec: ExecutionContext): Future[RunStatus] = {
-    super[JesStatusRequestClient].pollStatus(handle.runInfo.get)
-  }
-
+  override def pollStatusAsync(handle: JesPendingExecutionHandle): Future[RunStatus] = super[JesStatusRequestClient].pollStatus(handle.runInfo.get)
 
   override def customPollStatusFailure: PartialFunction[(ExecutionHandle, Exception), ExecutionHandle] = {
     case (oldHandle: JesPendingExecutionHandle@unchecked, e: GoogleJsonResponseException) if e.getStatusCode == 404 =>
@@ -321,10 +359,8 @@ class JesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn
   override def isSuccess(runStatus: RunStatus): Boolean = {
     runStatus match {
       case _: RunStatus.Success => true
-      case _: RunStatus.Failed => false
-      case unknown =>
-        throw new RuntimeException("isSuccess not called with RunStatus.Success or RunStatus.Failed. " +
-          s"Instead got $unknown")
+      case _: RunStatus.UnsuccessfulRunStatus => false
+      case _ => throw new RuntimeException(s"Cromwell programmer blunder: isSuccess was called on an incomplete RunStatus ($runStatus).")
     }
   }
 
@@ -344,62 +380,87 @@ class JesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn
     }
   }
 
-  private def extractErrorCodeFromErrorMessage(errorMessage: String): Int = {
-    errorMessage.substring(0, errorMessage.indexOf(':')).toInt
+  // If one exists, extract the JES error code (not the google RPC) from the error message
+  private[jes] def getJesErrorCode(errorMessage: String): Option[Int] = {
+    Try { errorMessage.substring(0, errorMessage.indexOf(':')).toInt } toOption
   }
 
-  private def preempted(errorCode: Int, errorMessage: String): Boolean = {
-    def isPreemptionCode(code: Int) = code == 13 || code == 14
+  override def handleExecutionFailure(runStatus: RunStatus,
+                                      handle: StandardAsyncPendingExecutionHandle,
+                                      returnCode: Option[Int]): Future[ExecutionHandle] = {
+    // Inner function: Handles a 'Failed' runStatus (or Preempted if preemptible was false)
+    def handleFailedRunStatus(runStatus: RunStatus.UnsuccessfulRunStatus,
+                              handle: StandardAsyncPendingExecutionHandle,
+                              returnCode: Option[Int]): Future[ExecutionHandle] = {
+      (runStatus.errorCode, runStatus.jesCode) match {
+        case (GoogleCancelledRpc, None) => Future.successful(AbortedExecutionHandle)
+        case (GoogleNotFoundRpc, Some(JesFailedToDelocalize)) => Future.successful(FailedNonRetryableExecutionHandle(FailedToDelocalizeFailure(runStatus.prettyPrintedError, jobTag, Option(jobPaths.stderr))))
+        case (GoogleAbortedRpc, Some(JesUnexpectedTermination)) => handleUnexpectedTermination(runStatus.errorCode, runStatus.prettyPrintedError, returnCode)
+        case _ => Future.successful(FailedNonRetryableExecutionHandle(StandardException(runStatus.errorCode, runStatus.prettyPrintedError, jobTag), returnCode))
+      }
+    }
 
-    try {
-      errorCode == 10 && isPreemptionCode(extractErrorCodeFromErrorMessage(errorMessage)) && preemptible
-    } catch {
-      case _: NumberFormatException | _: StringIndexOutOfBoundsException =>
-        jobLogger.warn(s"Unable to parse JES error code from error messages: [{}], assuming this was not a preempted VM.", errorMessage.mkString(", "))
-        false
+    runStatus match {
+      case preemptedStatus: RunStatus.Preempted if preemptible => handlePreemption(preemptedStatus, returnCode)
+      case failedStatus: RunStatus.UnsuccessfulRunStatus => handleFailedRunStatus(failedStatus, handle, returnCode)
+      case unknown => throw new RuntimeException(s"handleExecutionFailure not called with RunStatus.Failed or RunStatus.Preempted. Instead got $unknown")
     }
   }
 
-  override def handleExecutionFailure(runStatus: RunStatus,
-                                      handle: StandardAsyncPendingExecutionHandle,
-                                      returnCode: Option[Int]): ExecutionHandle = {
-    import lenthall.numeric.IntegerUtil._
+  private def writeFuturePreemptedAndUnexpectedRetryCounts(p: Int, ur: Int): Future[Unit] = {
+    val updateRequests = Seq(
+      KvPut(KvPair(ScopedKey(workflowId, futureKvJobKey, JesBackendLifecycleActorFactory.unexpectedRetryCountKey), Option(ur.toString))),
+      KvPut(KvPair(ScopedKey(workflowId, futureKvJobKey, JesBackendLifecycleActorFactory.preemptionCountKey), Option(p.toString)))
+    )
 
-    val failed = runStatus match {
-      case failedStatus: RunStatus.Failed => failedStatus
-      case unknown => throw new RuntimeException(s"handleExecutionFailure not called with RunStatus.Failed. Instead got $unknown")
-    }
+    makeKvRequest(updateRequests).map(_ => ())
+  }
+
+  private def handleUnexpectedTermination(errorCode: Int, errorMessage: String, jobReturnCode: Option[Int]): Future[ExecutionHandle] = {
+
+    val msg = s"Retrying. $errorMessage"
 
-    // In the event that the error isn't caught by the special cases, this will be used
-    def nonRetryableException = {
-      val exception = failed.toFailure(jobPaths.jobKey.tag, Option(jobPaths.stderr))
-      FailedNonRetryableExecutionHandle(exception, returnCode)
+    previousRetryReasons match {
+      case Valid(PreviousRetryReasons(p, ur)) =>
+        val thisUnexpectedRetry = ur + 1
+        if (thisUnexpectedRetry <= maxUnexpectedRetries) {
+          // Increment unexpected retry count and preemption count stays the same
+          writeFuturePreemptedAndUnexpectedRetryCounts(p, thisUnexpectedRetry).map { _ =>
+            FailedRetryableExecutionHandle(StandardException(errorCode, msg, jobTag), jobReturnCode)
+          }
+        }
+        else {
+          Future.successful(FailedNonRetryableExecutionHandle(StandardException(errorCode, errorMessage, jobTag), jobReturnCode))
+        }
+      case Invalid(_) =>
+        Future.successful(FailedNonRetryableExecutionHandle(StandardException(errorCode, errorMessage, jobTag), jobReturnCode))
     }
+  }
 
-    val errorCode = failed.errorCode
-    val taskName = s"${workflowDescriptor.id}:${call.unqualifiedName}"
-    val attempt = jobDescriptor.key.attempt
-
-    failed.errorMessage match {
-      case Some(errorMessage) =>
-        if (errorMessage.contains("Operation canceled at"))  {
-          AbortedExecutionHandle
-        } else if (preempted(errorCode, errorMessage)) {
-          val preemptedMsg = s"Task $taskName was preempted for the ${attempt.toOrdinal} time."
-          if (attempt < maxPreemption) {
-            val e = PreemptedException(
-              s"""$preemptedMsg The call will be restarted with another preemptible VM (max preemptible attempts number is $maxPreemption).
-                 |Error code $errorCode. Message: $errorMessage""".stripMargin
-            )
-            FailedRetryableExecutionHandle(e, returnCode)
-          } else {
-            val e = PreemptedException(
-              s"""$preemptedMsg The maximum number of preemptible attempts ($maxPreemption) has been reached. The call will be restarted with a non-preemptible VM.
-                 |Error code $errorCode. Message: $errorMessage)""".stripMargin)
-            FailedRetryableExecutionHandle(e, returnCode)
+  private def handlePreemption(runStatus: RunStatus.Preempted, jobReturnCode: Option[Int]): Future[ExecutionHandle] = {
+    import lenthall.numeric.IntegerUtil._
+
+    val errorCode: Int = runStatus.errorCode
+    val prettyPrintedError: String = runStatus.prettyPrintedError
+    previousRetryReasons match {
+      case Valid(PreviousRetryReasons(p, ur)) =>
+        val thisPreemption = p + 1
+        val taskName = s"${workflowDescriptor.id}:${call.unqualifiedName}"
+        val baseMsg = s"Task $taskName was preempted for the ${thisPreemption.toOrdinal} time."
+
+        writeFuturePreemptedAndUnexpectedRetryCounts(thisPreemption, ur).map { _ =>
+          if (thisPreemption < maxPreemption) {
+            // Increment preemption count and unexpectedRetryCount stays the same
+            val msg = s"""$baseMsg The call will be restarted with another preemptible VM (max preemptible attempts number is $maxPreemption). Error code $errorCode.$prettyPrintedError""".stripMargin
+            FailedRetryableExecutionHandle(StandardException(errorCode, msg, jobTag), jobReturnCode)
+          }
+          else {
+            val msg = s"""$baseMsg The maximum number of preemptible attempts ($maxPreemption) has been reached. The call will be restarted with a non-preemptible VM. Error code $errorCode.$prettyPrintedError)""".stripMargin
+            FailedRetryableExecutionHandle(StandardException(errorCode, msg, jobTag), jobReturnCode)
           }
-        } else { nonRetryableException }
-      case None => nonRetryableException
+        }
+      case Invalid(_) =>
+        Future.successful(FailedNonRetryableExecutionHandle(StandardException(errorCode, prettyPrintedError, jobTag), jobReturnCode))
     }
   }
 
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala
index 5b85dc05e..73d681812 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala
@@ -2,12 +2,12 @@ package cromwell.backend.impl.jes
 
 import java.net.{URI, URL}
 
-import cats.data.NonEmptyList
 import cats.data.Validated._
 import cats.syntax.cartesian._
 import cats.syntax.validated._
 import com.typesafe.config.{Config, ConfigValue}
 import cromwell.backend.impl.jes.authentication.JesAuths
+import cromwell.backend.impl.jes.callcaching.{CopyCachedOutputs, JesCacheHitDuplicationStrategy, UseOriginalCachedOutputs}
 import cromwell.filesystems.gcs.GoogleConfiguration
 import eu.timepit.refined.api.Refined
 import eu.timepit.refined.numeric.Positive
@@ -16,18 +16,19 @@ import lenthall.validation.ErrorOr._
 import lenthall.validation.Validation._
 import net.ceedubs.ficus.Ficus._
 import net.ceedubs.ficus.readers.{StringReader, ValueReader}
-import org.slf4j.LoggerFactory
+import org.slf4j.{Logger, LoggerFactory}
 
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
 
 case class JesAttributes(project: String,
                          computeServiceAccount: String,
                          auths: JesAuths,
+                         restrictMetadataAccess: Boolean,
                          executionBucket: String,
                          endpointUrl: URL,
                          maxPollingInterval: Int,
                          qps: Int Refined Positive,
-                         defaultZones: NonEmptyList[String])
+                         duplicationStrategy: JesCacheHitDuplicationStrategy)
 
 object JesAttributes {
   lazy val Logger = LoggerFactory.getLogger("JesAttributes") 
@@ -45,10 +46,15 @@ object JesAttributes {
     "genomics",
     "filesystems",
     "genomics.auth",
+    "genomics.restrict-metadata-access",
     "genomics.endpoint-url",
     "filesystems.gcs.auth",
-    "genomics-api-queries-per-100-seconds",
-    "genomics.default-zones"
+    "filesystems.gcs.caching.duplication-strategy",
+    "genomics-api-queries-per-100-seconds"
+  )
+
+  private val deprecatedJesKeys: Map[String, String] = Map(
+    "genomics.default-zones" -> "default-runtime-attributes.zones"
   )
 
   private val context = "Jes"
@@ -56,24 +62,37 @@ object JesAttributes {
   implicit val urlReader: ValueReader[URL] = StringReader.stringValueReader.map { URI.create(_).toURL }
   
   def apply(googleConfig: GoogleConfiguration, backendConfig: Config): JesAttributes = {
-    val configKeys = backendConfig.entrySet().toSet map { entry: java.util.Map.Entry[String, ConfigValue] => entry.getKey }
+    val configKeys = backendConfig.entrySet().asScala.toSet map { entry: java.util.Map.Entry[String, ConfigValue] => entry.getKey }
     warnNotRecognized(configKeys, jesKeys, context, Logger)
 
+    def warnDeprecated(keys: Set[String], deprecated: Map[String, String], context: String, logger: Logger) = {
+      val deprecatedKeys = keys.intersect(deprecated.keySet)
+      deprecatedKeys foreach { key => logger.warn(s"Found deprecated configuration key $key, replaced with ${deprecated.get(key)}") }
+    }
+
+    warnDeprecated(configKeys, deprecatedJesKeys, context, Logger)
+
     val project: ErrorOr[String] = validate { backendConfig.as[String]("project") }
     val executionBucket: ErrorOr[String] = validate { backendConfig.as[String]("root") }
     val endpointUrl: ErrorOr[URL] = validate { backendConfig.as[URL]("genomics.endpoint-url") }
     val maxPollingInterval: Int = backendConfig.as[Option[Int]]("maximum-polling-interval").getOrElse(600)
     val computeServiceAccount: String = backendConfig.as[Option[String]]("genomics.compute-service-account").getOrElse("default")
     val genomicsAuthName: ErrorOr[String] = validate { backendConfig.as[String]("genomics.auth") }
+    val genomicsRestrictMetadataAccess: ErrorOr[Boolean] = validate { backendConfig.as[Option[Boolean]]("genomics.restrict-metadata-access").getOrElse(false) }
     val gcsFilesystemAuthName: ErrorOr[String] = validate { backendConfig.as[String]("filesystems.gcs.auth") }
     val qpsValidation = validateQps(backendConfig)
-    val defaultZones = defaultZonesFromConfig(backendConfig)
+    val duplicationStrategy = validate { backendConfig.as[Option[String]]("filesystems.gcs.caching.duplication-strategy").getOrElse("copy") match {
+      case "copy" => CopyCachedOutputs
+      case "reference" => UseOriginalCachedOutputs
+      case other => throw new IllegalArgumentException(s"Unrecognized caching duplication strategy: $other. Supported strategies are copy and reference. See reference.conf for more details.")
+    } }
 
-    (project |@| executionBucket |@| endpointUrl |@| genomicsAuthName |@| gcsFilesystemAuthName |@| defaultZones |@| qpsValidation) map {
-      (_, _, _, _, _, _, _)
-    } flatMap { case (p, b, u, genomicsName, gcsName, d, qps) =>
+
+    (project |@| executionBucket |@| endpointUrl |@| genomicsAuthName |@| genomicsRestrictMetadataAccess |@| gcsFilesystemAuthName |@|
+      qpsValidation |@| duplicationStrategy).tupled flatMap {
+      case (p, b, u, genomicsName, restrictMetadata, gcsName, qps, cachingStrategy) =>
       (googleConfig.auth(genomicsName) |@| googleConfig.auth(gcsName)) map { case (genomicsAuth, gcsAuth) =>
-        JesAttributes(p, computeServiceAccount, JesAuths(genomicsAuth, gcsAuth), b, u, maxPollingInterval, qps, d)
+        JesAttributes(p, computeServiceAccount, JesAuths(genomicsAuth, gcsAuth), restrictMetadata, b, u, maxPollingInterval, qps, cachingStrategy)
       }
     } match {
       case Valid(r) => r
@@ -85,14 +104,6 @@ object JesAttributes {
     }
   }
 
-  def defaultZonesFromConfig(config: Config): ErrorOr[NonEmptyList[String]] = {
-    val zones = config.as[Option[List[String]]]("genomics.default-zones").getOrElse(List("us-central1-b"))
-    zones match {
-      case x :: xs => NonEmptyList(x, xs).validNel
-      case _ => "genomics.default-zones was set but no values were provided".invalidNel
-    }
-  }
-
   def validateQps(config: Config): ErrorOr[Int Refined Positive] = {
     import eu.timepit.refined._
 
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendInitializationData.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendInitializationData.scala
index de4295a49..8aeebe639 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendInitializationData.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendInitializationData.scala
@@ -1,14 +1,14 @@
 package cromwell.backend.impl.jes
 
 import com.google.api.services.genomics.Genomics
+import com.google.auth.Credentials
 import cromwell.backend.standard.{StandardInitializationData, StandardValidatedRuntimeAttributesBuilder}
-import cromwell.filesystems.gcs.auth.GoogleCredentialBundle
 
 case class JesBackendInitializationData
 (
   override val workflowPaths: JesWorkflowPaths,
   override val runtimeAttributesBuilder: StandardValidatedRuntimeAttributesBuilder,
   jesConfiguration: JesConfiguration,
-  gcsCredentials: GoogleCredentialBundle,
+  gcsCredentials: Credentials,
   genomics: Genomics
 ) extends StandardInitializationData(workflowPaths, runtimeAttributesBuilder, classOf[JesExpressionFunctions])
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala
index 82d11e271..7476e3c79 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesBackendLifecycleActorFactory.scala
@@ -2,13 +2,14 @@ package cromwell.backend.impl.jes
 
 import akka.actor.ActorRef
 import cromwell.backend._
-import cromwell.backend.callcaching.FileHashingActor.FileHashingFunction
-import cromwell.backend.impl.jes.callcaching.JesBackendFileHashing
+import cromwell.backend.impl.jes.callcaching.{JesBackendCacheHitCopyingActor, JesBackendFileHashingActor}
 import cromwell.backend.standard._
+import cromwell.backend.standard.callcaching.{StandardCacheHitCopyingActor, StandardFileHashingActor}
 import cromwell.core.CallOutputs
-import wdl4s.TaskCall
+import wdl4s.wdl.WdlTaskCall
 
 import scala.util.{Success, Try}
+import cromwell.backend.impl.jes.JesBackendLifecycleActorFactory._
 
 case class JesBackendLifecycleActorFactory(name: String, configurationDescriptor: BackendConfigurationDescriptor)
   extends StandardLifecycleActorFactory {
@@ -25,12 +26,12 @@ case class JesBackendLifecycleActorFactory(name: String, configurationDescriptor
 
   val jesConfiguration = new JesConfiguration(configurationDescriptor)
 
-  override def workflowInitializationActorParams(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[TaskCall],
-                                                 serviceRegistryActor: ActorRef): StandardInitializationActorParams = {
-    JesInitializationActorParams(workflowDescriptor, calls, jesConfiguration, serviceRegistryActor)
+  override def workflowInitializationActorParams(workflowDescriptor: BackendWorkflowDescriptor, ioActor: ActorRef, calls: Set[WdlTaskCall],
+                                                 serviceRegistryActor: ActorRef, restart: Boolean): StandardInitializationActorParams = {
+    JesInitializationActorParams(workflowDescriptor, ioActor, calls, jesConfiguration, serviceRegistryActor, restart)
   }
 
-  override def workflowFinalizationActorParams(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[TaskCall],
+  override def workflowFinalizationActorParams(workflowDescriptor: BackendWorkflowDescriptor, ioActor: ActorRef, calls: Set[WdlTaskCall],
                                               jobExecutionMap: JobExecutionMap, workflowOutputs: CallOutputs,
                                               initializationDataOption: Option[BackendInitializationData]):
   StandardFinalizationActorParams = {
@@ -38,21 +39,31 @@ case class JesBackendLifecycleActorFactory(name: String, configurationDescriptor
     // invocation.  HOWEVER, the finalization actor is created regardless of whether workflow initialization was successful
     // or not.  So the finalization actor must be able to handle an empty `JesBackendInitializationData` option, and there is no
     // `.get` on the initialization data as there is with the execution or cache hit copying actor methods.
-    JesFinalizationActorParams(workflowDescriptor, calls, jesConfiguration, jobExecutionMap, workflowOutputs,
+    JesFinalizationActorParams(workflowDescriptor, ioActor, calls, jesConfiguration, jobExecutionMap, workflowOutputs,
       initializationDataOption)
   }
 
+  override lazy val cacheHitCopyingActorClassOption: Option[Class[_ <: StandardCacheHitCopyingActor]] = {
+    Option(classOf[JesBackendCacheHitCopyingActor])
+  }
+
   override def backendSingletonActorProps = Option(JesBackendSingletonActor.props(jesConfiguration.qps))
 
-  override lazy val fileHashingFunction: Option[FileHashingFunction] = Option(FileHashingFunction(JesBackendFileHashing.getCrc32c))
+  override lazy val fileHashingActorClassOption: Option[Class[_ <: StandardFileHashingActor]] = Option(classOf[JesBackendFileHashingActor])
   
   override def dockerHashCredentials(initializationData: Option[BackendInitializationData]) = {
     Try(BackendInitializationData.as[JesBackendInitializationData](initializationData)) match {
-      case Success(jesData) => 
+      case Success(jesData) =>
         val maybeDockerHubCredentials = jesData.jesConfiguration.dockerCredentials
-        val googleCredentials = Option(jesData.gcsCredentials.credential)
+        val googleCredentials = Option(jesData.gcsCredentials)
         List(maybeDockerHubCredentials, googleCredentials).flatten
       case _ => List.empty[Any]
     }
   }
+  override val requestedKeyValueStoreKeys: Seq[String] = Seq(preemptionCountKey, unexpectedRetryCountKey)
+}
+
+object JesBackendLifecycleActorFactory {
+  val preemptionCountKey = "PreemptionCount"
+  val unexpectedRetryCountKey = "UnexpectedRetryCount"
 }
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesConfiguration.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesConfiguration.scala
index 2c1e82f92..f0e863da6 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesConfiguration.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesConfiguration.scala
@@ -2,37 +2,21 @@ package cromwell.backend.impl.jes
 
 import cromwell.backend.BackendConfigurationDescriptor
 import cromwell.backend.impl.jes.authentication.JesDockerCredentials
-import cromwell.backend.impl.jes.io._
-import cromwell.core.DockerConfiguration
-import cromwell.core.path.CustomRetryParams
-import cromwell.core.retry.SimpleExponentialBackoff
-import cromwell.filesystems.gcs.{GoogleConfiguration, RetryableGcsPathBuilderFactory}
-
-import scala.concurrent.duration._
-import scala.language.postfixOps
-
-object JesConfiguration {
-  val GcsRetryParams = CustomRetryParams(
-    timeout = Duration.Inf,
-    maxRetries = Option(3),
-    backoff = SimpleExponentialBackoff(1 seconds, 3 seconds, 1.5D),
-    isTransient = isTransientJesException,
-    isFatal = isFatalJesException
-  )
-}
+import cromwell.core.BackendDockerConfiguration
+import cromwell.filesystems.gcs.{GcsPathBuilderFactory, GoogleConfiguration}
 
 class JesConfiguration(val configurationDescriptor: BackendConfigurationDescriptor) {
 
-  private val googleConfig = GoogleConfiguration(configurationDescriptor.globalConfig)
+  val googleConfig = GoogleConfiguration(configurationDescriptor.globalConfig)
 
   val root = configurationDescriptor.backendConfig.getString("root")
+  val runtimeConfig = configurationDescriptor.backendRuntimeConfig
   val jesAttributes = JesAttributes(googleConfig, configurationDescriptor.backendConfig)
   val jesAuths = jesAttributes.auths
   val jesComputeServiceAccount = jesAttributes.computeServiceAccount
-  val gcsPathBuilderFactory = RetryableGcsPathBuilderFactory(jesAuths.gcs, customRetryParams = JesConfiguration.GcsRetryParams)
+  val gcsPathBuilderFactory = GcsPathBuilderFactory(jesAuths.gcs, googleConfig.applicationName)
   val genomicsFactory = GenomicsFactory(googleConfig.applicationName, jesAuths.genomics, jesAttributes.endpointUrl)
-  val dockerCredentials = DockerConfiguration.build(configurationDescriptor.backendConfig).dockerCredentials map JesDockerCredentials.apply
-  val needAuthFileUpload = jesAuths.gcs.requiresAuthFile || dockerCredentials.isDefined
+  val dockerCredentials = BackendDockerConfiguration.build(configurationDescriptor.backendConfig).dockerCredentials map JesDockerCredentials.apply
+  val needAuthFileUpload = jesAuths.gcs.requiresAuthFile || dockerCredentials.isDefined || jesAttributes.restrictMetadataAccess
   val qps = jesAttributes.qps
-  val defaultZones = jesAttributes.defaultZones
 }
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesExpressionFunctions.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesExpressionFunctions.scala
index ceda6846f..aed500f22 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesExpressionFunctions.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesExpressionFunctions.scala
@@ -2,14 +2,16 @@ package cromwell.backend.impl.jes
 
 import cromwell.backend.standard.{StandardExpressionFunctions, StandardExpressionFunctionsParams}
 import cromwell.filesystems.gcs.GcsPathBuilder
+import cromwell.filesystems.gcs.GcsPathBuilder.{InvalidGcsPath, PossiblyValidRelativeGcsPath, ValidFullGcsPath}
 
 class JesExpressionFunctions(standardParams: StandardExpressionFunctionsParams)
   extends StandardExpressionFunctions(standardParams) {
 
-  override def preMapping(str: String) =
-    if (!GcsPathBuilder.isValidGcsUrl(str)) {
-      callContext.root.resolve(str.stripPrefix("/")).pathAsString
-    } else {
-      str
+  override def preMapping(str: String) = {
+    GcsPathBuilder.validateGcsPath(str) match {
+      case ValidFullGcsPath => str
+      case PossiblyValidRelativeGcsPath => callContext.root.resolve(str.stripPrefix("/")).pathAsString
+      case invalid: InvalidGcsPath => throw new IllegalArgumentException(invalid.errorMessage)
     }
+  }
 }
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesFinalizationActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesFinalizationActor.scala
index 131cea544..dc03764ac 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesFinalizationActor.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesFinalizationActor.scala
@@ -1,19 +1,20 @@
 package cromwell.backend.impl.jes
 
-import cats.instances.future._
-import cats.syntax.functor._
+import akka.actor.ActorRef
 import cromwell.backend._
 import cromwell.backend.standard.{StandardFinalizationActor, StandardFinalizationActorParams}
 import cromwell.core.CallOutputs
-import wdl4s.TaskCall
+import cromwell.core.io.AsyncIo
+import cromwell.filesystems.gcs.batch.GcsBatchCommandBuilder
+import wdl4s.wdl.WdlTaskCall
 
 import scala.concurrent.Future
-import scala.language.postfixOps
 
 case class JesFinalizationActorParams
 (
   workflowDescriptor: BackendWorkflowDescriptor,
-  calls: Set[TaskCall],
+  ioActor: ActorRef,
+  calls: Set[WdlTaskCall],
   jesConfiguration: JesConfiguration,
   jobExecutionMap: JobExecutionMap,
   workflowOutputs: CallOutputs,
@@ -23,9 +24,11 @@ case class JesFinalizationActorParams
 }
 
 class JesFinalizationActor(val jesParams: JesFinalizationActorParams)
-  extends StandardFinalizationActor(jesParams) {
+  extends StandardFinalizationActor(jesParams) with AsyncIo with GcsBatchCommandBuilder {
 
   lazy val jesConfiguration: JesConfiguration = jesParams.jesConfiguration
+  
+  override def receive = ioReceive orElse super.receive
 
   override def afterAll(): Future[Unit] = {
     for {
@@ -37,8 +40,10 @@ class JesFinalizationActor(val jesParams: JesFinalizationActorParams)
 
   private def deleteAuthenticationFile(): Future[Unit] = {
     (jesConfiguration.needAuthFileUpload, workflowPaths) match {
-      case (true, Some(paths: JesWorkflowPaths)) => Future { paths.gcsAuthFilePath.delete() } void
+      case (true, Some(paths: JesWorkflowPaths)) => deleteAsync(paths.gcsAuthFilePath)
       case _ => Future.successful(())
     }
   }
+
+  override def ioActor: ActorRef = jesParams.ioActor
 }
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesInitializationActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesInitializationActor.scala
index 3a23a9832..4586762b1 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesInitializationActor.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesInitializationActor.scala
@@ -1,80 +1,134 @@
 package cromwell.backend.impl.jes
 
 import java.io.IOException
+import java.nio.file.StandardOpenOption
 
 import akka.actor.ActorRef
-import cromwell.backend.impl.jes.authentication.{GcsLocalizing, JesAuthInformation}
-import cromwell.backend.impl.jes.io._
+import akka.http.scaladsl.model.StatusCodes
+import com.google.api.services.genomics.Genomics
+import com.google.auth.Credentials
+import com.google.cloud.storage.StorageException
+import com.google.cloud.storage.contrib.nio.CloudStorageOptions
+import cromwell.backend.impl.jes.authentication.{GcsLocalizing, JesAuthObject, JesDockerCredentials}
 import cromwell.backend.standard.{StandardInitializationActor, StandardInitializationActorParams, StandardValidatedRuntimeAttributesBuilder}
 import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendWorkflowDescriptor}
+import cromwell.core.CromwellFatalException
+import cromwell.core.io.AsyncIo
 import cromwell.filesystems.gcs.auth.{ClientSecrets, GoogleAuthMode}
-import spray.json.JsObject
-import wdl4s.TaskCall
+import cromwell.filesystems.gcs.batch.GcsBatchCommandBuilder
+import spray.json.{JsObject, JsTrue}
+import wdl4s.wdl.WdlTaskCall
 
 import scala.concurrent.Future
-import scala.util.Try
 
 case class JesInitializationActorParams
 (
   workflowDescriptor: BackendWorkflowDescriptor,
-  calls: Set[TaskCall],
+  ioActor: ActorRef,
+  calls: Set[WdlTaskCall],
   jesConfiguration: JesConfiguration,
-  serviceRegistryActor: ActorRef
+  serviceRegistryActor: ActorRef,
+  restarting: Boolean
 ) extends StandardInitializationActorParams {
   override val configurationDescriptor: BackendConfigurationDescriptor = jesConfiguration.configurationDescriptor
 }
 
 class JesInitializationActor(jesParams: JesInitializationActorParams)
-  extends StandardInitializationActor(jesParams) {
+  extends StandardInitializationActor(jesParams) with AsyncIo with GcsBatchCommandBuilder {
 
+  override lazy val ioActor = jesParams.ioActor
   private val jesConfiguration = jesParams.jesConfiguration
+  private val workflowOptions = workflowDescriptor.workflowOptions
+  implicit private val system = context.system
+
+  context.become(ioReceive orElse receive)
 
   override lazy val runtimeAttributesBuilder: StandardValidatedRuntimeAttributesBuilder =
     JesRuntimeAttributes.runtimeAttributesBuilder(jesConfiguration)
 
-  private[jes] lazy val refreshTokenAuth: Option[JesAuthInformation] = {
+  // From the gcs auth and the workflow options, optionally builds a GcsLocalizing that contains
+  // the information (client Id/Secrets + refresh token) that will be uploaded to Gcs before the workflow start
+  private[jes] lazy val refreshTokenAuth: Option[JesAuthObject] = {
     for {
       clientSecrets <- List(jesConfiguration.jesAttributes.auths.gcs) collectFirst { case s: ClientSecrets => s }
       token <- workflowDescriptor.workflowOptions.get(GoogleAuthMode.RefreshTokenOptionKey).toOption
     } yield GcsLocalizing(clientSecrets, token)
   }
 
-  private lazy val genomics = jesConfiguration.genomicsFactory.withOptions(workflowDescriptor.workflowOptions)
-  // FIXME: workflow paths indirectly re create part of those credentials via the GcsPathBuilder
-  // This is unnecessary duplication of credentials. They are needed here so they can be added to the initialization data
-  // and used to retrieve docker hashes
-  private lazy val gcsCredentials = jesConfiguration.jesAuths.gcs.credentialBundle(workflowDescriptor.workflowOptions)
+  // Credentials object for the GCS API
+  private lazy val gcsCredentials: Future[Credentials] =
+    jesConfiguration.jesAttributes.auths.gcs.credential(workflowOptions)
 
-  override lazy val workflowPaths: JesWorkflowPaths =
-    new JesWorkflowPaths(workflowDescriptor, jesConfiguration)(context.system)
+  // Credentials object for the Genomics API
+  private lazy val genomicsCredentials: Future[Credentials] =
+    jesConfiguration.jesAttributes.auths.genomics.credential(workflowOptions)
 
-  override lazy val initializationData: JesBackendInitializationData =
-        JesBackendInitializationData(workflowPaths, runtimeAttributesBuilder, jesConfiguration, gcsCredentials, genomics)
+  // Genomics object to access the Genomics API
+  private lazy val genomics: Future[Genomics] = {
+    genomicsCredentials map jesConfiguration.genomicsFactory.fromCredentials
+  }
 
-  override def beforeAll(): Future[Option[BackendInitializationData]] = Future.fromTry(Try {
-    if (jesConfiguration.needAuthFileUpload) writeAuthenticationFile(workflowPaths)
-    publishWorkflowRoot(workflowPaths.workflowRoot.pathAsString)
-    Option(initializationData)
-  })
+  override lazy val workflowPaths: Future[JesWorkflowPaths] = for {
+    gcsCred <- gcsCredentials
+    genomicsCred <- genomicsCredentials
+  } yield new JesWorkflowPaths(workflowDescriptor, gcsCred, genomicsCred, jesConfiguration)
 
-  private def writeAuthenticationFile(workflowPath: JesWorkflowPaths): Unit = {
-    generateAuthJson(jesConfiguration.dockerCredentials, refreshTokenAuth) foreach { content =>
-      val path = workflowPath.gcsAuthFilePath
-      workflowLogger.info(s"Creating authentication file for workflow ${workflowDescriptor.id} at \n $path")
-      try {
-        path.writeAsJson(content)
-      } catch {
-        case exception: Exception => throw new IOException("Failed to upload authentication file", exception)
+  override lazy val initializationData: Future[JesBackendInitializationData] = for {
+    jesWorkflowPaths <- workflowPaths
+    gcsCreds <- gcsCredentials
+    genomicsFactory <- genomics
+  } yield JesBackendInitializationData(jesWorkflowPaths, runtimeAttributesBuilder, jesConfiguration, gcsCreds, genomicsFactory)
+
+  override def beforeAll(): Future[Option[BackendInitializationData]] = {
+    def fileUpload(paths: JesWorkflowPaths): Future[Unit] = {
+      writeAuthenticationFile(paths, jesConfiguration.jesAttributes.restrictMetadataAccess, jesConfiguration.dockerCredentials) recoverWith {
+        case CromwellFatalException(e: StorageException) if e.getCode == StatusCodes.PreconditionFailed.intValue && jesParams.restarting =>
+          workflowLogger.debug(s"Authentication file already exists but this is a restart, proceeding.")
+          Future.successful(())
+        case CromwellFatalException(e: StorageException) if e.getCode == StatusCodes.PreconditionFailed.intValue =>
+            Future.failed(new IOException(s"Failed to upload authentication file:" +
+              " there was already a file at the same location and this workflow was not being restarted."))
+        case failure => Future.failed(new IOException(s"Failed to upload authentication file", failure))
       }
     }
+
+    for {
+      paths <- workflowPaths
+      _ = publishWorkflowRoot(paths.workflowRoot.pathAsString)
+      _ <- if (jesConfiguration.needAuthFileUpload) fileUpload(paths) else Future.successful(())
+      data <- initializationData
+    } yield Option(data)
   }
 
-  def generateAuthJson(authInformation: Option[JesAuthInformation]*): Option[String] = {
-    authInformation.flatten map { _.toMap } match {
-      case Nil => None
-      case jsons =>
-        val authsValues = jsons.reduce(_ ++ _) mapValues JsObject.apply
-        Option(JsObject("auths" -> JsObject(authsValues)).prettyPrint)
+  private def writeAuthenticationFile(workflowPath: JesWorkflowPaths,
+                                      restrictMetadataAccess: Boolean,
+                                      dockerCredentials: Option[JesDockerCredentials]): Future[Unit] = {
+    val authObjects = List(dockerCredentials, refreshTokenAuth).flatten
+    generateAuthJson(authObjects, restrictMetadataAccess) map { content =>
+      val path = workflowPath.gcsAuthFilePath
+      workflowLogger.info(s"Creating authentication file for workflow ${workflowDescriptor.id} at \n $path")
+      val openOptions = Seq(
+        CloudStorageOptions.withMimeType("application/json"),
+        // Will fail if the file already exists - 
+        // In case of a restart this avoids rewriting the file as the JES job might be trying to localize it at the same time 
+        StandardOpenOption.CREATE_NEW
+      )
+      writeAsync(path, content, openOptions)
+    } getOrElse Future.successful(())
+  }
+
+  def generateAuthJson(authObjects: List[JesAuthObject], restrictMetadataAccess: Boolean): Option[String] = {
+    def generateAuthObject(): Map[String, JsObject] = {
+      if (authObjects.nonEmpty) {
+        val authObjectMaps = authObjects map { _.toMap }
+        Map("auths" -> JsObject(authObjectMaps.reduce(_ ++ _) map { case (k, v) => k -> JsObject.apply(v) }))
+      } else Map.empty
     }
+
+    val authMap = generateAuthObject()
+    val jsonMap = if (restrictMetadataAccess) authMap ++ Map("restrictMetadataAccess" -> JsTrue) else authMap
+
+    if (jsonMap.nonEmpty) Option(JsObject(jsonMap).prettyPrint)
+    else None
   }
 }
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobCachingActorHelper.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobCachingActorHelper.scala
index ff6d8dc59..f913047be 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobCachingActorHelper.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobCachingActorHelper.scala
@@ -3,8 +3,12 @@ package cromwell.backend.impl.jes
 import akka.actor.Actor
 import cromwell.backend.impl.jes.io.{JesAttachedDisk, JesWorkingDisk}
 import cromwell.backend.standard.StandardCachingActorHelper
+import cromwell.core.labels.Labels
 import cromwell.core.logging.JobLogging
 import cromwell.core.path.Path
+import cromwell.services.metadata.CallMetadataKeys
+
+import scala.language.postfixOps
 
 trait JesJobCachingActorHelper extends StandardCachingActorHelper {
   this: Actor with JobLogging =>
@@ -23,9 +27,8 @@ trait JesJobCachingActorHelper extends StandardCachingActorHelper {
 
   lazy val jesCallPaths: JesJobPaths = jobPaths.asInstanceOf[JesJobPaths]
 
-  lazy val runtimeAttributes = JesRuntimeAttributes(validatedRuntimeAttributes)
+  lazy val runtimeAttributes = JesRuntimeAttributes(validatedRuntimeAttributes, jesConfiguration.runtimeConfig)
 
-  lazy val retryable: Boolean = jobDescriptor.key.attempt <= runtimeAttributes.preemptible
   lazy val workingDisk: JesAttachedDisk = runtimeAttributes.disks.find(_.name == JesWorkingDisk.Name).get
 
   lazy val callRootPath: Path = jesCallPaths.callExecutionRoot
@@ -37,11 +40,11 @@ trait JesJobCachingActorHelper extends StandardCachingActorHelper {
   lazy val defaultMonitoringOutputPath: Path = callRootPath.resolve(JesMonitoringLogFile)
 
   lazy val maxPreemption: Int = runtimeAttributes.preemptible
-  lazy val preemptible: Boolean = jobDescriptor.key.attempt <= maxPreemption
+  def preemptible: Boolean
 
   lazy val jesAttributes: JesAttributes = jesConfiguration.jesAttributes
   lazy val monitoringScript: Option[JesInput] = {
-    jesCallPaths.monitoringPath map { path =>
+    jesCallPaths.workflowPaths.monitoringPath map { path =>
       JesFileInput(s"$MonitoringParamName-in", path.pathAsString,
         JesWorkingDisk.MountPoint.resolve(JesMonitoringScript), workingDisk)
     }
@@ -50,12 +53,41 @@ trait JesJobCachingActorHelper extends StandardCachingActorHelper {
     defaultMonitoringOutputPath.pathAsString, JesMonitoringLogFile, workingDisk)
   }
 
+  lazy val defaultLabels: Labels = {
+    val workflow = jobDescriptor.workflowDescriptor
+    val call = jobDescriptor.call
+    val subWorkflow = workflow.workflow
+    val subWorkflowLabels = if (!subWorkflow.equals(workflow.rootWorkflow))
+      Labels("cromwell-sub-workflow-name" -> subWorkflow.unqualifiedName)
+    else
+      Labels.empty
+
+    val alias = call.unqualifiedName
+    val aliasLabels = if (!alias.equals(call.task.name))
+      Labels("wdl-call-alias" -> alias)
+    else
+      Labels.empty
+
+    Labels(
+      "cromwell-workflow-id" -> s"cromwell-${workflow.rootWorkflowId}",
+      "wdl-task-name" -> call.task.name
+    ) ++ subWorkflowLabels ++ aliasLabels
+  }
+
+  lazy val originalLabels: Labels = defaultLabels ++ workflowDescriptor.customLabels
+
+  lazy val backendLabels: Labels = GoogleLabels.toLabels(originalLabels.asTuple :_*)
+
+  lazy val originalLabelEvents: Map[String, String] = originalLabels.value map { l => s"${CallMetadataKeys.Labels}:${l.key}" -> l.value } toMap
+
+  lazy val backendLabelEvents: Map[String, String] = backendLabels.value map { l => s"${CallMetadataKeys.BackendLabels}:${l.key}" -> l.value } toMap
+
   override protected def nonStandardMetadata: Map[String, Any] = {
     Map(
       JesMetadataKeys.GoogleProject -> jesAttributes.project,
       JesMetadataKeys.ExecutionBucket -> jesAttributes.executionBucket,
       JesMetadataKeys.EndpointUrl -> jesAttributes.endpointUrl,
       "preemptible" -> preemptible
-    )
+    ) ++ backendLabelEvents ++ originalLabelEvents
   }
 }
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobPaths.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobPaths.scala
index bd4090619..e3c349ec0 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobPaths.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobPaths.scala
@@ -1,24 +1,16 @@
 package cromwell.backend.impl.jes
 
-import akka.actor.ActorSystem
+import cromwell.backend.BackendJobDescriptorKey
 import cromwell.backend.io.JobPaths
-import cromwell.backend.{BackendJobDescriptorKey, BackendWorkflowDescriptor}
 import cromwell.core.path.Path
 import cromwell.services.metadata.CallMetadataKeys
 
 object JesJobPaths {
-  def apply(jobKey: BackendJobDescriptorKey, workflowDescriptor: BackendWorkflowDescriptor,
-            jesConfiguration: JesConfiguration)(implicit actorSystem: ActorSystem): JesJobPaths = {
-    new JesJobPaths(jobKey, workflowDescriptor, jesConfiguration)
-  }
-
   val JesLogPathKey = "jesLog"
   val GcsExecPathKey = "gcsExec"
 }
 
-class JesJobPaths(val jobKey: BackendJobDescriptorKey, workflowDescriptor: BackendWorkflowDescriptor,
-                  jesConfiguration: JesConfiguration)(implicit actorSystem: ActorSystem) extends
-  JesWorkflowPaths(workflowDescriptor, jesConfiguration)(actorSystem) with JobPaths {
+final case class JesJobPaths(override val workflowPaths: JesWorkflowPaths, jobKey: BackendJobDescriptorKey) extends JobPaths {
 
   val jesLogBasename = {
     val index = jobKey.index.map(s => s"-$s").getOrElse("")
@@ -46,11 +38,10 @@ class JesJobPaths(val jobKey: BackendJobDescriptorKey, workflowDescriptor: Backe
   override lazy val customMetadataPaths = Map(
     CallMetadataKeys.BackendLogsPrefix + ":log" -> jesLogPath
   ) ++ (
-    monitoringPath map { p => Map(JesMetadataKeys.MonitoringLog -> p) } getOrElse Map.empty  
+    workflowPaths.monitoringPath map { p => Map(JesMetadataKeys.MonitoringLog -> p) } getOrElse Map.empty  
   )
 
   override lazy val customDetritusPaths: Map[String, Path] = Map(
-    JesJobPaths.GcsExecPathKey -> script,
     JesJobPaths.JesLogPathKey -> jesLogPath
   )
 
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesPipelineInfo.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesPipelineInfo.scala
index 990ee7c98..ba052cd31 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesPipelineInfo.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesPipelineInfo.scala
@@ -26,20 +26,18 @@ trait JesPipelineInfoBuilder {
       .set(Run.NoAddressFieldName, runtimeAttributes.noAddress)
   }
 
-  def build(commandLine: String, runtimeAttributes: JesRuntimeAttributes): JesPipelineInfo
+  def build(commandLine: String, runtimeAttributes: JesRuntimeAttributes, docker: String): JesPipelineInfo
 }
 
 object NonPreemptibleJesPipelineInfoBuilder extends JesPipelineInfoBuilder {
-  def build(commandLine: String, runtimeAttributes: JesRuntimeAttributes): JesPipelineInfo = {
-    val dockerImage = runtimeAttributes.dockerImage
+  def build(commandLine: String, runtimeAttributes: JesRuntimeAttributes, dockerImage: String): JesPipelineInfo = {
     val resources = buildResources(runtimeAttributes).setPreemptible(false)
     new NonPreemptibleJesPipelineInfoBuilder(resources, buildDockerExecutor(commandLine, dockerImage))
   }
 }
 
 object PreemptibleJesPipelineInfoBuilder extends JesPipelineInfoBuilder {
-  def build(commandLine: String, runtimeAttributes: JesRuntimeAttributes): JesPipelineInfo = {
-    val dockerImage = runtimeAttributes.dockerImage
+  def build(commandLine: String, runtimeAttributes: JesRuntimeAttributes, dockerImage: String): JesPipelineInfo = {
     val resources = buildResources(runtimeAttributes).setPreemptible(true)
     new PreemptibleJesPipelineInfoBuilder(resources, buildDockerExecutor(commandLine, dockerImage))
   }
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesRuntimeAttributes.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesRuntimeAttributes.scala
index f64716d9b..05b9ca358 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesRuntimeAttributes.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesRuntimeAttributes.scala
@@ -1,16 +1,17 @@
 package cromwell.backend.impl.jes
 
-import cats.data.NonEmptyList
+import cats.syntax.validated._
 import cats.data.Validated._
 import cats.syntax.cartesian._
-import cats.syntax.validated._
+import com.typesafe.config.Config
 import cromwell.backend.MemorySize
 import cromwell.backend.impl.jes.io.{JesAttachedDisk, JesWorkingDisk}
 import cromwell.backend.standard.StandardValidatedRuntimeAttributesBuilder
 import cromwell.backend.validation.{BooleanRuntimeAttributesValidation, _}
 import lenthall.validation.ErrorOr._
-import wdl4s.types._
-import wdl4s.values._
+import wdl4s.wdl.types._
+import wdl4s.wdl.values._
+
 
 case class JesRuntimeAttributes(cpu: Int,
                                 zones: Vector[String],
@@ -24,72 +25,82 @@ case class JesRuntimeAttributes(cpu: Int,
                                 noAddress: Boolean)
 
 object JesRuntimeAttributes {
-  private val MemoryDefaultValue = "2 GB"
 
   val ZonesKey = "zones"
+  private val ZonesDefaultValue = WdlString("us-central1-b")
 
   val PreemptibleKey = "preemptible"
-  private val PreemptibleDefaultValue = 0
+  private val preemptibleValidationInstance = new IntRuntimeAttributesValidation(PreemptibleKey)
+  private val PreemptibleDefaultValue = WdlInteger(0)
 
   val BootDiskSizeKey = "bootDiskSizeGb"
-  private val BootDiskSizeDefaultValue = 10
+  private val bootDiskValidationInstance = new IntRuntimeAttributesValidation(BootDiskSizeKey)
+  private val BootDiskDefaultValue = WdlInteger(10)
 
   val NoAddressKey = "noAddress"
-  private val NoAddressDefaultValue = false
+  private val noAddressValidationInstance = new BooleanRuntimeAttributesValidation(NoAddressKey)
+  private val NoAddressDefaultValue = WdlBoolean(false)
 
   val DisksKey = "disks"
-  private val DisksDefaultValue = s"${JesWorkingDisk.Name} 10 SSD"
+  private val DisksDefaultValue = WdlString(s"${JesWorkingDisk.Name} 10 SSD")
+
+  private val MemoryDefaultValue = "2 GB"
+
+  private def cpuValidation(runtimeConfig: Option[Config]): RuntimeAttributesValidation[Int] = CpuValidation.instance
+    .withDefault(CpuValidation.configDefaultWdlValue(runtimeConfig) getOrElse CpuValidation.default)
 
-  private val cpuValidation: RuntimeAttributesValidation[Int] = CpuValidation.default
+  private def failOnStderrValidation(runtimeConfig: Option[Config]) = FailOnStderrValidation.default(runtimeConfig)
 
-  private val disksValidation: RuntimeAttributesValidation[Seq[JesAttachedDisk]] =
-    DisksValidation.withDefault(WdlString(DisksDefaultValue))
+  private def continueOnReturnCodeValidation(runtimeConfig: Option[Config]) = ContinueOnReturnCodeValidation.default(runtimeConfig)
 
-  private def zonesValidation(defaultZones: NonEmptyList[String]): RuntimeAttributesValidation[Vector[String]] =
-    ZonesValidation.withDefault(WdlString(defaultZones.toList.mkString(" ")))
+  private def disksValidation(runtimeConfig: Option[Config]): RuntimeAttributesValidation[Seq[JesAttachedDisk]] = DisksValidation
+    .withDefault(DisksValidation.configDefaultWdlValue(runtimeConfig) getOrElse DisksDefaultValue)
 
-  private val preemptibleValidation: RuntimeAttributesValidation[Int] =
-    new IntRuntimeAttributesValidation(JesRuntimeAttributes.PreemptibleKey)
-      .withDefault(WdlInteger(PreemptibleDefaultValue))
+  private def zonesValidation(runtimeConfig: Option[Config]): RuntimeAttributesValidation[Vector[String]] = ZonesValidation
+    .withDefault(ZonesValidation.configDefaultWdlValue(runtimeConfig) getOrElse ZonesDefaultValue)
 
-  private val memoryValidation: RuntimeAttributesValidation[MemorySize] =
-    MemoryValidation.withDefaultMemory(MemorySize.parse(MemoryDefaultValue).get)
+  private def preemptibleValidation(runtimeConfig: Option[Config]): RuntimeAttributesValidation[Int] = preemptibleValidationInstance
+    .withDefault(preemptibleValidationInstance.configDefaultWdlValue(runtimeConfig) getOrElse PreemptibleDefaultValue)
 
-  private val bootDiskSizeValidation: RuntimeAttributesValidation[Int] =
-    new IntRuntimeAttributesValidation(JesRuntimeAttributes.BootDiskSizeKey)
-      .withDefault(WdlInteger(BootDiskSizeDefaultValue))
+  private def memoryValidation(runtimeConfig: Option[Config]): RuntimeAttributesValidation[MemorySize] = {
+    MemoryValidation.withDefaultMemory(
+      RuntimeAttributesKeys.MemoryKey,
+      MemoryValidation.configDefaultString(RuntimeAttributesKeys.MemoryKey, runtimeConfig) getOrElse MemoryDefaultValue)
+  }
 
-  private val noAddressValidation: RuntimeAttributesValidation[Boolean] =
-    new BooleanRuntimeAttributesValidation(JesRuntimeAttributes.NoAddressKey)
-      .withDefault(WdlBoolean(NoAddressDefaultValue))
+  private def bootDiskSizeValidation(runtimeConfig: Option[Config]): RuntimeAttributesValidation[Int] = bootDiskValidationInstance
+    .withDefault(bootDiskValidationInstance.configDefaultWdlValue(runtimeConfig) getOrElse BootDiskDefaultValue)
+
+  private def noAddressValidation(runtimeConfig: Option[Config]): RuntimeAttributesValidation[Boolean] = noAddressValidationInstance
+    .withDefault(noAddressValidationInstance.configDefaultWdlValue(runtimeConfig) getOrElse NoAddressDefaultValue)
 
   private val dockerValidation: RuntimeAttributesValidation[String] = DockerValidation.instance
 
-  def runtimeAttributesBuilder(jesConfiguration: JesConfiguration): StandardValidatedRuntimeAttributesBuilder =
-    StandardValidatedRuntimeAttributesBuilder.default.withValidation(
-      cpuValidation,
-      disksValidation,
-      zonesValidation(jesConfiguration.defaultZones),
-      preemptibleValidation,
-      memoryValidation,
-      bootDiskSizeValidation,
-      noAddressValidation,
+  def runtimeAttributesBuilder(jesConfiguration: JesConfiguration): StandardValidatedRuntimeAttributesBuilder = {
+    val runtimeConfig = jesConfiguration.runtimeConfig
+    StandardValidatedRuntimeAttributesBuilder.default(runtimeConfig).withValidation(
+      cpuValidation(runtimeConfig),
+      disksValidation(runtimeConfig),
+      zonesValidation(runtimeConfig),
+      preemptibleValidation(runtimeConfig),
+      memoryValidation(runtimeConfig),
+      bootDiskSizeValidation(runtimeConfig),
+      noAddressValidation(runtimeConfig),
       dockerValidation
     )
+  }
 
-  def apply(validatedRuntimeAttributes: ValidatedRuntimeAttributes): JesRuntimeAttributes = {
-    val cpu: Int = RuntimeAttributesValidation.extract(cpuValidation, validatedRuntimeAttributes)
+  def apply(validatedRuntimeAttributes: ValidatedRuntimeAttributes, runtimeAttrsConfig: Option[Config]): JesRuntimeAttributes = {
+    val cpu: Int = RuntimeAttributesValidation.extract(cpuValidation(runtimeAttrsConfig), validatedRuntimeAttributes)
     val zones: Vector[String] = RuntimeAttributesValidation.extract(ZonesValidation, validatedRuntimeAttributes)
-    val preemptible: Int = RuntimeAttributesValidation.extract(preemptibleValidation, validatedRuntimeAttributes)
-    val bootDiskSize: Int = RuntimeAttributesValidation.extract(bootDiskSizeValidation, validatedRuntimeAttributes)
-    val memory: MemorySize = RuntimeAttributesValidation.extract(memoryValidation, validatedRuntimeAttributes)
-    val disks: Seq[JesAttachedDisk] = RuntimeAttributesValidation.extract(disksValidation, validatedRuntimeAttributes)
+    val preemptible: Int = RuntimeAttributesValidation.extract(preemptibleValidation(runtimeAttrsConfig), validatedRuntimeAttributes)
+    val bootDiskSize: Int = RuntimeAttributesValidation.extract(bootDiskSizeValidation(runtimeAttrsConfig), validatedRuntimeAttributes)
+    val memory: MemorySize = RuntimeAttributesValidation.extract(memoryValidation(runtimeAttrsConfig), validatedRuntimeAttributes)
+    val disks: Seq[JesAttachedDisk] = RuntimeAttributesValidation.extract(disksValidation(runtimeAttrsConfig), validatedRuntimeAttributes)
     val docker: String = RuntimeAttributesValidation.extract(dockerValidation, validatedRuntimeAttributes)
-    val failOnStderr: Boolean =
-      RuntimeAttributesValidation.extract(FailOnStderrValidation.default, validatedRuntimeAttributes)
-    val continueOnReturnCode: ContinueOnReturnCode =
-      RuntimeAttributesValidation.extract(ContinueOnReturnCodeValidation.default, validatedRuntimeAttributes)
-    val noAddress: Boolean = RuntimeAttributesValidation.extract(noAddressValidation, validatedRuntimeAttributes)
+    val failOnStderr: Boolean = RuntimeAttributesValidation.extract(failOnStderrValidation(runtimeAttrsConfig), validatedRuntimeAttributes)
+    val continueOnReturnCode: ContinueOnReturnCode = RuntimeAttributesValidation.extract(continueOnReturnCodeValidation(runtimeAttrsConfig), validatedRuntimeAttributes)
+    val noAddress: Boolean = RuntimeAttributesValidation.extract(noAddressValidation(runtimeAttrsConfig), validatedRuntimeAttributes)
 
     new JesRuntimeAttributes(
       cpu,
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesWorkflowPaths.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesWorkflowPaths.scala
index 8b06f117b..3d9d669ec 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesWorkflowPaths.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesWorkflowPaths.scala
@@ -1,49 +1,52 @@
 package cromwell.backend.impl.jes
 
 import akka.actor.ActorSystem
+import com.google.auth.Credentials
 import com.typesafe.config.Config
 import cromwell.backend.impl.jes.JesAsyncBackendJobExecutionActor.WorkflowOptionKeys
 import cromwell.backend.io.WorkflowPaths
 import cromwell.backend.{BackendJobDescriptorKey, BackendWorkflowDescriptor}
 import cromwell.core.WorkflowOptions
 import cromwell.core.path.{Path, PathBuilder}
-import cromwell.filesystems.gcs.{GcsPathBuilderFactory, RetryableGcsPathBuilder}
+import cromwell.filesystems.gcs.{GcsPathBuilder, GcsPathBuilderFactory}
 
 import scala.language.postfixOps
-import scala.util.Try
 
 object JesWorkflowPaths {
   private val GcsRootOptionKey = "jes_gcs_root"
   private val AuthFilePathOptionKey = "auth_bucket"
-
-  def apply(workflowDescriptor: BackendWorkflowDescriptor,
-            jesConfiguration: JesConfiguration)(implicit actorSystem: ActorSystem): JesWorkflowPaths = {
-    new JesWorkflowPaths(workflowDescriptor, jesConfiguration)
-  }
 }
 
-class JesWorkflowPaths(val workflowDescriptor: BackendWorkflowDescriptor,
-                       jesConfiguration: JesConfiguration)(implicit actorSystem: ActorSystem) extends WorkflowPaths {
+case class JesWorkflowPaths(workflowDescriptor: BackendWorkflowDescriptor,
+                            gcsCredentials: Credentials,
+                            genomicsCredentials: Credentials,
+                            jesConfiguration: JesConfiguration)(implicit actorSystem: ActorSystem) extends WorkflowPaths {
 
   override lazy val executionRootString: String =
     workflowDescriptor.workflowOptions.getOrElse(JesWorkflowPaths.GcsRootOptionKey, jesConfiguration.root)
+
   private val workflowOptions: WorkflowOptions = workflowDescriptor.workflowOptions
-  val gcsPathBuilder: RetryableGcsPathBuilder = jesConfiguration.gcsPathBuilderFactory.withOptions(workflowOptions)
 
-  def getHash(gcsUrl: Path): Try[String] = gcsPathBuilder.getHash(gcsUrl)
+  private val gcsPathBuilder: GcsPathBuilder = jesConfiguration.gcsPathBuilderFactory.fromCredentials(workflowOptions, gcsCredentials)
 
   val gcsAuthFilePath: Path = {
+    // The default auth file bucket is always at the root of the root workflow
+    val defaultBucket = executionRoot.resolve(workflowDescriptor.rootWorkflow.unqualifiedName).resolve(workflowDescriptor.rootWorkflowId.toString)
+    val bucket = workflowDescriptor.workflowOptions.get(JesWorkflowPaths.AuthFilePathOptionKey) getOrElse defaultBucket.pathAsString
+
     /*
      * This is an "exception". The filesystem used here is built from genomicsAuth
      * unlike everywhere else where the filesystem used is built from gcsFileSystemAuth
      */
-    val genomicsCredentials = jesConfiguration.jesAuths.genomics
+    val pathBuilderWithGenomicsAuth = GcsPathBuilder.fromCredentials(
+      genomicsCredentials,
+      jesConfiguration.googleConfig.applicationName,
+      None,
+      GcsPathBuilderFactory.DefaultCloudStorageConfiguration,
+      workflowOptions
+    )
 
-    // The default auth file bucket is always at the root of the root workflow
-    val defaultBucket = executionRoot.resolve(workflowDescriptor.rootWorkflow.unqualifiedName).resolve(workflowDescriptor.rootWorkflowId.toString)
-
-    val bucket = workflowDescriptor.workflowOptions.get(JesWorkflowPaths.AuthFilePathOptionKey) getOrElse defaultBucket.pathAsString
-    val authBucket = GcsPathBuilderFactory(genomicsCredentials).withOptions(workflowOptions).build(bucket) recover {
+    val authBucket = pathBuilderWithGenomicsAuth.build(bucket) recover {
       case ex => throw new Exception(s"Invalid gcs auth_bucket path $bucket", ex)
     } get
 
@@ -55,10 +58,12 @@ class JesWorkflowPaths(val workflowDescriptor: BackendWorkflowDescriptor,
     getPath(path).get
   }
 
-  override def toJobPaths(jobKey: BackendJobDescriptorKey,
-                          jobWorkflowDescriptor: BackendWorkflowDescriptor): JesJobPaths = {
-    JesJobPaths(jobKey, jobWorkflowDescriptor, jesConfiguration)
+  override def toJobPaths(workflowPaths: WorkflowPaths, jobKey: BackendJobDescriptorKey): JesJobPaths = {
+    new JesJobPaths(workflowPaths.asInstanceOf[JesWorkflowPaths], jobKey)
   }
+
+  override protected def withDescriptor(workflowDescriptor: BackendWorkflowDescriptor): WorkflowPaths = this.copy(workflowDescriptor = workflowDescriptor)
+
   override def config: Config = jesConfiguration.configurationDescriptor.backendConfig
   override def pathBuilders: List[PathBuilder] = List(gcsPathBuilder)
 }
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/PreviousRetryReasons.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/PreviousRetryReasons.scala
new file mode 100644
index 000000000..1d3eba07e
--- /dev/null
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/PreviousRetryReasons.scala
@@ -0,0 +1,45 @@
+package cromwell.backend.impl.jes
+
+import cromwell.services.keyvalue.KeyValueServiceActor._
+import lenthall.validation.ErrorOr.ErrorOr
+import cats.syntax.validated._
+import cats.syntax.cartesian._
+
+import scala.util.{Failure, Success, Try}
+import JesBackendLifecycleActorFactory.preemptionCountKey
+import JesBackendLifecycleActorFactory.unexpectedRetryCountKey
+
+case class PreviousRetryReasons(preempted: Int, unexpectedRetry: Int)
+
+object PreviousRetryReasons {
+  def tryApply(prefetchedKvEntries: Map[String, KvResponse], attemptNumber: Int): ErrorOr[PreviousRetryReasons] = {
+    val validatedPreemptionCount = validatedKvResponse(prefetchedKvEntries.get(preemptionCountKey), preemptionCountKey)
+    val validatedUnexpectedRetryCount = validatedKvResponse(prefetchedKvEntries.get(unexpectedRetryCountKey), unexpectedRetryCountKey)
+
+    validatedPreemptionCount |@| validatedUnexpectedRetryCount map { case (p, uf) => PreviousRetryReasons(p, uf) }
+  }
+
+  def apply(knownPreemptedCount: Int, knownUnexpectedRetryCount: Int, attempt: Int): PreviousRetryReasons = {
+    // If we have anything unaccounted for, we can top up the unexpected retry count.
+    // NB: 'attempt' is 1-indexed, so, magic number:
+    // NB2: for sanity's sake, I won't let this unaccounted for drop below 0, just in case...
+    val unaccountedFor = Math.max(attempt - 1 - knownPreemptedCount - knownUnexpectedRetryCount, 0)
+    PreviousRetryReasons(knownPreemptedCount, knownUnexpectedRetryCount + unaccountedFor)
+  }
+
+  private def validatedKvResponse(r: Option[KvResponse], fromKey: String): ErrorOr[Int] = r match {
+    case Some(KvPair(_, v)) => validatedIntOption(v, fromKey)
+    case Some(_: KvKeyLookupFailed) => 0.validNel
+    case Some(KvFailure(_, failure)) => s"Failed to get key $fromKey: ${failure.getMessage}".invalidNel
+    case Some(_: KvPutSuccess) => s"Programmer Error: Got a KvPutSuccess from a Get request...".invalidNel
+    case None => s"Programmer Error: Engine made no effort to prefetch $fromKey".invalidNel
+  }
+
+  private def validatedIntOption(s: Option[String], fromKey: String): ErrorOr[Int] = validatedInt(s.getOrElse(""), fromKey)
+  private def validatedInt(s: String, fromKey: String): ErrorOr[Int] = {
+    Try(s.toInt) match {
+      case Success(i) => i.validNel
+      case Failure(_) => s"Unexpected value found in the KV store: $fromKey='$s'".invalidNel
+    }
+  }
+}
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala
index acafef4f5..d3b014316 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/Run.scala
@@ -21,18 +21,20 @@ object Run {
 
   def makeRunPipelineRequest(jobDescriptor: BackendJobDescriptor,
                              runtimeAttributes: JesRuntimeAttributes,
+                             dockerImage: String,
                              callRootPath: String,
                              commandLine: String,
                              logFileName: String,
                              jesParameters: Seq[JesParameter],
                              projectId: String,
                              computeServiceAccount: String,
+                             labels: Labels,
                              preemptible: Boolean,
                              genomicsInterface: Genomics): RunPipelineRequest = {
 
     lazy val workflow = jobDescriptor.workflowDescriptor
     val pipelineInfoBuilder = if (preemptible) PreemptibleJesPipelineInfoBuilder else NonPreemptibleJesPipelineInfoBuilder
-    val pipelineInfo = pipelineInfoBuilder.build(commandLine, runtimeAttributes)
+    val pipelineInfo = pipelineInfoBuilder.build(commandLine, runtimeAttributes, dockerImage)
 
     val pipeline = new Pipeline()
       .setProjectId(projectId)
@@ -44,41 +46,20 @@ object Run {
 
     // disks cannot have mount points at runtime, so set them null
     val runtimePipelineResources = {
-      val resources = pipelineInfoBuilder.build(commandLine, runtimeAttributes).resources
+      val resources = pipelineInfoBuilder.build(commandLine, runtimeAttributes, dockerImage).resources
       val disksWithoutMountPoint = resources.getDisks.asScala map {
         _.setMountPoint(null)
       }
       resources.setDisks(disksWithoutMountPoint.asJava)
     }
 
-    lazy val labels: Labels = {
-
-      val subWorkflow = workflow.workflow
-      val subWorkflowLabels = if (!subWorkflow.equals(workflow.rootWorkflow))
-        Labels("cromwell-sub-workflow-name" -> subWorkflow.unqualifiedName)
-      else
-        Labels.empty
-
-      val alias = jobDescriptor.call.unqualifiedName
-      val aliasLabels = if (!alias.equals(jobDescriptor.call.task.name))
-        Labels("wdl-call-alias" -> alias)
-      else
-        Labels.empty
-
-      Labels(
-        "cromwell-workflow-id" -> s"cromwell-${workflow.rootWorkflowId}",
-        "cromwell-workflow-name" -> workflow.rootWorkflow.unqualifiedName,
-        "wdl-task-name" -> jobDescriptor.call.task.name
-      ) ++ subWorkflowLabels ++ aliasLabels ++ jobDescriptor.workflowDescriptor.customLabels
-    }
-
     val svcAccount = new ServiceAccount().setEmail(computeServiceAccount).setScopes(GenomicsScopes)
     val rpargs = new RunPipelineArgs().setProjectId(projectId).setServiceAccount(svcAccount).setResources(runtimePipelineResources)
 
     rpargs.setInputs(jesParameters.collect({ case i: JesInput => i.name -> i.toGoogleRunParameter }).toMap.asJava)
     rpargs.setOutputs(jesParameters.collect({ case i: JesFileOutput => i.name -> i.toGoogleRunParameter }).toMap.asJava)
 
-    rpargs.setLabels(labels.asJesLabels)
+    rpargs.setLabels(labels.asJavaMap)
 
     val rpr = new RunPipelineRequest().setEphemeralPipeline(pipeline).setPipelineArgs(rpargs)
 
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/RunStatus.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/RunStatus.scala
index c2a260b0b..7cbe16038 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/RunStatus.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/RunStatus.scala
@@ -1,8 +1,8 @@
 package cromwell.backend.impl.jes
 
-import cromwell.backend.impl.jes.errors.JesError
 import cromwell.core.ExecutionEvent
-import cromwell.core.path.Path
+
+import scala.util.Try
 
 sealed trait RunStatus {
   import RunStatus._
@@ -25,22 +25,57 @@ object RunStatus {
     def instanceName: Option[String]
   }
 
-  case class Success(eventList: Seq[ExecutionEvent], machineType: Option[String], zone: Option[String], instanceName: Option[String]) extends TerminalRunStatus {
+  sealed trait UnsuccessfulRunStatus extends TerminalRunStatus {
+    val errorMessage: Option[String]
+    lazy val prettyPrintedError: String = errorMessage map { e => s" Message: $e" } getOrElse ""
+    val errorCode: Int
+
+    /**
+      * If one exists, the JES error code (not the google RPC) (extracted from the error message)
+      */
+    val jesCode: Option[Int]
+  }
+
+  case class Success(eventList: Seq[ExecutionEvent],
+                     machineType: Option[String],
+                     zone: Option[String],
+                     instanceName: Option[String]) extends TerminalRunStatus {
     override def toString = "Success"
   }
 
-  final case class Failed(errorCode: Int, errorMessage: Option[String], eventList: Seq[ExecutionEvent], machineType: Option[String], zone: Option[String], instanceName: Option[String])
-    extends TerminalRunStatus {
-    
-    private def unknownError(jobTag: String) = {
-      new RuntimeException(s"Task $jobTag failed: error code $errorCode. Message: $errorMessage")
+  object UnsuccessfulRunStatus {
+    def apply(errorCode: Int,
+              errorMessage: Option[String],
+              eventList: Seq[ExecutionEvent],
+              machineType: Option[String],
+              zone: Option[String],
+              instanceName: Option[String]): UnsuccessfulRunStatus = {
+      val jesCode: Option[Int] = errorMessage flatMap { em => Try(em.substring(0, em.indexOf(':')).toInt).toOption }
+      if (errorCode == JesAsyncBackendJobExecutionActor.GoogleAbortedRpc && jesCode.contains(JesAsyncBackendJobExecutionActor.JesPreemption)) {
+        Preempted(errorCode, jesCode, errorMessage, eventList, machineType, zone, instanceName)
+      } else {
+        Failed(errorCode, jesCode, errorMessage, eventList, machineType, zone, instanceName)
+      }
     }
-    
-    // Don't want to include errorMessage or code in the snappy status toString:
+  }
+
+  final case class Failed(errorCode: Int,
+                          jesCode: Option[Int],
+                          errorMessage: Option[String],
+                          eventList: Seq[ExecutionEvent],
+                          machineType: Option[String],
+                          zone: Option[String],
+                          instanceName: Option[String]) extends UnsuccessfulRunStatus {
     override def toString = "Failed"
-    
-    def toFailure(jobTag: String, stderrPath: Option[Path]): Exception = {
-      JesError.fromFailedStatus(this, jobTag, stderrPath) getOrElse unknownError(jobTag)
-    }
+  }
+
+  final case class Preempted(errorCode: Int,
+                             jesCode: Option[Int],
+                             errorMessage: Option[String],
+                             eventList: Seq[ExecutionEvent],
+                             machineType: Option[String],
+                             zone: Option[String],
+                             instanceName: Option[String]) extends UnsuccessfulRunStatus {
+    override def toString = "Preempted"
   }
 }
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/authentication/JesVMAuthentication.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/authentication/JesVMAuthentication.scala
index 9c92b380a..0f6a259a3 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/authentication/JesVMAuthentication.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/authentication/JesVMAuthentication.scala
@@ -5,10 +5,10 @@ import cromwell.filesystems.gcs.auth.ClientSecrets
 import spray.json.{JsString, JsValue}
 
 /**
- * Interface for Authentication information that can be included in the json file uploaded to GCS
+ * Interface for Authentication information that can be included as a json object in the file uploaded to GCS
  * upon workflow creation and used in the VM.
  */
-sealed trait JesAuthInformation {
+sealed trait JesAuthObject {
   def context: String
   def map: Map[String, JsValue]
 
@@ -18,7 +18,7 @@ sealed trait JesAuthInformation {
 /**
  * Authentication information for data (de)localization as the user.
  */
-case class GcsLocalizing(clientSecrets: ClientSecrets, token: String) extends JesAuthInformation {
+case class GcsLocalizing(clientSecrets: ClientSecrets, token: String) extends JesAuthObject {
   override val context = "boto"
   override val map = Map(
     "client_id" -> JsString(clientSecrets.clientId),
@@ -35,7 +35,7 @@ object JesDockerCredentials {
 /**
  * Authentication information to pull docker images as the user.
  */
-class JesDockerCredentials(account: String, token: String) extends DockerCredentials(account, token) with JesAuthInformation {
+class JesDockerCredentials(account: String, token: String) extends DockerCredentials(account, token) with JesAuthObject {
   override val context = "docker"
   override val map = Map(
     "account" -> JsString(account),
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/callcaching/JesBackendCacheHitCopyingActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/callcaching/JesBackendCacheHitCopyingActor.scala
new file mode 100644
index 000000000..d7ded8fc1
--- /dev/null
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/callcaching/JesBackendCacheHitCopyingActor.scala
@@ -0,0 +1,65 @@
+package cromwell.backend.impl.jes.callcaching
+
+import com.google.cloud.storage.contrib.nio.CloudStorageOptions
+import cromwell.backend.BackendInitializationData
+import cromwell.backend.impl.jes.JesBackendInitializationData
+import cromwell.backend.io.JobPaths
+import cromwell.backend.standard.callcaching.{StandardCacheHitCopyingActor, StandardCacheHitCopyingActorParams}
+import cromwell.core.CallOutputs
+import cromwell.core.io.{IoCommand, IoTouchCommand}
+import cromwell.core.path.Path
+import cromwell.core.simpleton.{WdlValueBuilder, WdlValueSimpleton}
+import cromwell.filesystems.gcs.batch.GcsBatchCommandBuilder
+import lenthall.util.TryUtil
+import wdl4s.wdl.values.WdlFile
+
+import scala.language.postfixOps
+import scala.util.Try
+
+class JesBackendCacheHitCopyingActor(standardParams: StandardCacheHitCopyingActorParams) extends StandardCacheHitCopyingActor(standardParams) with GcsBatchCommandBuilder {
+  
+  private val cachingStrategy = BackendInitializationData
+    .as[JesBackendInitializationData](standardParams.backendInitializationDataOption)
+    .jesConfiguration.jesAttributes.duplicationStrategy
+  
+  override def processSimpletons(wdlValueSimpletons: Seq[WdlValueSimpleton], sourceCallRootPath: Path) = cachingStrategy match {
+    case CopyCachedOutputs => super.processSimpletons(wdlValueSimpletons, sourceCallRootPath)
+    case UseOriginalCachedOutputs =>
+      val touchCommands: Seq[Try[IoTouchCommand]] = wdlValueSimpletons collect {
+        case WdlValueSimpleton(_, wdlFile: WdlFile) => getPath(wdlFile.value) map touchCommand
+      }
+      
+      TryUtil.sequence(touchCommands) map {
+        WdlValueBuilder.toJobOutputs(jobDescriptor.call.task.outputs, wdlValueSimpletons) -> _.toSet
+      }
+  }
+  
+  override def processDetritus(sourceJobDetritusFiles: Map[String, String]) = cachingStrategy match {
+    case CopyCachedOutputs => super.processDetritus(sourceJobDetritusFiles)
+    case UseOriginalCachedOutputs =>
+      // apply getPath on each detritus string file
+      val detritusAsPaths = detritusFileKeys(sourceJobDetritusFiles).toSeq map { key =>
+        key -> getPath(sourceJobDetritusFiles(key))
+      } toMap
+
+      // Don't forget to re-add the CallRootPathKey that has been filtered out by detritusFileKeys
+      TryUtil.sequenceMap(detritusAsPaths, "Failed to make paths out of job detritus") map { newDetritus =>
+        (newDetritus + (JobPaths.CallRootPathKey -> destinationCallRootPath)) -> newDetritus.values.map(touchCommand).toSet
+      }
+  }
+
+  override protected def additionalIoCommands(sourceCallRootPath: Path,
+                                              originalSimpletons: Seq[WdlValueSimpleton],
+                                              newOutputs: CallOutputs,
+                                              originalDetritus:  Map[String, String],
+                                              newDetritus: Map[String, Path]): List[Set[IoCommand[_]]] = {
+    val content =
+      s"""
+         |This directory does not contain any output files because this job matched an identical job that was previously run, thus it was a cache-hit.
+         |Cromwell is configured to not copy outputs during call caching. To change this, edit the filesystems.gcs.caching.duplication-strategy field in your backend configuration.
+         |The original outputs can be found at this location: ${sourceCallRootPath.pathAsString}
+      """.stripMargin
+
+    List(Set(writeCommand(jobPaths.callExecutionRoot / "call_caching_placeholder.txt", content, Seq(CloudStorageOptions.withMimeType("text/plain")))))
+  }
+}
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/callcaching/JesBackendFileHashing.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/callcaching/JesBackendFileHashing.scala
deleted file mode 100644
index 42cf8f7be..000000000
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/callcaching/JesBackendFileHashing.scala
+++ /dev/null
@@ -1,21 +0,0 @@
-package cromwell.backend.impl.jes.callcaching
-
-import akka.event.LoggingAdapter
-import cromwell.backend.callcaching.FileHashingActor.SingleFileHashRequest
-import cromwell.backend.impl.jes.JesBackendInitializationData
-
-import scala.util.{Failure, Try}
-
-private[jes] object JesBackendFileHashing {
-  def getCrc32c(singleFileHashRequest: SingleFileHashRequest, log: LoggingAdapter): Try[String] = {
-    def usingJesInitData(jesInitData: JesBackendInitializationData) = for {
-      path <- jesInitData.workflowPaths.getPath(singleFileHashRequest.file.valueString)
-      crc32c <- jesInitData.workflowPaths.getHash(path)
-    } yield crc32c
-
-    singleFileHashRequest.initializationData match {
-      case Some(jesInitData: JesBackendInitializationData) => usingJesInitData(jesInitData)
-      case _ => Failure(new IllegalArgumentException("Need JesBackendInitializationData to generate a GCS CRC32C hash"))
-    }
-  }
-}
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/callcaching/JesBackendFileHashingActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/callcaching/JesBackendFileHashingActor.scala
new file mode 100644
index 000000000..af65a07d8
--- /dev/null
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/callcaching/JesBackendFileHashingActor.scala
@@ -0,0 +1,6 @@
+package cromwell.backend.impl.jes.callcaching
+
+import cromwell.backend.standard.callcaching.{StandardFileHashingActor, StandardFileHashingActorParams}
+import cromwell.filesystems.gcs.batch.GcsBatchCommandBuilder
+
+class JesBackendFileHashingActor(standardParams: StandardFileHashingActorParams) extends StandardFileHashingActor(standardParams) with GcsBatchCommandBuilder
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/callcaching/JesCacheHitDuplicationStrategy.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/callcaching/JesCacheHitDuplicationStrategy.scala
new file mode 100644
index 000000000..9713e9a04
--- /dev/null
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/callcaching/JesCacheHitDuplicationStrategy.scala
@@ -0,0 +1,6 @@
+package cromwell.backend.impl.jes.callcaching
+
+sealed trait JesCacheHitDuplicationStrategy
+
+case object CopyCachedOutputs extends JesCacheHitDuplicationStrategy
+case object UseOriginalCachedOutputs extends JesCacheHitDuplicationStrategy
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/errors/JesError.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/errors/JesError.scala
deleted file mode 100644
index 225941b73..000000000
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/errors/JesError.scala
+++ /dev/null
@@ -1,52 +0,0 @@
-package cromwell.backend.impl.jes.errors
-
-import cromwell.backend.impl.jes.RunStatus
-import cromwell.core.path.Path
-
-import scala.util.Try
-
-object JesError {
-  /**
-    * List of known and mapped JES errors
-    */
-  private val KnownErrors = List(
-    FailedToDelocalize
-  )
-
-  /**
-    * The error message looks like "10: Something bad happened"
-    * Extract 10 as an inner error code and then the rest of the message
-    */
-  private def splitInnerCodeAndMessage(errorMessage: String): Option[(Int, String)] = Try {
-    val sep = errorMessage.indexOf(':')
-    val innerCode = errorMessage.substring(0, sep).toInt
-    val message = errorMessage.substring(sep + 1, errorMessage.length - 1).trim
-    (innerCode, message)
-  }.toOption
-  
-  def fromFailedStatus(failedStatus: RunStatus.Failed, jobTag: String, stderrPath: Option[Path]): Option[JesKnownJobFailure] = {
-    def lookupError(innerCode: Int, message: String) = {
-      val search = KnownErrors.toStream map { _.toFailureOption(failedStatus.errorCode, innerCode, message, jobTag, stderrPath) } find { _.isDefined }
-      search.flatten
-    }
-    
-    failedStatus.errorMessage flatMap splitInnerCodeAndMessage flatMap Function.tupled(lookupError)
-  }
-}
-
-sealed abstract class JesError(val outerCode: Int, val innerCode: Int, val messageStart: String) {
-  def toFailureOption(errorOuterCode: Int, errorInnerCode: Int, errorMessage: String, jobTag: String, stderrPath: Option[Path]): Option[JesKnownJobFailure] = {
-    if (
-        errorOuterCode == outerCode &&
-        errorInnerCode == innerCode &&
-        errorMessage.startsWith(messageStart)
-    ) Option(toJobFailure(errorMessage, jobTag, stderrPath))
-    else None
-  }
-  
-  protected def toJobFailure(message: String, jobTag: String, stderrPath: Option[Path]): JesKnownJobFailure
-}
-
-private case object FailedToDelocalize extends JesError(5, 10, "Failed to delocalize files") {
-  def toJobFailure(message: String, jobTag: String, stderrPath: Option[Path]) = FailedToDelocalizeFailure(message, jobTag, stderrPath)
-}
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/JesAttachedDisk.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/JesAttachedDisk.scala
index 672ab159a..d4d1f2397 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/JesAttachedDisk.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/io/JesAttachedDisk.scala
@@ -7,7 +7,7 @@ import com.google.api.services.genomics.model.Disk
 import cromwell.core.path.{DefaultPathBuilder, Path}
 import lenthall.exception.MessageAggregation
 import lenthall.validation.ErrorOr._
-import wdl4s.values._
+import wdl4s.wdl.values._
 
 import scala.util.Try
 import scala.util.matching.Regex
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesApiQueryManager.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesApiQueryManager.scala
index c24c9d76e..dd65d8225 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesApiQueryManager.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesApiQueryManager.scala
@@ -52,15 +52,13 @@ class JesApiQueryManager(val qps: Int Refined Positive) extends Actor with Actor
     case other => log.error(s"Unexpected message to JesPollingManager: $other")
   }
 
-  private def handleQueryFailure(failure: JesApiQueryFailed) = failure match {
-    case JesApiQueryFailed(request, _) if request.failedAttempts < maxRetries =>
-      val nextRequest = request.withFailedAttempt
-      val delay = nextRequest.backoff.backoffMillis.millis
-      context.system.scheduler.scheduleOnce(delay, self, nextRequest)
-      ()
-    case JesApiQueryFailed(request, _) =>
-      request.requester ! failure
-      ()
+  private def handleQueryFailure(failure: JesApiQueryFailed) = if (failure.query.failedAttempts < maxRetries) {
+    val nextRequest = failure.query.withFailedAttempt
+    val delay = nextRequest.backoff.backoffMillis.millis
+    context.system.scheduler.scheduleOnce(delay, self, nextRequest)
+    ()
+  } else {
+    failure.query.requester ! failure
   }
 
   private def handleJesPollingRequest(workPullingJesPollingActor: ActorRef, maxBatchSize: Int) = {
@@ -80,7 +78,8 @@ class JesApiQueryManager(val qps: Int Refined Positive) extends Actor with Actor
     workQueue = beheaded.newWorkQueue
   }
 
-  private final case class BeheadedWorkQueue(workToDo: Option[NonEmptyList[JesApiQuery]], newWorkQueue: Queue[JesApiQuery])
+  // Intentionally not final, this runs afoul of SI-4440 (I believe)
+  private case class BeheadedWorkQueue(workToDo: Option[NonEmptyList[JesApiQuery]], newWorkQueue: Queue[JesApiQuery])
   private def beheadWorkQueue(maxBatchSize: Int): BeheadedWorkQueue = {
 
     val head = workQueue.take(maxBatchSize).toList
@@ -100,8 +99,11 @@ class JesApiQueryManager(val qps: Int Refined Positive) extends Actor with Actor
         // Most likely due to an unexpected HTTP error, push the work back on the queue and keep going
         log.error(s"The JES API worker actor $terminee unexpectedly terminated while conducting ${work.workBatch.tail.size + 1} polls. Making a new one...")
         workInProgress -= terminee
-        work.workBatch.toList.foreach { toResubmit =>
-          self ! JesApiQueryFailed(toResubmit, new JesApiException(cause))
+        work.workBatch.toList.foreach {
+          case statusQuery: JesStatusPollQuery =>
+            self ! JesApiStatusQueryFailed(statusQuery, new JesApiException(cause))
+          case runCreationQuery: JesRunCreationQuery =>
+            self ! JesApiRunCreationQueryFailed(runCreationQuery, new JesApiException(cause))
         }
       case None =>
         // It managed to die while doing absolutely nothing...!?
@@ -158,12 +160,24 @@ object JesApiQueryManager {
     override def withFailedAttempt = this.copy(failedAttempts = failedAttempts + 1, backoff = backoff.next)
   }
 
+  trait JesApiQueryFailed {
+    val query: JesApiQuery
+    val cause: JesApiException
+  }
+
+  final case class JesApiStatusQueryFailed(query: JesApiQuery, cause: JesApiException) extends JesApiQueryFailed
+  final case class JesApiRunCreationQueryFailed(query: JesApiQuery, cause: JesApiException) extends JesApiQueryFailed
+
   private[statuspolling] final case class JesPollingWorkBatch(workBatch: NonEmptyList[JesApiQuery])
   private[statuspolling] case object NoWorkToDo
 
   private[statuspolling] final case class RequestJesPollingWork(maxBatchSize: Int)
 
-  final case class GoogleJsonException(e: GoogleJsonError, responseHeaders: HttpHeaders) extends IOException with CromwellFatalExceptionMarker
-  final class JesApiException(e: Throwable) extends RuntimeException(e) with CromwellFatalExceptionMarker
-  final case class JesApiQueryFailed(query: JesApiQuery, cause: JesApiException)
+  final case class GoogleJsonException(e: GoogleJsonError, responseHeaders: HttpHeaders) extends IOException with CromwellFatalExceptionMarker {
+    override def getMessage: String = e.getMessage
+  }
+
+  final class JesApiException(e: Throwable) extends RuntimeException(e) with CromwellFatalExceptionMarker {
+    override def getMessage: String = "Unable to complete JES Api Request"
+  }
 }
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActor.scala
index 4f0aff53d..0d953d251 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActor.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActor.scala
@@ -14,7 +14,7 @@ import eu.timepit.refined.numeric._
 import scala.concurrent.{ExecutionContext, Future}
 import scala.util.{Failure, Success, Try}
 import scala.concurrent.duration._
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
 
 /**
   * Sends batched requests to JES as a worker to the JesApiQueryManager
@@ -91,7 +91,7 @@ class JesPollingActor(val pollingManager: ActorRef, val qps: Int Refined Positiv
     ()
   }
 
-  private[statuspolling] def mkErrorString(e: GoogleJsonError) = e.getErrors.toList.mkString(", ")
+  private[statuspolling] def mkErrorString(e: GoogleJsonError) = e.getErrors.asScala.toList.mkString(", ")
 }
 
 object JesPollingActor {
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesRunCreationClient.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesRunCreationClient.scala
index 1114e108b..89785a74c 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesRunCreationClient.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesRunCreationClient.scala
@@ -3,7 +3,7 @@ package cromwell.backend.impl.jes.statuspolling
 import akka.actor.{Actor, ActorLogging, ActorRef}
 import com.google.api.services.genomics.Genomics
 import com.google.api.services.genomics.model.RunPipelineRequest
-import cromwell.backend.impl.jes.statuspolling.JesApiQueryManager.JesApiQueryFailed
+import cromwell.backend.impl.jes.statuspolling.JesApiQueryManager.{JesApiRunCreationQueryFailed}
 import cromwell.backend.standard.StandardAsyncJob
 
 import scala.concurrent.{Future, Promise}
@@ -24,7 +24,7 @@ trait JesRunCreationClient { this: Actor with ActorLogging =>
   def runCreationClientReceive: Actor.Receive = {
     case job: StandardAsyncJob =>
       completePromise(Success(job))
-    case JesApiQueryFailed(_, e) => completePromise(Failure(e))
+    case JesApiRunCreationQueryFailed(_, e) => completePromise(Failure(e))
   }
 
   private def completePromise(job: Try[StandardAsyncJob]) = {
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesStatusRequestClient.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesStatusRequestClient.scala
index 6b7f8ee00..3ddff56ea 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesStatusRequestClient.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/JesStatusRequestClient.scala
@@ -1,7 +1,7 @@
 package cromwell.backend.impl.jes.statuspolling
 
 import akka.actor.{Actor, ActorLogging, ActorRef}
-import cromwell.backend.impl.jes.statuspolling.JesApiQueryManager.JesApiQueryFailed
+import cromwell.backend.impl.jes.statuspolling.JesApiQueryManager.{JesApiStatusQueryFailed}
 import cromwell.backend.impl.jes.{Run, RunStatus}
 
 import scala.concurrent.{Future, Promise}
@@ -23,7 +23,7 @@ trait JesStatusRequestClient { this: Actor with ActorLogging =>
     case r: RunStatus =>
       log.debug(s"Polled status received: $r")
       completePromise(Success(r))
-    case JesApiQueryFailed(_, e) =>
+    case JesApiStatusQueryFailed(_, e) =>
       log.debug("JES poll failed!")
       completePromise(Failure(e))
   }
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/RunCreation.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/RunCreation.scala
index 3e6536597..54759590b 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/RunCreation.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/RunCreation.scala
@@ -22,7 +22,7 @@ private[statuspolling] trait RunCreation { this: JesPollingActor =>
     }
 
     override def onFailure(e: GoogleJsonError, responseHeaders: HttpHeaders): Unit = {
-      pollingManager ! JesApiQueryFailed(originalRequest, new JesApiException(GoogleJsonException(e, responseHeaders)))
+      pollingManager ! JesApiRunCreationQueryFailed(originalRequest, new JesApiException(GoogleJsonException(e, responseHeaders)))
       completionPromise.trySuccess(Failure(new Exception(mkErrorString(e))))
       ()
     }
diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/StatusPolling.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/StatusPolling.scala
index afd4b0673..80291334f 100644
--- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/StatusPolling.scala
+++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/statuspolling/StatusPolling.scala
@@ -9,9 +9,9 @@ import com.google.api.client.googleapis.batch.json.JsonBatchCallback
 import com.google.api.client.googleapis.json.GoogleJsonError
 import com.google.api.client.http.HttpHeaders
 import com.google.api.services.genomics.model.Operation
-import cromwell.backend.impl.jes.RunStatus.{Failed, Initializing, Running, Success}
+import cromwell.backend.impl.jes.RunStatus._
 import cromwell.backend.impl.jes.{Run, RunStatus}
-import cromwell.backend.impl.jes.statuspolling.JesApiQueryManager.{GoogleJsonException, JesApiException, JesApiQueryFailed, JesStatusPollQuery}
+import cromwell.backend.impl.jes.statuspolling.JesApiQueryManager._
 import cromwell.core.ExecutionEvent
 
 import scala.language.postfixOps
@@ -29,7 +29,7 @@ private[statuspolling] trait StatusPolling { this: JesPollingActor =>
     }
 
     override def onFailure(e: GoogleJsonError, responseHeaders: HttpHeaders): Unit = {
-      pollingManager ! JesApiQueryFailed(originalRequest, new JesApiException(GoogleJsonException(e, responseHeaders)))
+      pollingManager ! JesApiStatusQueryFailed(originalRequest, new JesApiException(GoogleJsonException(e, responseHeaders)))
       completionPromise.trySuccess(Failure(new Exception(mkErrorString(e))))
       ()
     }
@@ -42,7 +42,7 @@ private[statuspolling] trait StatusPolling { this: JesPollingActor =>
     completionPromise.future
   }
 
-  // Done so that the specs can override this behaviour
+  // Done so that the specs can override this behavior
   def interpretOperationStatus(op: Operation) = StatusPolling.interpretOperationStatus(op)
 
   def addStatusPollToBatch(run: Run, batch: BatchRequest, resultHandler: JsonBatchCallback[Operation]) =
@@ -70,11 +70,10 @@ private[statuspolling] object StatusPolling {
         lazy val instanceName = computeEngineOption.flatMap(_.get("instanceName"))
         lazy val zone = computeEngineOption.flatMap(_.get("zone"))
 
-        // If there's an error, generate a Failed status. Otherwise, we were successful!
+        // If there's an error, generate an unsuccessful status. Otherwise, we were successful!
         Option(op.getError) match {
+          case Some(error) => UnsuccessfulRunStatus(error.getCode, Option(error.getMessage), eventList, machineType, zone, instanceName)
           case None => Success(eventList, machineType, zone, instanceName)
-          case Some(error) =>
-            Failed(error.getCode, Option(error.getMessage), eventList, machineType, zone, instanceName)
         }
       } else if (op.hasStarted) {
         Running
diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/GoogleLabelsSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/GoogleLabelsSpec.scala
new file mode 100644
index 000000000..c2fb14444
--- /dev/null
+++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/GoogleLabelsSpec.scala
@@ -0,0 +1,37 @@
+package cromwell.backend.impl.jes
+
+import cats.data.Validated.{Invalid, Valid}
+import cromwell.core.labels.Label
+import org.scalatest.{FlatSpec, Matchers}
+
+class GoogleLabelsSpec extends FlatSpec with Matchers {
+
+  behavior of "GoogleLabels"
+
+  /**
+    * In the format 'to validate', 'expected result'
+    */
+  val googleLabelConversions = List(
+    "11f2468c-39d6-4be3-85c8-32735c01e66b" -> "x--11f2468c-39d6-4be3-85c8-32735c01e66b",
+    "0-cromwell-root-workflow-id" -> "x--0-cromwell-root-workflow-id",
+    "cromwell-root-workflow-id-" -> "cromwell-root-workflow-id---x",
+    "0-cromwell-root-workflow-id-" -> "x--0-cromwell-root-workflow-id---x",
+    "Cromwell-root-workflow-id" -> "cromwell-root-workflow-id",
+    "cromwell_root_workflow_id" -> "cromwell-root-workflow-id",
+    "too-long-too-long-too-long-too-long-too-long-too-long-too-long-t" -> "too-long-too-long-too-long-too---g-too-long-too-long-too-long-t",
+    "0-too-long-and-invalid-too-long-and-invalid-too-long-and-invali+" -> "x--0-too-long-and-invalid-too----nvalid-too-long-and-invali---x"
+  )
+
+  googleLabelConversions foreach { case (label: String, conversion: String) =>
+    it should s"not validate the bad label string '$label'" in {
+      Label.validateLabelRegex(label, GoogleLabels.GoogleLabelsRegexPattern.r) match {
+        case Invalid(_) => // Good!
+        case Valid(_) => fail(s"Label validation succeeded but should have failed.")
+      }
+    }
+
+    it should s"convert the bad label string '$label' into the safe label string '$conversion'" in {
+      GoogleLabels.safeGoogleName(label) should be(conversion)
+    }
+  }
+}
diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala
index eb5272cbd..4848e8546 100644
--- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala
+++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActorSpec.scala
@@ -4,45 +4,51 @@ import java.util.UUID
 
 import akka.actor.{ActorRef, Props}
 import akka.testkit.{ImplicitSender, TestActorRef, TestDuration, TestProbe}
-import cromwell.backend.BackendJobExecutionActor.BackendJobExecutionResponse
+import com.google.cloud.NoCredentials
+import cromwell.backend.BackendJobExecutionActor.{BackendJobExecutionResponse, JobFailedNonRetryableResponse, JobFailedRetryableResponse}
 import cromwell.backend._
 import cromwell.backend.async.AsyncBackendJobExecutionActor.{Execute, ExecutionMode}
 import cromwell.backend.async.{AbortedExecutionHandle, ExecutionHandle, FailedNonRetryableExecutionHandle, FailedRetryableExecutionHandle}
 import cromwell.backend.impl.jes.JesAsyncBackendJobExecutionActor.JesPendingExecutionHandle
-import cromwell.backend.impl.jes.RunStatus.Failed
+import cromwell.backend.impl.jes.RunStatus.UnsuccessfulRunStatus
 import cromwell.backend.impl.jes.io.{DiskType, JesWorkingDisk}
 import cromwell.backend.impl.jes.statuspolling.JesApiQueryManager.DoPoll
 import cromwell.backend.standard.{DefaultStandardAsyncExecutionActorParams, StandardAsyncExecutionActorParams, StandardAsyncJob, StandardExpressionFunctionsParams}
 import cromwell.backend.wdl.WdlFileMapper
 import cromwell.core._
-import cromwell.core.callcaching.CallCachingEligible
+import cromwell.core.callcaching.NoDocker
 import cromwell.core.labels.Labels
 import cromwell.core.logging.JobLogger
 import cromwell.core.path.{DefaultPathBuilder, PathBuilder}
-import cromwell.filesystems.gcs.auth.GoogleAuthMode.NoAuthMode
 import cromwell.filesystems.gcs.{GcsPath, GcsPathBuilder, GcsPathBuilderFactory}
+import cromwell.services.keyvalue.InMemoryKvServiceActor
+import cromwell.services.keyvalue.KeyValueServiceActor._
 import cromwell.util.SampleWdl
 import org.scalatest._
 import org.scalatest.prop.Tables.Table
 import org.slf4j.Logger
 import org.specs2.mock.Mockito
 import spray.json.{JsObject, JsValue}
-import wdl4s._
-import wdl4s.types.{WdlArrayType, WdlFileType, WdlMapType, WdlStringType}
-import wdl4s.values.{WdlArray, WdlFile, WdlMap, WdlString, WdlValue}
+import wdl4s.wdl._
+import wdl4s.wdl.types.{WdlArrayType, WdlFileType, WdlMapType, WdlStringType}
+import wdl4s.wdl.values.{WdlArray, WdlFile, WdlMap, WdlString, WdlValue}
 
 import scala.concurrent.duration._
 import scala.concurrent.{Await, ExecutionContext, Future, Promise}
+import scala.language.postfixOps
 import scala.util.{Success, Try}
 
 class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackendJobExecutionActorSpec")
-  with FlatSpecLike with Matchers with ImplicitSender with Mockito with BackendSpec {
+  with FlatSpecLike with Matchers with ImplicitSender with Mockito with BackendSpec with BeforeAndAfter {
 
-  val mockPathBuilder: GcsPathBuilder = GcsPathBuilderFactory(NoAuthMode).withOptions(WorkflowOptions.empty)
+  val mockPathBuilder: GcsPathBuilder = GcsPathBuilder.fromCredentials(NoCredentials.getInstance(),
+    "test-cromwell", None, GcsPathBuilderFactory.DefaultCloudStorageConfiguration, WorkflowOptions.empty)
+  
+  var kvService: ActorRef = system.actorOf(Props(new InMemoryKvServiceActor))
 
   import JesTestConfig._
 
-  implicit val Timeout: FiniteDuration = 5.seconds.dilated
+  implicit val Timeout: FiniteDuration = 25.seconds.dilated
 
   val YoSup: String =
     s"""
@@ -81,7 +87,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
   }
 
   private def buildInitializationData(jobDescriptor: BackendJobDescriptor, configuration: JesConfiguration) = {
-    val workflowPaths = JesWorkflowPaths(jobDescriptor.workflowDescriptor, configuration)(system)
+    val workflowPaths = JesWorkflowPaths(jobDescriptor.workflowDescriptor, NoCredentials.getInstance(), NoCredentials.getInstance(), configuration)(system)
     val runtimeAttributesBuilder = JesRuntimeAttributes.runtimeAttributesBuilder(configuration)
     JesBackendInitializationData(workflowPaths, runtimeAttributesBuilder, configuration, null, null)
   }
@@ -93,16 +99,19 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
              promise: Promise[BackendJobExecutionResponse],
              jesConfiguration: JesConfiguration,
              functions: JesExpressionFunctions = TestableJesExpressionFunctions,
-             jesSingletonActor: ActorRef = emptyActor) = {
+             jesSingletonActor: ActorRef = emptyActor,
+             ioActor: ActorRef = mockIoActor) = {
+
       this(
         DefaultStandardAsyncExecutionActorParams(
-          JesAsyncBackendJobExecutionActor.JesOperationIdKey,
-          emptyActor,
-          jobDescriptor,
-          jesConfiguration.configurationDescriptor,
-          Option(buildInitializationData(jobDescriptor, jesConfiguration)),
-          Option(jesSingletonActor),
-          promise
+          jobIdKey = JesAsyncBackendJobExecutionActor.JesOperationIdKey,
+          serviceRegistryActor = kvService,
+          ioActor = ioActor,
+          jobDescriptor = jobDescriptor,
+          configurationDescriptor = jesConfiguration.configurationDescriptor,
+          backendInitializationDataOption = Option(buildInitializationData(jobDescriptor, jesConfiguration)),
+          backendSingletonActorOption = Option(jesSingletonActor),
+          completionPromise = promise
         ),
         functions
       )
@@ -128,7 +137,8 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
       |}
     """.stripMargin
 
-  private def buildPreemptibleJobDescriptor(attempt: Int, preemptible: Int): BackendJobDescriptor = {
+  private def buildPreemptibleJobDescriptor(preemptible: Int, previousPreemptions: Int, previousUnexpectedRetries: Int): BackendJobDescriptor = {
+    val attempt = previousPreemptions + previousUnexpectedRetries + 1
     val workflowDescriptor = BackendWorkflowDescriptor(
       WorkflowId.randomId(),
       WdlNamespaceWithWorkflow.load(YoSup.replace("[PREEMPTIBLE]", s"preemptible: $preemptible"), Seq.empty[ImportResolver]).get.workflow,
@@ -140,33 +150,44 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
     val job = workflowDescriptor.workflow.taskCalls.head
     val key = BackendJobDescriptorKey(job, None, attempt)
     val runtimeAttributes = makeRuntimeAttributes(job)
-    BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, fqnMapToDeclarationMap(Inputs), CallCachingEligible)
+    val prefetchedKvEntries = Map(
+      JesBackendLifecycleActorFactory.preemptionCountKey -> KvPair(ScopedKey(workflowDescriptor.id, KvJobKey(key), JesBackendLifecycleActorFactory.preemptionCountKey), Some(previousPreemptions.toString)),
+      JesBackendLifecycleActorFactory.unexpectedRetryCountKey -> KvPair(ScopedKey(workflowDescriptor.id, KvJobKey(key), JesBackendLifecycleActorFactory.unexpectedRetryCountKey), Some(previousUnexpectedRetries.toString)))
+    BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, fqnMapToDeclarationMap(Inputs), NoDocker, prefetchedKvEntries)
   }
 
   private def executionActor(jobDescriptor: BackendJobDescriptor,
                              configurationDescriptor: BackendConfigurationDescriptor,
                              promise: Promise[BackendJobExecutionResponse],
-                             jesSingletonActor: ActorRef): ActorRef = {
+                             jesSingletonActor: ActorRef,
+                             shouldBePreemptible: Boolean): ActorRef = {
 
     val job = StandardAsyncJob(UUID.randomUUID().toString)
     val run = Run(job, null)
     val handle = new JesPendingExecutionHandle(jobDescriptor, run.job, Option(run), None)
 
     class ExecuteOrRecoverActor extends TestableJesJobExecutionActor(jobDescriptor, promise, jesConfiguration, jesSingletonActor = jesSingletonActor) {
-      override def executeOrRecover(mode: ExecutionMode)(implicit ec: ExecutionContext): Future[ExecutionHandle] = Future.successful(handle)
+      override def executeOrRecover(mode: ExecutionMode)(implicit ec: ExecutionContext): Future[ExecutionHandle] = {
+        if(preemptible == shouldBePreemptible) Future.successful(handle)
+        else Future.failed(new Exception(s"Test expected preemptible to be $shouldBePreemptible but got $preemptible"))
+      }
     }
 
     system.actorOf(Props(new ExecuteOrRecoverActor), "ExecuteOrRecoverActor-" + UUID.randomUUID)
   }
 
-  private def runAndFail(attempt: Int, preemptible: Int, errorCode: Int, innerErrorCode: Int): BackendJobExecutionResponse = {
+  private def runAndFail(previousPreemptions: Int, previousUnexpectedRetries: Int, preemptible: Int, errorCode: Int, innerErrorCode: Int, expectPreemptible: Boolean): BackendJobExecutionResponse = {
 
-    val runStatus = Failed(errorCode, Option(s"$innerErrorCode: I seen some things man"), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance"))
+    val runStatus = UnsuccessfulRunStatus(errorCode, Option(s"$innerErrorCode: I seen some things man"), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance"))
     val statusPoller = TestProbe()
 
     val promise = Promise[BackendJobExecutionResponse]()
-    val jobDescriptor =  buildPreemptibleJobDescriptor(attempt, preemptible)
-    val backend = executionActor(jobDescriptor, JesBackendConfigurationDescriptor, promise, statusPoller.ref)
+    val jobDescriptor =  buildPreemptibleJobDescriptor(preemptible, previousPreemptions, previousUnexpectedRetries)
+
+    // TODO: Use this to check the new KV entries are there!
+    //val kvProbe = TestProbe()
+
+    val backend = executionActor(jobDescriptor, JesBackendConfigurationDescriptor, promise, statusPoller.ref, expectPreemptible)
     backend ! Execute
     statusPoller.expectMsgPF(max = Timeout, hint = "awaiting status poll") {
       case DoPoll(_) => backend ! runStatus
@@ -176,42 +197,58 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
   }
 
   def buildPreemptibleTestActorRef(attempt: Int, preemptible: Int): TestActorRef[TestableJesJobExecutionActor] = {
-    val jobDescriptor = buildPreemptibleJobDescriptor(attempt, preemptible)
-    val props = Props(new TestableJesJobExecutionActor(jobDescriptor, Promise(), jesConfiguration))
+    // For this test we say that all previous attempts were preempted:
+    val jobDescriptor = buildPreemptibleJobDescriptor(preemptible, attempt - 1, 0)
+    val props = Props(new TestableJesJobExecutionActor(jobDescriptor, Promise(),
+      jesConfiguration,
+      TestableJesExpressionFunctions,
+      emptyActor,
+      failIoActor))
     TestActorRef(props, s"TestableJesJobExecutionActor-${jobDescriptor.workflowDescriptor.id}")
   }
 
   behavior of "JesAsyncBackendJobExecutionActor"
+  
+  val timeout = 25 seconds
 
-  { // Set of "handle call failures appropriately with respect to preemption" tests
+  { // Set of "handle call failures appropriately with respect to preemption and failure" tests
     val expectations = Table(
-      ("attempt", "preemptible", "errorCode", "innerErrorCode", "shouldRetry"),
-      // No preemptible attempts allowed, nothing should be retryable.
-      (1, 0, 10, 13, false),
-      (1, 0, 10, 14, false),
-      (1, 0, 10, 15, false),
-      (1, 0, 11, 13, false),
-      (1, 0, 11, 14, false),
+      ("previous_preemptions", "previous_unexpectedRetries", "preemptible", "errorCode", "innerErrorCode", "shouldRunAsPreemptible", "shouldRetry"),
+      // No preemptible attempts allowed, but standard failures should be retried.
+      (0, 0, 0, 10, 13, false, true), // This is the new "unexpected failure" mode, which is now retried
+      (0, 1, 0, 10, 13, false, true),
+      (0, 2, 0, 10, 13, false, false), // The third unexpected failure is a real failure.
+      (0, 0, 0, 10, 14, false, false), // Usually means "preempted', but this wasn't a preemptible VM, so this should just be a failure.
+      (0, 0, 0, 10, 15, false, false),
+      (0, 0, 0, 11, 13, false, false),
+      (0, 0, 0, 11, 14, false, false),
       // 1 preemptible attempt allowed, but not all failures represent preemptions.
-      (1, 1, 10, 13, true),
-      (1, 1, 10, 14, true),
-      (1, 1, 10, 15, false),
-      (1, 1, 11, 13, false),
-      (1, 1, 11, 14, false),
-      // 1 preemptible attempt allowed, but now on the second attempt nothing should be retryable.
-      (2, 1, 10, 13, false),
-      (2, 1, 10, 14, false),
-      (2, 1, 10, 15, false),
-      (2, 1, 11, 13, false),
-      (2, 1, 11, 14, false)
+      (0, 0, 1, 10, 13, true, true),
+      (0, 1, 1, 10, 13, true, true),
+      (0, 2, 1, 10, 13, true, false),
+      (0, 0, 1, 10, 14, true, true),
+      (0, 0, 1, 10, 15, true, false),
+      (0, 0, 1, 11, 13, true, false),
+      (0, 0, 1, 11, 14, true, false),
+      // 1 preemptible attempt allowed, but since we're now on the second preemption attempt only 13s should be retryable.
+      (1, 0, 1, 10, 13, false, true),
+      (1, 1, 1, 10, 13, false, true),
+      (1, 2, 1, 10, 13, false, false),
+      (1, 0, 1, 10, 14, false, false),
+      (1, 0, 1, 10, 15, false, false),
+      (1, 0, 1, 11, 13, false, false),
+      (1, 0, 1, 11, 14, false, false)
     )
 
-    expectations foreach { case (attempt, preemptible, errorCode, innerErrorCode, shouldRetry) =>
-      it should s"handle call failures appropriately with respect to preemption (attempt=$attempt, preemptible=$preemptible, errorCode=$errorCode, innerErrorCode=$innerErrorCode)" in {
-        runAndFail(attempt, preemptible, errorCode, innerErrorCode).getClass.getSimpleName match {
-          case "JobFailedNonRetryableResponse" => false shouldBe shouldRetry
-          case "JobFailedRetryableResponse" => true shouldBe shouldRetry
-          case huh => fail(s"Unexpected response class name: '$huh'")
+    expectations foreach { case (previousPreemptions, previousUnexpectedRetries, preemptible, errorCode, innerErrorCode, shouldBePreemptible, shouldRetry) =>
+      val descriptor = s"previousPreemptions=$previousPreemptions, previousUnexpectedRetries=$previousUnexpectedRetries preemptible=$preemptible, errorCode=$errorCode, innerErrorCode=$innerErrorCode"
+      it should s"handle call failures appropriately with respect to preemption and failure ($descriptor)" in {
+        runAndFail(previousPreemptions, previousUnexpectedRetries, preemptible, errorCode, innerErrorCode, shouldBePreemptible) match {
+          case response: JobFailedNonRetryableResponse =>
+            if(shouldRetry) fail(s"A should-be-retried job ($descriptor) was sent back to the engine with: $response")
+          case response: JobFailedRetryableResponse =>
+            if(!shouldRetry) fail(s"A shouldn't-be-retried job ($descriptor) was sent back to the engine with $response")
+          case huh => fail(s"Unexpected response: $huh")
         }
       }
     }
@@ -223,10 +260,11 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
     val runId = StandardAsyncJob(UUID.randomUUID().toString)
     val handle = new JesPendingExecutionHandle(null, runId, None, None)
 
-    val failedStatus = Failed(10, Option("14: VM XXX shut down unexpectedly."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance"))
+    val failedStatus = UnsuccessfulRunStatus(10, Option("14: VM XXX shut down unexpectedly."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance"))
     val executionResult = jesBackend.handleExecutionResult(failedStatus, handle)
-    executionResult.isInstanceOf[FailedNonRetryableExecutionHandle] shouldBe true
-    val failedHandle = executionResult.asInstanceOf[FailedNonRetryableExecutionHandle]
+    val result = Await.result(executionResult, timeout)
+    result.isInstanceOf[FailedNonRetryableExecutionHandle] shouldBe true
+    val failedHandle = result.asInstanceOf[FailedNonRetryableExecutionHandle]
     failedHandle.returnCode shouldBe None
   }
 
@@ -236,14 +274,13 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
     val runId = StandardAsyncJob(UUID.randomUUID().toString)
     val handle = new JesPendingExecutionHandle(null, runId, None, None)
 
-    val failedStatus = Failed(10, Option("14: VM XXX shut down unexpectedly."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance"))
+    val failedStatus = UnsuccessfulRunStatus(10, Option("14: VM XXX shut down unexpectedly."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance"))
     val executionResult = jesBackend.handleExecutionResult(failedStatus, handle)
-    executionResult.isInstanceOf[FailedRetryableExecutionHandle] shouldBe true
-    val retryableHandle = executionResult.asInstanceOf[FailedRetryableExecutionHandle]
-    retryableHandle.throwable.isInstanceOf[PreemptedException] shouldBe true
+    val result = Await.result(executionResult, timeout)
+    result.isInstanceOf[FailedRetryableExecutionHandle] shouldBe true
+    val retryableHandle = result.asInstanceOf[FailedRetryableExecutionHandle]
     retryableHandle.returnCode shouldBe None
-    val preemptedException = retryableHandle.throwable.asInstanceOf[PreemptedException]
-    preemptedException.getMessage should include("will be restarted with a non-preemptible VM")
+    retryableHandle.throwable.getMessage should include("will be restarted with a non-preemptible VM")
   }
 
   it should "restart 1 of 2 unexpected shutdowns with another preemptible VM" in {
@@ -252,14 +289,13 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
     val runId = StandardAsyncJob(UUID.randomUUID().toString)
     val handle = new JesPendingExecutionHandle(null, runId, None, None)
 
-    val failedStatus = Failed(10, Option("14: VM XXX shut down unexpectedly."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance"))
+    val failedStatus = UnsuccessfulRunStatus(10, Option("14: VM XXX shut down unexpectedly."), Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance"))
     val executionResult = jesBackend.handleExecutionResult(failedStatus, handle)
-    executionResult.isInstanceOf[FailedRetryableExecutionHandle] shouldBe true
-    val retryableHandle = executionResult.asInstanceOf[FailedRetryableExecutionHandle]
-    retryableHandle.throwable.isInstanceOf[PreemptedException] shouldBe true
+    val result = Await.result(executionResult, timeout)
+    result.isInstanceOf[FailedRetryableExecutionHandle] shouldBe true
+    val retryableHandle = result.asInstanceOf[FailedRetryableExecutionHandle]
     retryableHandle.returnCode shouldBe None
-    val preemptedException2 = retryableHandle.throwable.asInstanceOf[PreemptedException]
-    preemptedException2.getMessage should include("will be restarted with another preemptible VM")
+    retryableHandle.throwable.getMessage should include("will be restarted with another preemptible VM")
   }
 
   it should "handle Failure Status for various errors" in {
@@ -269,9 +305,8 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
     val handle = new JesPendingExecutionHandle(null, runId, None, None)
 
     def checkFailedResult(errorCode: Int, errorMessage: Option[String]): ExecutionHandle = {
-      val failed =
-        Failed(errorCode, errorMessage, Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance"))
-      jesBackend.handleExecutionResult(failed, handle)
+      val failed = UnsuccessfulRunStatus(errorCode, errorMessage, Seq.empty, Option("fakeMachine"), Option("fakeZone"), Option("fakeInstance"))
+      Await.result(jesBackend.handleExecutionResult(failed, handle), timeout)
     }
 
     checkFailedResult(10, Option("15: Other type of error."))
@@ -281,7 +316,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
     checkFailedResult(10, Option("UnparsableInt: Even weirder error message."))
       .isInstanceOf[FailedNonRetryableExecutionHandle] shouldBe true
     checkFailedResult(10, None).isInstanceOf[FailedNonRetryableExecutionHandle] shouldBe true
-    checkFailedResult(10, Option("Operation canceled at")) shouldBe AbortedExecutionHandle
+    checkFailedResult(1, Option("Operation canceled at")) shouldBe AbortedExecutionHandle
 
     actorRef.stop()
   }
@@ -311,7 +346,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
     val call = workflowDescriptor.workflow.taskCalls.head
     val key = BackendJobDescriptorKey(call, None, 1)
     val runtimeAttributes = makeRuntimeAttributes(call)
-    val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, fqnMapToDeclarationMap(inputs), CallCachingEligible)
+    val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, fqnMapToDeclarationMap(inputs), NoDocker, Map.empty)
 
     val props = Props(new TestableJesJobExecutionActor(jobDescriptor, Promise(), jesConfiguration))
     val testActorRef = TestActorRef[TestableJesJobExecutionActor](
@@ -362,7 +397,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
 
     val workflowDescriptor = BackendWorkflowDescriptor(
       WorkflowId.randomId(),
-      WdlNamespaceWithWorkflow.load(SampleWdl.CurrentDirectory.asWorkflowSources(DockerAndDiskRuntime).wdlSource, Seq.empty[ImportResolver]).get.workflow,
+      WdlNamespaceWithWorkflow.load(SampleWdl.CurrentDirectory.asWorkflowSources(DockerAndDiskRuntime).workflowSource, Seq.empty[ImportResolver]).get.workflow,
       inputs,
       NoOptions,
       Labels.empty
@@ -371,7 +406,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
     val job = workflowDescriptor.workflow.taskCalls.head
     val runtimeAttributes = makeRuntimeAttributes(job)
     val key = BackendJobDescriptorKey(job, None, 1)
-    val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, fqnMapToDeclarationMap(inputs), CallCachingEligible)
+    val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, fqnMapToDeclarationMap(inputs), NoDocker, Map.empty)
 
     val props = Props(new TestableJesJobExecutionActor(jobDescriptor, Promise(), jesConfiguration))
     val testActorRef = TestActorRef[TestableJesJobExecutionActor](
@@ -402,16 +437,16 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
   TestActorRef[TestableJesJobExecutionActor] = {
     val workflowDescriptor = BackendWorkflowDescriptor(
       WorkflowId.randomId(),
-      WdlNamespaceWithWorkflow.load(sampleWdl.asWorkflowSources(DockerAndDiskRuntime).wdlSource, Seq.empty[ImportResolver]).get.workflow,
+      WdlNamespaceWithWorkflow.load(sampleWdl.asWorkflowSources(DockerAndDiskRuntime).workflowSource, Seq.empty[ImportResolver]).get.workflow,
       inputs,
       NoOptions,
       Labels.empty
     )
 
-    val call = workflowDescriptor.workflow.findCallByName(callName).get.asInstanceOf[TaskCall]
+    val call = workflowDescriptor.workflow.findCallByName(callName).get.asInstanceOf[WdlTaskCall]
     val key = BackendJobDescriptorKey(call, None, 1)
     val runtimeAttributes = makeRuntimeAttributes(call)
-    val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, fqnMapToDeclarationMap(inputs), CallCachingEligible)
+    val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, fqnMapToDeclarationMap(inputs), NoDocker, Map.empty)
 
     val props = Props(new TestableJesJobExecutionActor(jobDescriptor, Promise(), jesConfiguration, functions))
     TestActorRef[TestableJesJobExecutionActor](props, s"TestableJesJobExecutionActor-${jobDescriptor.workflowDescriptor.id}")
@@ -463,7 +498,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
 
     val workflowDescriptor = BackendWorkflowDescriptor(
       WorkflowId.randomId(),
-      WdlNamespaceWithWorkflow.load(SampleWdl.CurrentDirectory.asWorkflowSources(DockerAndDiskRuntime).wdlSource, Seq.empty[ImportResolver]).get.workflow,
+      WdlNamespaceWithWorkflow.load(SampleWdl.CurrentDirectory.asWorkflowSources(DockerAndDiskRuntime).workflowSource, Seq.empty[ImportResolver]).get.workflow,
       inputs,
       NoOptions,
       Labels.empty
@@ -472,7 +507,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
     val job = workflowDescriptor.workflow.taskCalls.head
     val runtimeAttributes = makeRuntimeAttributes(job)
     val key = BackendJobDescriptorKey(job, None, 1)
-    val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, fqnMapToDeclarationMap(inputs), CallCachingEligible)
+    val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, fqnMapToDeclarationMap(inputs), NoDocker, Map.empty)
 
     val props = Props(new TestableJesJobExecutionActor(jobDescriptor, Promise(), jesConfiguration))
     val testActorRef = TestActorRef[TestableJesJobExecutionActor](
@@ -492,7 +527,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
 
     val workflowDescriptor = BackendWorkflowDescriptor(
       WorkflowId.randomId(),
-      WdlNamespaceWithWorkflow.load(SampleWdl.CurrentDirectory.asWorkflowSources(DockerAndDiskRuntime).wdlSource, Seq.empty[ImportResolver]).get.workflow,
+      WdlNamespaceWithWorkflow.load(SampleWdl.CurrentDirectory.asWorkflowSources(DockerAndDiskRuntime).workflowSource, Seq.empty[ImportResolver]).get.workflow,
       inputs,
       NoOptions,
       Labels.empty
@@ -501,7 +536,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
     val job = workflowDescriptor.workflow.taskCalls.head
     val runtimeAttributes = makeRuntimeAttributes(job)
     val key = BackendJobDescriptorKey(job, None, 1)
-    val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, fqnMapToDeclarationMap(inputs), CallCachingEligible)
+    val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, fqnMapToDeclarationMap(inputs), NoDocker, Map.empty)
 
     val props = Props(new TestableJesJobExecutionActor(jobDescriptor, Promise(), jesConfiguration))
     val testActorRef = TestActorRef[TestableJesJobExecutionActor](
@@ -537,7 +572,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
 
     val workflowDescriptor = BackendWorkflowDescriptor(
       WorkflowId.randomId(),
-      WdlNamespaceWithWorkflow.load(SampleWdl.EmptyString.asWorkflowSources(DockerAndDiskRuntime).wdlSource, Seq.empty[ImportResolver]).get.workflow,
+      WdlNamespaceWithWorkflow.load(SampleWdl.EmptyString.asWorkflowSources(DockerAndDiskRuntime).workflowSource, Seq.empty[ImportResolver]).get.workflow,
       Map.empty,
       NoOptions,
       Labels.empty
@@ -546,7 +581,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
     val call = workflowDescriptor.workflow.taskCalls.head
     val key = BackendJobDescriptorKey(call, None, 1)
     val runtimeAttributes = makeRuntimeAttributes(call)
-    val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, Map.empty, CallCachingEligible)
+    val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, Map.empty, NoDocker, Map.empty)
 
     val props = Props(new TestableJesJobExecutionActor(jobDescriptor, Promise(), jesConfiguration))
     val testActorRef = TestActorRef[TestableJesJobExecutionActor](
@@ -570,7 +605,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
   it should "create a JesFileInput for the monitoring script, when specified" in {
     val workflowDescriptor = BackendWorkflowDescriptor(
       WorkflowId.randomId(),
-      WdlNamespaceWithWorkflow.load(SampleWdl.EmptyString.asWorkflowSources(DockerAndDiskRuntime).wdlSource, Seq.empty[ImportResolver]).get.workflow,
+      WdlNamespaceWithWorkflow.load(SampleWdl.EmptyString.asWorkflowSources(DockerAndDiskRuntime).workflowSource, Seq.empty[ImportResolver]).get.workflow,
       Map.empty,
       WorkflowOptions.fromJsonString("""{"monitoring_script": "gs://path/to/script"}""").get,
       Labels.empty
@@ -579,7 +614,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
     val job = workflowDescriptor.workflow.taskCalls.head
     val runtimeAttributes = makeRuntimeAttributes(job)
     val key = BackendJobDescriptorKey(job, None, 1)
-    val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, Map.empty, CallCachingEligible)
+    val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, Map.empty, NoDocker, Map.empty)
 
     val props = Props(new TestableJesJobExecutionActor(jobDescriptor, Promise(), jesConfiguration))
     val testActorRef = TestActorRef[TestableJesJobExecutionActor](
@@ -592,7 +627,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
   it should "not create a JesFileInput for the monitoring script, when not specified" in {
     val workflowDescriptor = BackendWorkflowDescriptor(
       WorkflowId.randomId(),
-      WdlNamespaceWithWorkflow.load(SampleWdl.EmptyString.asWorkflowSources(DockerAndDiskRuntime).wdlSource, Seq.empty[ImportResolver]).get.workflow,
+      WdlNamespaceWithWorkflow.load(SampleWdl.EmptyString.asWorkflowSources(DockerAndDiskRuntime).workflowSource, Seq.empty[ImportResolver]).get.workflow,
       Map.empty,
       NoOptions,
       Labels.empty
@@ -601,7 +636,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
     val job = workflowDescriptor.workflow.taskCalls.head
     val key = BackendJobDescriptorKey(job, None, 1)
     val runtimeAttributes = makeRuntimeAttributes(job)
-    val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, Map.empty, CallCachingEligible)
+    val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, Map.empty, NoDocker, Map.empty)
 
     val props = Props(new TestableJesJobExecutionActor(jobDescriptor, Promise(), jesConfiguration))
     val testActorRef = TestActorRef[TestableJesJobExecutionActor](
@@ -614,16 +649,16 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
     val workflowDescriptor = BackendWorkflowDescriptor(
       WorkflowId(UUID.fromString("e6236763-c518-41d0-9688-432549a8bf7c")),
       WdlNamespaceWithWorkflow.load(
-        SampleWdl.HelloWorld.asWorkflowSources(""" runtime {docker: "ubuntu:latest"} """).wdlSource, Seq.empty[ImportResolver]).get.workflow,
+        SampleWdl.HelloWorld.asWorkflowSources(""" runtime {docker: "ubuntu:latest"} """).workflowSource, Seq.empty[ImportResolver]).get.workflow,
       Map.empty,
       WorkflowOptions.fromJsonString(""" {"jes_gcs_root": "gs://path/to/gcs_root"} """).get,
       Labels.empty
     )
 
-    val call = workflowDescriptor.workflow.findCallByName("hello").get.asInstanceOf[TaskCall]
+    val call = workflowDescriptor.workflow.findCallByName("hello").get.asInstanceOf[WdlTaskCall]
     val key = BackendJobDescriptorKey(call, None, 1)
     val runtimeAttributes = makeRuntimeAttributes(call)
-    val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, Map.empty, CallCachingEligible)
+    val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, Map.empty, NoDocker, Map.empty)
 
     val props = Props(new TestableJesJobExecutionActor(jobDescriptor, Promise(), jesConfiguration))
     val testActorRef = TestActorRef[TestableJesJobExecutionActor](
@@ -646,16 +681,16 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
     val workflowDescriptor = BackendWorkflowDescriptor(
       WorkflowId(UUID.fromString("e6236763-c518-41d0-9688-432549a8bf7d")),
       WdlNamespaceWithWorkflow.load(
-        new SampleWdl.ScatterWdl().asWorkflowSources(""" runtime {docker: "ubuntu:latest"} """).wdlSource, Seq.empty[ImportResolver]).get.workflow,
+        new SampleWdl.ScatterWdl().asWorkflowSources(""" runtime {docker: "ubuntu:latest"} """).workflowSource, Seq.empty[ImportResolver]).get.workflow,
       Map.empty,
       WorkflowOptions.fromJsonString(""" {"jes_gcs_root": "gs://path/to/gcs_root"} """).get,
       Labels.empty
     )
 
-    val call = workflowDescriptor.workflow.findCallByName("B").get.asInstanceOf[TaskCall]
+    val call = workflowDescriptor.workflow.findCallByName("B").get.asInstanceOf[WdlTaskCall]
     val key = BackendJobDescriptorKey(call, Option(2), 1)
     val runtimeAttributes = makeRuntimeAttributes(call)
-    val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, Map.empty, CallCachingEligible)
+    val jobDescriptor = BackendJobDescriptor(workflowDescriptor, key, runtimeAttributes, Map.empty, NoDocker, Map.empty)
 
     val props = Props(new TestableJesJobExecutionActor(jobDescriptor, Promise(), jesConfiguration))
     val testActorRef = TestActorRef[TestableJesJobExecutionActor](
@@ -697,7 +732,7 @@ class JesAsyncBackendJobExecutionActorSpec extends TestKitSuite("JesAsyncBackend
     descriptorWithMax2AndKey2.preemptible shouldBe true
   }
 
-  private def makeRuntimeAttributes(job: TaskCall) = {
+  private def makeRuntimeAttributes(job: WdlTaskCall) = {
     val evaluatedAttributes = RuntimeAttributeDefinition.evaluateRuntimeAttributes(job.task.runtimeAttributes, TestableJesExpressionFunctions, Map.empty)
     RuntimeAttributeDefinition.addDefaultsToAttributes(
       runtimeAttributesBuilder.definitions.toSet, NoOptions)(evaluatedAttributes.get)
diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAttributesSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAttributesSpec.scala
index 5d9a5724e..695305781 100644
--- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAttributesSpec.scala
+++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAttributesSpec.scala
@@ -14,8 +14,11 @@ class JesAttributesSpec extends FlatSpec with Matchers {
 
   behavior of "JesAttributes"
 
+  val googleConfig = GoogleConfiguration(JesGlobalConfig)
+  val runtimeConfig = ConfigFactory.load()
+
   it should "parse correct JES config" taggedAs IntegrationTest in {
-    val googleConfig = GoogleConfiguration(JesGlobalConfig)
+
     val backendConfig = ConfigFactory.parseString(configString())
 
     val jesAttributes = JesAttributes(googleConfig, backendConfig)
@@ -24,11 +27,10 @@ class JesAttributesSpec extends FlatSpec with Matchers {
     jesAttributes.executionBucket should be("gs://myBucket")
     jesAttributes.maxPollingInterval should be(600)
     jesAttributes.computeServiceAccount should be("default")
-    jesAttributes.defaultZones.toList should be (List("us-central1-a"))
+    jesAttributes.restrictMetadataAccess should be(false)
   }
 
   it should "parse correct preemptible config" taggedAs IntegrationTest in {
-    val googleConfig = GoogleConfiguration(JesGlobalConfig)
     val backendConfig = ConfigFactory.parseString(configString(preemptible = "preemptible = 3"))
 
     val jesAttributes = JesAttributes(googleConfig, backendConfig)
@@ -36,17 +38,23 @@ class JesAttributesSpec extends FlatSpec with Matchers {
     jesAttributes.project should be("myProject")
     jesAttributes.executionBucket should be("gs://myBucket")
     jesAttributes.maxPollingInterval should be(600)
-    jesAttributes.defaultZones.toList should be (List("us-central1-a"))
   }
 
   it should "parse compute service account" taggedAs IntegrationTest in {
-    val googleConfig = GoogleConfiguration(JesGlobalConfig)
     val backendConfig = ConfigFactory.parseString(configString(genomics = """compute-service-account = "testing" """))
 
     val jesAttributes = JesAttributes(googleConfig, backendConfig)
     jesAttributes.computeServiceAccount should be("testing")
   }
 
+  it should "parse restrict-metadata-access" taggedAs IntegrationTest in {
+    val backendConfig = ConfigFactory.parseString(configString(genomics = "restrict-metadata-access = true"))
+
+    val jesAttributes = JesAttributes(googleConfig, backendConfig)
+    jesAttributes.restrictMetadataAccess should be(true)
+
+  }
+
   it should "not parse invalid config" taggedAs IntegrationTest in {
     val nakedConfig =
       ConfigFactory.parseString(
@@ -54,13 +62,10 @@ class JesAttributesSpec extends FlatSpec with Matchers {
           |{
           |   genomics {
           |     endpoint-url = "myEndpoint"
-          |     default-zones = []
           |   }
           |}
         """.stripMargin)
 
-    val googleConfig = GoogleConfiguration(JesGlobalConfig)
-
     val exception = intercept[IllegalArgumentException with MessageAggregation] {
       JesAttributes(googleConfig, nakedConfig)
     }
@@ -70,7 +75,6 @@ class JesAttributesSpec extends FlatSpec with Matchers {
     errorsList should contain("No configuration setting found for key 'genomics.auth'")
     errorsList should contain("No configuration setting found for key 'filesystems'")
     errorsList should contain("URI is not absolute")
-    errorsList should contain("genomics.default-zones was set but no values were provided")
   }
 
   def configString(preemptible: String = "", genomics: String = ""): String =
@@ -86,7 +90,6 @@ class JesAttributesSpec extends FlatSpec with Matchers {
       |     auth = "application-default"
       |    $genomics
       |     endpoint-url = "http://myEndpoint"
-      |     default-zones = ["us-central1-a"]
       |   }
       |
       |   filesystems = {
diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesCallPathsSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesCallPathsSpec.scala
index b8859d6cb..73a1cf85f 100644
--- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesCallPathsSpec.scala
+++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesCallPathsSpec.scala
@@ -1,5 +1,6 @@
 package cromwell.backend.impl.jes
 
+import com.google.cloud.NoCredentials
 import cromwell.backend.BackendSpec
 import cromwell.core.TestKitSuite
 import cromwell.filesystems.gcs.auth.GoogleAuthModeSpec
@@ -17,12 +18,13 @@ class JesCallPathsSpec extends TestKitSuite with FlatSpecLike with Matchers with
   it should "map the correct filenames" in {
     GoogleAuthModeSpec.assumeHasApplicationDefaultCredentials()
 
-    val workflowDescriptor = buildWorkflowDescriptor(SampleWdl.HelloWorld.wdlSource())
+    val workflowDescriptor = buildWorkflowDescriptor(SampleWdl.HelloWorld.workflowSource())
     val jobDescriptorKey = firstJobDescriptorKey(workflowDescriptor)
     val jesConfiguration = new JesConfiguration(JesBackendConfigurationDescriptor)
-
-    val callPaths = JesJobPaths(jobDescriptorKey, workflowDescriptor,
-      jesConfiguration)
+    val workflowPaths = JesWorkflowPaths(workflowDescriptor, NoCredentials.getInstance(), NoCredentials.getInstance(), jesConfiguration)
+    
+    val callPaths = JesJobPaths(workflowPaths, jobDescriptorKey)
+    
     callPaths.returnCodeFilename should be("hello-rc.txt")
     callPaths.stderrFilename should be("hello-stderr.log")
     callPaths.stdoutFilename should be("hello-stdout.log")
@@ -32,11 +34,13 @@ class JesCallPathsSpec extends TestKitSuite with FlatSpecLike with Matchers with
   it should "map the correct paths" in {
     GoogleAuthModeSpec.assumeHasApplicationDefaultCredentials()
 
-    val workflowDescriptor = buildWorkflowDescriptor(SampleWdl.HelloWorld.wdlSource())
+    val workflowDescriptor = buildWorkflowDescriptor(SampleWdl.HelloWorld.workflowSource())
     val jobDescriptorKey = firstJobDescriptorKey(workflowDescriptor)
     val jesConfiguration = new JesConfiguration(JesBackendConfigurationDescriptor)
+    val workflowPaths = JesWorkflowPaths(workflowDescriptor, NoCredentials.getInstance(), NoCredentials.getInstance(), jesConfiguration)
 
-    val callPaths = JesJobPaths(jobDescriptorKey, workflowDescriptor, jesConfiguration)
+    val callPaths = JesJobPaths(workflowPaths, jobDescriptorKey)
+    
     callPaths.returnCode.pathAsString should
       be(s"gs://my-cromwell-workflows-bucket/wf_hello/${workflowDescriptor.id}/call-hello/hello-rc.txt")
     callPaths.stdout.pathAsString should
@@ -50,11 +54,13 @@ class JesCallPathsSpec extends TestKitSuite with FlatSpecLike with Matchers with
   it should "map the correct call context" in {
     GoogleAuthModeSpec.assumeHasApplicationDefaultCredentials()
 
-    val workflowDescriptor = buildWorkflowDescriptor(SampleWdl.HelloWorld.wdlSource())
+    val workflowDescriptor = buildWorkflowDescriptor(SampleWdl.HelloWorld.workflowSource())
     val jobDescriptorKey = firstJobDescriptorKey(workflowDescriptor)
     val jesConfiguration = new JesConfiguration(JesBackendConfigurationDescriptor)
+    val workflowPaths = JesWorkflowPaths(workflowDescriptor, NoCredentials.getInstance(), NoCredentials.getInstance(), jesConfiguration)
 
-    val callPaths = JesJobPaths(jobDescriptorKey, workflowDescriptor, jesConfiguration)
+    val callPaths = JesJobPaths(workflowPaths, jobDescriptorKey)
+    
     callPaths.callContext.root.pathAsString should
       be(s"gs://my-cromwell-workflows-bucket/wf_hello/${workflowDescriptor.id}/call-hello")
     callPaths.callContext.stdout should
diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesConfigurationSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesConfigurationSpec.scala
index 23b029086..2ead86d6c 100644
--- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesConfigurationSpec.scala
+++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesConfigurationSpec.scala
@@ -42,6 +42,7 @@ class JesConfigurationSpec extends FlatSpec with Matchers with TableDrivenProper
       |    }
       |  ]
       |}
+      |
     """.stripMargin)
 
   val backendConfig = ConfigFactory.parseString(
@@ -62,7 +63,18 @@ class JesConfigurationSpec extends FlatSpec with Matchers with TableDrivenProper
       |     auth = "application-default"
       |     // Endpoint for APIs, no reason to change this unless directed by Google.
       |     endpoint-url = "https://genomics.googleapis.com/"
-      |     default-zones = ["us-central1-a", "us-central1-b"]
+      |  }
+      |
+      |  default-runtime-attributes {
+      |      failOnStderr: false
+      |      continueOnReturnCode: 0
+      |      cpu: 1
+      |      memory: "2 GB"
+      |      bootDiskSizeGb: 10
+      |      disks: "local-disk 10 SSD"
+      |      noAddress: false
+      |      preemptible: 3
+      |      zones:["us-central1-a", "us-central1-b"]
       |  }
       |
       |  dockerhub {
@@ -76,6 +88,7 @@ class JesConfigurationSpec extends FlatSpec with Matchers with TableDrivenProper
       |      auth = "application-default"
       |    }
       |  }
+      |
     """.stripMargin)
 
   it should "fail to instantiate if any required configuration is missing" in {
@@ -103,10 +116,6 @@ class JesConfigurationSpec extends FlatSpec with Matchers with TableDrivenProper
     new JesConfiguration(BackendConfigurationDescriptor(backendConfig, globalConfig)).root shouldBe "gs://my-cromwell-workflows-bucket"
   }
 
-  it should "have the correct default zones" in {
-    new JesConfiguration(BackendConfigurationDescriptor(backendConfig, globalConfig)).defaultZones.toList shouldBe List("us-central1-a", "us-central1-b")
-  }
-
   it should "have correct docker" in {
     val dockerConf = new JesConfiguration(BackendConfigurationDescriptor(backendConfig, globalConfig)).dockerCredentials
     dockerConf shouldBe defined
diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala
index 2fa9f6bd2..121d4603f 100644
--- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala
+++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala
@@ -7,7 +7,7 @@ import akka.testkit._
 import com.typesafe.config.{Config, ConfigFactory}
 import cromwell.backend.BackendWorkflowInitializationActor.{InitializationFailed, InitializationSuccess, Initialize}
 import cromwell.backend.async.RuntimeAttributeValidationFailures
-import cromwell.backend.impl.jes.authentication.GcsLocalizing
+import cromwell.backend.impl.jes.authentication.{GcsLocalizing, JesAuthObject}
 import cromwell.backend.{BackendConfigurationDescriptor, BackendSpec, BackendWorkflowDescriptor}
 import cromwell.core.Dispatcher.BackendDispatcher
 import cromwell.core.Tags.IntegrationTest
@@ -19,7 +19,7 @@ import cromwell.util.{EncryptionSpec, SampleWdl}
 import org.scalatest.{FlatSpecLike, Matchers}
 import org.specs2.mock.Mockito
 import spray.json._
-import wdl4s.TaskCall
+import wdl4s.wdl.WdlTaskCall
 
 import scala.concurrent.duration._
 
@@ -28,6 +28,7 @@ class JesInitializationActorSpec extends TestKitSuite("JesInitializationActorSpe
   val Timeout: FiniteDuration = 5.second.dilated
 
   import BackendSpec._
+  import JesInitializationActorSpec._
 
   val HelloWorld: String =
     s"""
@@ -67,7 +68,7 @@ class JesInitializationActorSpec extends TestKitSuite("JesInitializationActorSpe
       |    }
       |  ]
       |}
-      | """.stripMargin)
+      |""".stripMargin)
 
   val backendConfigTemplate: String =
     """
@@ -89,6 +90,19 @@ class JesInitializationActorSpec extends TestKitSuite("JesInitializationActorSpe
       |     endpoint-url = "https://genomics.googleapis.com/"
       |  }
       |
+      |  default-runtime-attributes {
+      |     cpu: 1
+      |     failOnStderr: false
+      |     # Allowed to be a boolean, or a list of Ints, or an Int
+      |     continueOnReturnCode: 0
+      |     memory: "2 GB"
+      |     bootDiskSizeGb: 10
+      |     # Allowed to be a String, or a list of Strings
+      |     disks: "local-disk 10 SSD"
+      |     noAddress: false
+      |     preemptible: 0
+      |     zones: ["us-central1-a", "us-central1-b"]
+      |  }
       |  filesystems {
       |    gcs {
       |      // A reference to a potentially different auth for manipulating files via engine functions.
@@ -119,6 +133,20 @@ class JesInitializationActorSpec extends TestKitSuite("JesInitializationActorSpe
       |     endpoint-url = "https://genomics.googleapis.com/"
       |  }
       |
+      |  default-runtime-attributes {
+      |     cpu: 1
+      |     failOnStderr: false
+      |     # Allowed to be a boolean, or a list of Ints, or an Int
+      |     continueOnReturnCode: 0
+      |     memory: "2 GB"
+      |     bootDiskSizeGb: 10
+      |     # Allowed to be a String, or a list of Strings
+      |     disks: "local-disk 10 SSD"
+      |     noAddress: false
+      |     preemptible: 0
+      |     zones: ["us-central1-a", "us-central1-b"]
+      |  }
+      |
       |  filesystems {
       |    gcs {
       |      // A reference to a potentially different auth for manipulating files via engine functions.
@@ -142,13 +170,14 @@ class JesInitializationActorSpec extends TestKitSuite("JesInitializationActorSpe
   val refreshTokenConfig: Config = ConfigFactory.parseString(refreshTokenConfigTemplate)
 
   private def getJesBackendProps(workflowDescriptor: BackendWorkflowDescriptor,
-                                 calls: Set[TaskCall],
+                                 calls: Set[WdlTaskCall],
                                  jesConfiguration: JesConfiguration): Props = {
-    val params = JesInitializationActorParams(workflowDescriptor, calls, jesConfiguration, emptyActor)
+    val ioActor = mockIoActor
+    val params = JesInitializationActorParams(workflowDescriptor, ioActor, calls, jesConfiguration, emptyActor, restarting = false)
     Props(new JesInitializationActor(params)).withDispatcher(BackendDispatcher)
   }
 
-  private def getJesBackend(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[TaskCall], conf: BackendConfigurationDescriptor) = {
+  private def getJesBackend(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[WdlTaskCall], conf: BackendConfigurationDescriptor) = {
     val props = getJesBackendProps(workflowDescriptor, calls, new JesConfiguration(conf))
     system.actorOf(props, "TestableJesInitializationActor-" + UUID.randomUUID)
   }
@@ -196,7 +225,7 @@ class JesInitializationActorSpec extends TestKitSuite("JesInitializationActorSpe
 
   private def buildJesInitializationTestingBits(backendConfig: Config = dockerBackendConfig): TestingBits = {
     val workflowOptions = WorkflowOptions.fromMap(Map("refresh_token" -> "mytoken")).get
-    val workflowDescriptor = buildWorkflowDescriptor(SampleWdl.HelloWorld.wdlSource(), options = workflowOptions)
+    val workflowDescriptor = buildWorkflowDescriptor(SampleWdl.HelloWorld.workflowSource(), options = workflowOptions)
     val calls = workflowDescriptor.workflow.taskCalls
     val backendConfigurationDescriptor = BackendConfigurationDescriptor(backendConfig, globalConfig)
     val jesConfiguration = new JesConfiguration(backendConfigurationDescriptor)
@@ -221,9 +250,9 @@ class JesInitializationActorSpec extends TestKitSuite("JesInitializationActorSpe
     val TestingBits(actorRef, _) = buildJesInitializationTestingBits()
     val actor = actorRef.underlyingActor
 
-    actor.generateAuthJson(None, None) should be(empty)
+    actor.generateAuthJson(flattenAuthOptions(None, None), false) should be(empty)
 
-    val authJsonOption = actor.generateAuthJson(None, None)
+    val authJsonOption = actor.generateAuthJson(flattenAuthOptions(None, None), false)
     authJsonOption should be(empty)
 
     actorRef.stop()
@@ -235,7 +264,7 @@ class JesInitializationActorSpec extends TestKitSuite("JesInitializationActorSpe
     val TestingBits(actorRef, jesConfiguration) = buildJesInitializationTestingBits()
     val actor = actorRef.underlyingActor
 
-    val authJsonOption = actor.generateAuthJson(jesConfiguration.dockerCredentials, None)
+    val authJsonOption = actor.generateAuthJson(flattenAuthOptions(jesConfiguration.dockerCredentials, None), false)
     authJsonOption shouldNot be(empty)
     authJsonOption.get should be(
       normalize(
@@ -261,7 +290,7 @@ class JesInitializationActorSpec extends TestKitSuite("JesInitializationActorSpe
     val actor = actorRef.underlyingActor
 
     val gcsUserAuth = Option(GcsLocalizing(SimpleClientSecrets("myclientid", "myclientsecret"), "mytoken"))
-    val authJsonOption = actor.generateAuthJson(None, gcsUserAuth)
+    val authJsonOption = actor.generateAuthJson(flattenAuthOptions(None, gcsUserAuth), false)
     authJsonOption shouldNot be(empty)
     authJsonOption.get should be(
       normalize(
@@ -288,7 +317,7 @@ class JesInitializationActorSpec extends TestKitSuite("JesInitializationActorSpe
     val actor = actorRef.underlyingActor
 
     val gcsUserAuth = Option(GcsLocalizing(SimpleClientSecrets("myclientid", "myclientsecret"), "mytoken"))
-    val authJsonOption = actor.generateAuthJson(jesConfiguration.dockerCredentials, gcsUserAuth)
+    val authJsonOption = actor.generateAuthJson(flattenAuthOptions(jesConfiguration.dockerCredentials, gcsUserAuth), false)
     authJsonOption shouldNot be(empty)
     authJsonOption.get should be(
       normalize(
@@ -312,7 +341,65 @@ class JesInitializationActorSpec extends TestKitSuite("JesInitializationActorSpe
     actorRef.stop()
   }
 
-  private def normalize(str: String) = {
+  it should "generate the correct json content for a docker token, a refresh token, and restrictMetadataAccess" in {
+    EncryptionSpec.assumeAes256Cbc()
+
+    val TestingBits(actorRef, jesConfiguration) = buildJesInitializationTestingBits()
+    val actor = actorRef.underlyingActor
+
+    val gcsUserAuth = Option(GcsLocalizing(SimpleClientSecrets("myclientid", "myclientsecret"), "mytoken"))
+    val authJsonOption = actor.generateAuthJson(flattenAuthOptions(jesConfiguration.dockerCredentials, gcsUserAuth), true)
+    authJsonOption shouldNot be(empty)
+    authJsonOption.get should be(
+      normalize(
+        """
+          |{
+          |    "auths": {
+          |        "docker": {
+          |            "account": "my@docker.account",
+          |            "token": "mydockertoken"
+          |        },
+          |        "boto": {
+          |            "client_id": "myclientid",
+          |            "client_secret": "myclientsecret",
+          |            "refresh_token": "mytoken"
+          |        }
+          |    },
+          |    "restrictMetadataAccess": true
+          |}
+        """.stripMargin)
+    )
+
+    actorRef.stop()
+  }
+
+  it should "generate the correct json content for just restrictMetadataAccess" in {
+    EncryptionSpec.assumeAes256Cbc()
+
+    val TestingBits(actorRef, _) = buildJesInitializationTestingBits()
+    val actor = actorRef.underlyingActor
+
+    val authJsonOption = actor.generateAuthJson(flattenAuthOptions(None, None), true)
+    authJsonOption shouldNot be(empty)
+    authJsonOption.get should be(
+      normalize(
+        """
+          |{
+          |    "restrictMetadataAccess": true
+          |}
+        """.stripMargin)
+    )
+
+    actorRef.stop()
+  }
+}
+
+object JesInitializationActorSpec {
+  def normalize(str: String) = {
     str.parseJson.prettyPrint
   }
+
+  def flattenAuthOptions(options: Option[JesAuthObject]*): List[JesAuthObject] = {
+    options.toList.flatten
+  }
 }
diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesJobExecutionActorSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesJobExecutionActorSpec.scala
index 5b72db08b..2ff644e47 100644
--- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesJobExecutionActorSpec.scala
+++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesJobExecutionActorSpec.scala
@@ -12,6 +12,7 @@ import org.specs2.mock.Mockito
 
 import scala.concurrent.duration._
 import scala.concurrent.{ExecutionContext, Promise}
+import scala.util.{Failure, Success}
 
 class JesJobExecutionActorSpec extends TestKitSuite("JesJobExecutionActorSpec") with FlatSpecLike with Matchers with Mockito {
 
@@ -26,13 +27,14 @@ class JesJobExecutionActorSpec extends TestKitSuite("JesJobExecutionActorSpec")
     val jesWorkflowInfo = mock[JesConfiguration]
     val initializationData = mock[JesBackendInitializationData]
     val serviceRegistryActor = system.actorOf(Props.empty)
+    val ioActor = system.actorOf(Props.empty)
     val jesBackendSingletonActor = Option(system.actorOf(Props.empty))
 
     initializationData.jesConfiguration returns jesWorkflowInfo
 
     val parent = TestProbe()
     val deathwatch = TestProbe()
-    val params = DefaultStandardSyncExecutionActorParams(JesAsyncBackendJobExecutionActor.JesOperationIdKey, serviceRegistryActor,
+    val params = DefaultStandardSyncExecutionActorParams(JesAsyncBackendJobExecutionActor.JesOperationIdKey, serviceRegistryActor, ioActor,
       jobDescriptor, null, Option(initializationData), jesBackendSingletonActor,
       classOf[JesAsyncBackendJobExecutionActor])
     val testJJEA = TestActorRef[TestJesJobExecutionActor](
@@ -57,6 +59,7 @@ class JesJobExecutionActorSpec extends TestKitSuite("JesJobExecutionActorSpec")
     val jesWorkflowInfo = mock[JesConfiguration]
     val initializationData = mock[JesBackendInitializationData]
     val serviceRegistryActor = system.actorOf(Props.empty)
+    val ioActor = system.actorOf(Props.empty)
     val jesBackendSingletonActor = Option(system.actorOf(Props.empty))
 
     initializationData.jesConfiguration returns jesWorkflowInfo
@@ -64,7 +67,7 @@ class JesJobExecutionActorSpec extends TestKitSuite("JesJobExecutionActorSpec")
     val parent = TestProbe()
     val deathwatch = TestProbe()
     val jabjeaConstructionPromise = Promise[ActorRef]()
-    val params = DefaultStandardSyncExecutionActorParams(JesAsyncBackendJobExecutionActor.JesOperationIdKey, serviceRegistryActor,
+    val params = DefaultStandardSyncExecutionActorParams(JesAsyncBackendJobExecutionActor.JesOperationIdKey, serviceRegistryActor, ioActor,
       jobDescriptor, null, Option(initializationData), jesBackendSingletonActor,
       classOf[JesAsyncBackendJobExecutionActor])
     val testJJEA = TestActorRef[TestJesJobExecutionActor](
@@ -81,7 +84,14 @@ class JesJobExecutionActorSpec extends TestKitSuite("JesJobExecutionActorSpec")
     // Wait for the JABJEA to be spawned. Then kill it:
     parent.expectNoMsg(max = AwaitAlmostNothing)
     deathwatch.expectNoMsg(max = AwaitAlmostNothing)
-    jabjeaConstructionPromise.future foreach { _ ! JabjeaExplode }
+    jabjeaConstructionPromise.future onComplete {
+      case Success(jabjea) =>
+        jabjea ! JabjeaExplode
+      case Failure(throwable) =>
+        val exception = new RuntimeException("Error creating jabjea for test!", throwable)
+        exception.printStackTrace()
+        throw exception
+    }
 
     parent.expectMsgPF(max = TimeoutDuration) {
       case JobFailedNonRetryableResponse(_, throwable, _) =>
diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesRuntimeAttributesSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesRuntimeAttributesSpec.scala
index 23ab89ed1..1de774fa6 100644
--- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesRuntimeAttributesSpec.scala
+++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesRuntimeAttributesSpec.scala
@@ -2,7 +2,7 @@ package cromwell.backend.impl.jes
 
 import cats.data.NonEmptyList
 import cromwell.backend.impl.jes.io.{DiskType, JesAttachedDisk, JesWorkingDisk}
-import cromwell.backend.validation.ContinueOnReturnCodeSet
+import cromwell.backend.validation.{ContinueOnReturnCodeFlag, ContinueOnReturnCodeSet}
 import cromwell.backend.{MemorySize, RuntimeAttributeDefinition}
 import cromwell.core.WorkflowOptions
 import org.scalatest.{Matchers, WordSpecLike}
@@ -10,8 +10,8 @@ import org.slf4j.helpers.NOPLogger
 import org.specs2.mock.Mockito
 import spray.json._
 import wdl4s.parser.MemoryUnit
-import wdl4s.types.{WdlArrayType, WdlIntegerType, WdlStringType}
-import wdl4s.values.{WdlArray, WdlBoolean, WdlInteger, WdlString, WdlValue}
+import wdl4s.wdl.types.{WdlArrayType, WdlIntegerType, WdlStringType}
+import wdl4s.wdl.values.{WdlArray, WdlBoolean, WdlInteger, WdlString, WdlValue}
 
 class JesRuntimeAttributesSpec extends WordSpecLike with Matchers with Mockito {
 
@@ -22,7 +22,7 @@ class JesRuntimeAttributesSpec extends WordSpecLike with Matchers with Mockito {
   }
 
   val expectedDefaults = new JesRuntimeAttributes(1, Vector("us-central1-b", "us-central1-a"), 0, 10,
-    MemorySize(2, MemoryUnit.GB), Seq(JesWorkingDisk(DiskType.SSD, 10)), "ubuntu:latest", false,
+    MemorySize(2, MemoryUnit.GB), Vector(JesWorkingDisk(DiskType.SSD, 10)), "ubuntu:latest", false,
     ContinueOnReturnCodeSet(Set(0)), false)
 
   "JesRuntimeAttributes" should {
@@ -32,6 +32,12 @@ class JesRuntimeAttributesSpec extends WordSpecLike with Matchers with Mockito {
       assertJesRuntimeAttributesFailedCreation(runtimeAttributes, "Can't find an attribute value for key docker")
     }
 
+    "use hardcoded defaults if not declared in task, workflow options, or config (except for docker)" in {
+      val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest"))
+      val expectedRuntimeAttributes = expectedDefaults
+      assertJesRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes, jesConfiguration = noDefaultsJesConfiguration)
+    }
+
     "validate a valid Docker entry" in {
       val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest"))
       val expectedRuntimeAttributes = expectedDefaults
@@ -54,12 +60,18 @@ class JesRuntimeAttributesSpec extends WordSpecLike with Matchers with Mockito {
       assertJesRuntimeAttributesFailedCreation(runtimeAttributes, "Expecting failOnStderr runtime attribute to be a Boolean or a String with values of 'true' or 'false'")
     }
 
-    "validate a valid continueOnReturnCode entry" in {
+    "validate a valid continueOnReturnCode integer entry" in {
       val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest"), "continueOnReturnCode" -> WdlInteger(1))
       val expectedRuntimeAttributes = expectedDefaults.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1)))
       assertJesRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes)
     }
 
+    "validate a valid continueOnReturnCode boolean entry" in {
+      val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest"), "continueOnReturnCode" -> WdlBoolean(false))
+      val expectedRuntimeAttributes = expectedDefaults.copy(continueOnReturnCode = ContinueOnReturnCodeFlag(false))
+      assertJesRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes)
+    }
+
     "validate a valid continueOnReturnCode array entry" in {
       val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest"), "continueOnReturnCode" -> WdlArray(WdlArrayType(WdlIntegerType), Array(WdlInteger(1), WdlInteger(2))))
       val expectedRuntimeAttributes = expectedDefaults.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1, 2)))
@@ -184,17 +196,54 @@ class JesRuntimeAttributesSpec extends WordSpecLike with Matchers with Mockito {
         "Expecting noAddress runtime attribute to be a Boolean")
     }
 
-    "use reasonable default values" in {
+    "override config default attributes with default attributes declared in workflow options" in {
       val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest"))
-      val expectedRuntimeAttributes = expectedDefaults
-      assertJesRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes)
+
+      val workflowOptionsJson =
+        """{
+          |  "default_runtime_attributes": { "cpu": 2 }
+          |}
+        """.stripMargin.parseJson.asInstanceOf[JsObject]
+
+      val workflowOptions = WorkflowOptions.fromJsonObject(workflowOptionsJson).get
+      val expectedRuntimeAttributes = expectedDefaults.copy(cpu = 2)
+      assertJesRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes, workflowOptions)
+    }
+
+    "override config default runtime attributes with task runtime attributes" in {
+      val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest"), "cpu" -> WdlInteger(4))
+
+      val workflowOptionsJson =
+        """{
+          |  "default_runtime_attributes": { "cpu": 2 }
+          |}
+        """.stripMargin.parseJson.asInstanceOf[JsObject]
+
+      val workflowOptions = WorkflowOptions.fromJsonObject(workflowOptionsJson).get
+      val expectedRuntimeAttributes = expectedDefaults.copy(cpu = 4)
+      assertJesRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes, workflowOptions)
+    }
+
+    "override invalid config default attributes with task runtime attributes" in {
+      val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest"), "cpu" -> WdlInteger(4))
+
+      val workflowOptionsJson =
+        """{
+          |  "default_runtime_attributes": { "cpu": 2.2 }
+          |}
+        """.stripMargin.parseJson.asInstanceOf[JsObject]
+
+      val workflowOptions = WorkflowOptions.fromJsonObject(workflowOptionsJson).get
+      val expectedRuntimeAttributes = expectedDefaults.copy(cpu = 4)
+      assertJesRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes, workflowOptions)
     }
   }
 
   private def assertJesRuntimeAttributesSuccessfulCreation(runtimeAttributes: Map[String, WdlValue],
                                                            expectedRuntimeAttributes: JesRuntimeAttributes,
                                                            workflowOptions: WorkflowOptions = emptyWorkflowOptions,
-                                                           defaultZones: NonEmptyList[String] = defaultZones): Unit = {
+                                                           defaultZones: NonEmptyList[String] = defaultZones,
+                                                           jesConfiguration: JesConfiguration = jesConfiguration): Unit = {
     try {
       val actualRuntimeAttributes = toJesRuntimeAttributes(runtimeAttributes, workflowOptions, jesConfiguration)
       assert(actualRuntimeAttributes == expectedRuntimeAttributes)
@@ -204,7 +253,9 @@ class JesRuntimeAttributesSpec extends WordSpecLike with Matchers with Mockito {
     ()
   }
 
-  private def assertJesRuntimeAttributesFailedCreation(runtimeAttributes: Map[String, WdlValue], exMsg: String, workflowOptions: WorkflowOptions = emptyWorkflowOptions): Unit = {
+  private def assertJesRuntimeAttributesFailedCreation(runtimeAttributes: Map[String, WdlValue],
+                                                       exMsg: String,
+                                                       workflowOptions: WorkflowOptions = emptyWorkflowOptions): Unit = {
     try {
       toJesRuntimeAttributes(runtimeAttributes, workflowOptions, jesConfiguration)
       fail(s"A RuntimeException was expected with message: $exMsg")
@@ -221,16 +272,13 @@ class JesRuntimeAttributesSpec extends WordSpecLike with Matchers with Mockito {
     val defaultedAttributes = RuntimeAttributeDefinition.addDefaultsToAttributes(
       staticRuntimeAttributeDefinitions, workflowOptions)(runtimeAttributes)
     val validatedRuntimeAttributes = runtimeAttributesBuilder.build(defaultedAttributes, NOPLogger.NOP_LOGGER)
-    JesRuntimeAttributes(validatedRuntimeAttributes)
+    JesRuntimeAttributes(validatedRuntimeAttributes, jesConfiguration.runtimeConfig)
   }
 
   private val emptyWorkflowOptions = WorkflowOptions.fromMap(Map.empty).get
   private val defaultZones = NonEmptyList.of("us-central1-b", "us-central1-a")
-  private val jesConfiguration = {
-    val config = mock[JesConfiguration]
-    config.defaultZones returns defaultZones
-    config
-  }
+  private val jesConfiguration = new JesConfiguration(JesTestConfig.JesBackendConfigurationDescriptor)
+  private val noDefaultsJesConfiguration = new JesConfiguration(JesTestConfig.NoDefaultsConfigurationDescriptor)
   private val staticRuntimeAttributeDefinitions: Set[RuntimeAttributeDefinition] =
     JesRuntimeAttributes.runtimeAttributesBuilder(jesConfiguration).definitions.toSet
 }
diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesTestConfig.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesTestConfig.scala
index 25f061387..3423a8b71 100644
--- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesTestConfig.scala
+++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesTestConfig.scala
@@ -4,6 +4,7 @@ import com.typesafe.config.ConfigFactory
 import cromwell.backend.BackendConfigurationDescriptor
 
 object JesTestConfig {
+
   private val JesBackendConfigString =
     """
       |project = "my-cromwell-workflows"
@@ -19,6 +20,37 @@ object JesTestConfig {
       |    auth = "application-default"
       |  }
       |}
+      |
+      |default-runtime-attributes {
+      |    cpu: 1
+      |    failOnStderr: false
+      |    continueOnReturnCode: 0
+      |    docker: "ubuntu:latest"
+      |    memory: "2 GB"
+      |    bootDiskSizeGb: 10
+      |    disks: "local-disk 10 SSD"
+      |    noAddress: false
+      |    preemptible: 0
+      |    zones:["us-central1-b", "us-central1-a"]
+      |}
+      |
+      |""".stripMargin
+
+  private val NoDefaultsConfigString =
+    """
+      |project = "my-cromwell-workflows"
+      |root = "gs://my-cromwell-workflows-bucket"
+      |
+      |genomics {
+      |  auth = "application-default"
+      |  endpoint-url = "https://genomics.googleapis.com/"
+      |}
+      |
+      |filesystems {
+      |  gcs {
+      |    auth = "application-default"
+      |  }
+      |}
       |""".stripMargin
 
   private val JesGlobalConfigString =
@@ -44,9 +76,12 @@ object JesTestConfig {
        |    }
        |  }
        |}
+       |
        |""".stripMargin
 
   val JesBackendConfig = ConfigFactory.parseString(JesBackendConfigString)
   val JesGlobalConfig = ConfigFactory.parseString(JesGlobalConfigString)
+  val JesBackendNoDefaultConfig = ConfigFactory.parseString(NoDefaultsConfigString)
   val JesBackendConfigurationDescriptor = BackendConfigurationDescriptor(JesBackendConfig, JesGlobalConfig)
+  val NoDefaultsConfigurationDescriptor = BackendConfigurationDescriptor(JesBackendNoDefaultConfig, JesGlobalConfig)
 }
diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesWorkflowPathsSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesWorkflowPathsSpec.scala
index 8cc12fd4b..e8faa6951 100644
--- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesWorkflowPathsSpec.scala
+++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesWorkflowPathsSpec.scala
@@ -1,5 +1,6 @@
 package cromwell.backend.impl.jes
 
+import com.google.cloud.NoCredentials
 import cromwell.backend.BackendSpec
 import cromwell.core.TestKitSuite
 import cromwell.filesystems.gcs.auth.GoogleAuthModeSpec
@@ -16,10 +17,10 @@ class JesWorkflowPathsSpec extends TestKitSuite with FlatSpecLike with Matchers
   it should "map the correct paths" in {
     GoogleAuthModeSpec.assumeHasApplicationDefaultCredentials()
 
-    val workflowDescriptor = buildWorkflowDescriptor(SampleWdl.HelloWorld.wdlSource())
+    val workflowDescriptor = buildWorkflowDescriptor(SampleWdl.HelloWorld.workflowSource())
     val jesConfiguration = new JesConfiguration(JesBackendConfigurationDescriptor)
 
-    val workflowPaths = JesWorkflowPaths(workflowDescriptor, jesConfiguration)(system)
+    val workflowPaths = JesWorkflowPaths(workflowDescriptor, NoCredentials.getInstance(), NoCredentials.getInstance(), jesConfiguration)(system)
     workflowPaths.executionRoot.pathAsString should be("gs://my-cromwell-workflows-bucket/")
     workflowPaths.workflowRoot.pathAsString should
       be(s"gs://my-cromwell-workflows-bucket/wf_hello/${workflowDescriptor.id}/")
diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActorSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActorSpec.scala
index 2b13ea4bc..c04564fc0 100644
--- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActorSpec.scala
+++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/statuspolling/JesPollingActorSpec.scala
@@ -64,7 +64,7 @@ class JesPollingActorSpec extends TestKitSuite("JesPollingActor") with FlatSpecL
     jpActor.underlyingActor.callbackResponses :+= CallbackFailure
 
     val successStatus = RunStatus.Success(Seq.empty[ExecutionEvent], None, None, None)
-    val failureStatus = RunStatus.Failed(-1, Option.empty[String], Seq.empty[ExecutionEvent], None, None, None)
+    val failureStatus = RunStatus.UnsuccessfulRunStatus(-1, Option.empty[String], Seq.empty[ExecutionEvent], None, None, None)
     jpActor.underlyingActor.operationStatusResponses :+= successStatus
     jpActor.underlyingActor.operationStatusResponses :+= failureStatus
 
@@ -119,7 +119,7 @@ class TestJesPollingActor(manager: ActorRef, qps: Int Refined Positive) extends
     resultHandlers.zip(callbackResponses) foreach { case (handler, response) => response match {
       case CallbackSuccess => handler.onSuccess(null, null)
       case CallbackFailure =>
-        val error: GoogleJsonError = null
+        val error: GoogleJsonError = new GoogleJsonError()
         handler.onFailure(error, null)
     }}
   }
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigAsyncJobExecutionActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigAsyncJobExecutionActor.scala
index 399532146..8c90e2049 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigAsyncJobExecutionActor.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigAsyncJobExecutionActor.scala
@@ -3,10 +3,11 @@ package cromwell.backend.impl.sfs.config
 import cromwell.backend.impl.sfs.config.ConfigConstants._
 import cromwell.backend.sfs._
 import cromwell.backend.standard.{StandardAsyncExecutionActorParams, StandardAsyncJob}
+import cromwell.backend.validation.DockerValidation
 import cromwell.core.path.Path
-import wdl4s._
-import wdl4s.expression.NoFunctions
-import wdl4s.values.WdlString
+import wdl4s.wdl._
+import wdl4s.wdl.expression.NoFunctions
+import wdl4s.wdl.values.WdlString
 
 /**
   * Base ConfigAsyncJobExecutionActor that reads the config and generates an outer script to submit an inner script
@@ -95,13 +96,21 @@ sealed trait ConfigAsyncJobExecutionActor extends SharedFileSystemAsyncJobExecut
     */
   private lazy val runtimeAttributeInputs: WorkflowCoercedInputs = {
     val declarationValidations = configInitializationData.declarationValidations
-    val inputOptions = declarationValidations map { declarationValidation =>
-      declarationValidation.extractWdlValueOption(validatedRuntimeAttributes) map { wdlValue =>
-        declarationValidation.key -> wdlValue
-      }
+    val inputOptions = declarationValidations map {
+      // Is it always the right thing to pass the Docker hash to a config backend?  What if it can't use hashes?
+      case declarationValidation if declarationValidation.key == DockerValidation.instance.key && jobDescriptor.maybeCallCachingEligible.dockerHash.isDefined =>
+        val dockerHash = jobDescriptor.maybeCallCachingEligible.dockerHash.get
+        Option(declarationValidation.key -> WdlString(dockerHash))
+      case declarationValidation =>
+        declarationValidation.extractWdlValueOption(validatedRuntimeAttributes) map { wdlValue =>
+          declarationValidation.key -> wdlValue
+        }
     }
     inputOptions.flatten.toMap
   }
+
+  // `runtimeAttributeInputs` has already adjusted for the case of a `JobDescriptor` with `DockerWithHash`.
+  override lazy val dockerImageUsed: Option[String] = runtimeAttributeInputs.get(DockerValidation.instance.key).map(_.valueString)
 }
 
 /**
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendFileHashing.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendFileHashing.scala
index afe8011c3..867647988 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendFileHashing.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendFileHashing.scala
@@ -1,7 +1,7 @@
 package cromwell.backend.impl.sfs.config
 
 import akka.event.LoggingAdapter
-import cromwell.backend.callcaching.FileHashingActor.SingleFileHashRequest
+import cromwell.backend.standard.callcaching.StandardFileHashingActor.SingleFileHashRequest
 import cromwell.core.path.DefaultPathBuilder
 import cromwell.util.TryWithResource._
 
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendFileHashingActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendFileHashingActor.scala
new file mode 100644
index 000000000..270a577be
--- /dev/null
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendFileHashingActor.scala
@@ -0,0 +1,26 @@
+package cromwell.backend.impl.sfs.config
+
+import akka.actor.Props
+import com.typesafe.config.Config
+import cromwell.backend.standard.callcaching.StandardFileHashingActor.SingleFileHashRequest
+import cromwell.backend.standard.callcaching.{StandardFileHashingActor, StandardFileHashingActorParams}
+import cromwell.filesystems.gcs.batch.GcsBatchCommandBuilder
+import net.ceedubs.ficus.Ficus._
+
+import scala.util.Try
+
+object ConfigBackendFileHashingActor {
+  def props(standardParams: StandardFileHashingActorParams) = Props(new ConfigBackendFileHashingActor(standardParams))
+}
+
+class ConfigBackendFileHashingActor(standardParams: StandardFileHashingActorParams) extends StandardFileHashingActor(standardParams) with GcsBatchCommandBuilder {
+
+  lazy val hashingStrategy: ConfigHashingStrategy = {
+    configurationDescriptor.backendConfig.as[Option[Config]]("filesystems.local.caching") map ConfigHashingStrategy.apply getOrElse ConfigHashingStrategy.defaultStrategy
+  }
+  
+  override def customHashStrategy(fileRequest: SingleFileHashRequest): Option[Try[String]] = {
+    log.debug(hashingStrategy.toString)
+    Option(hashingStrategy.getHash(fileRequest, log))
+  }
+}
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendLifecycleActorFactory.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendLifecycleActorFactory.scala
index ca17199d6..c30fb1d6e 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendLifecycleActorFactory.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigBackendLifecycleActorFactory.scala
@@ -2,9 +2,9 @@ package cromwell.backend.impl.sfs.config
 
 import com.typesafe.config.Config
 import cromwell.backend.BackendConfigurationDescriptor
-import cromwell.backend.callcaching.FileHashingActor.FileHashingFunction
 import cromwell.backend.impl.sfs.config.ConfigConstants._
 import cromwell.backend.sfs._
+import cromwell.backend.standard.callcaching.StandardFileHashingActor
 import cromwell.core.JobExecutionToken.JobExecutionTokenType
 import net.ceedubs.ficus.Ficus._
 import org.slf4j.{Logger, LoggerFactory}
@@ -32,12 +32,7 @@ class ConfigBackendLifecycleActorFactory(name: String, val configurationDescript
       classOf[DispatchedConfigAsyncJobExecutionActor]
   }
 
-  override lazy val fileHashingFunction: Option[FileHashingFunction] = {
-    logger.debug(hashingStrategy.toString)
-    Option(FileHashingFunction(hashingStrategy.getHash))
-  }
-
-  override lazy val fileHashingActorCount: Int = 5
+  override lazy val fileHashingActorClassOption: Option[Class[_ <: StandardFileHashingActor]] = Option(classOf[ConfigBackendFileHashingActor])
 
   override val jobExecutionTokenType: JobExecutionTokenType = {
     val concurrentJobLimit = configurationDescriptor.backendConfig.as[Option[Int]]("concurrent-job-limit")
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigConstants.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigConstants.scala
index 3a951be9b..682b794f4 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigConstants.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigConstants.scala
@@ -25,7 +25,8 @@ object ConfigConstants {
   val MemoryRuntimeAttribute = "memory"
   // See: MemoryDeclarationValidation
   val MemoryRuntimeAttributePrefix = "memory_"
-
+  val DiskRuntimeAttribute = "disk"
+  val DiskRuntimeAttributePrefix = "disk_"
   /*
   List of task names used internally.
   NOTE: underscore separated
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategy.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategy.scala
index 47136bae0..a497ee98b 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategy.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategy.scala
@@ -4,8 +4,8 @@ import java.io.FileNotFoundException
 
 import akka.event.LoggingAdapter
 import com.typesafe.config.Config
-import cromwell.backend.callcaching.FileHashingActor.SingleFileHashRequest
 import cromwell.backend.standard.StandardInitializationData
+import cromwell.backend.standard.callcaching.StandardFileHashingActor.SingleFileHashRequest
 import cromwell.core.path.{Path, PathFactory}
 import cromwell.util.TryWithResource._
 import net.ceedubs.ficus.Ficus._
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigInitializationActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigInitializationActor.scala
index 34f93b0ff..e5538b323 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigInitializationActor.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigInitializationActor.scala
@@ -3,7 +3,9 @@ package cromwell.backend.impl.sfs.config
 import cromwell.backend.io.WorkflowPaths
 import cromwell.backend.sfs._
 import cromwell.backend.standard.{StandardInitializationActorParams, StandardInitializationData, StandardValidatedRuntimeAttributesBuilder}
-import wdl4s.WdlNamespace
+import wdl4s.wdl.WdlNamespace
+
+import scala.concurrent.Future
 
 /**
   * Extension of the SharedFileSystemBackendInitializationData with declarations of extra runtime attributes, and a
@@ -40,9 +42,11 @@ class ConfigInitializationActor(params: StandardInitializationActorParams)
     DeclarationValidation.fromDeclarations(configWdlNamespace.runtimeDeclarations)
   }
 
-  override lazy val initializationData: ConfigInitializationData = {
+  override lazy val initializationData: Future[ConfigInitializationData] = {
     val wdlNamespace = configWdlNamespace.wdlNamespace
-    new ConfigInitializationData(workflowPaths, runtimeAttributesBuilder, declarationValidations, wdlNamespace)
+    workflowPaths map {
+      new ConfigInitializationData(_, runtimeAttributesBuilder, declarationValidations, wdlNamespace)
+    }
   }
 
   override lazy val runtimeAttributesBuilder: StandardValidatedRuntimeAttributesBuilder = {
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigWdlNamespace.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigWdlNamespace.scala
index 565819e48..1e434de56 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigWdlNamespace.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigWdlNamespace.scala
@@ -3,7 +3,7 @@ package cromwell.backend.impl.sfs.config
 import com.typesafe.config.Config
 import cromwell.backend.impl.sfs.config.ConfigConstants._
 import net.ceedubs.ficus.Ficus._
-import wdl4s._
+import wdl4s.wdl._
 
 import scala.util.{Failure, Success}
 
@@ -33,7 +33,7 @@ class ConfigWdlNamespace(backendConfig: Config) {
   private val checkAliveSourceOption = checkAliveCommandOption.map(makeWdlSource(
     CheckAliveTask, _, jobIdRuntimeAttributes))
 
-  private val wdlSource =
+  private val workflowSource =
     s"""
        |${submitSourceOption getOrElse ""}
        |${submitDockerSourceOption getOrElse ""}
@@ -45,9 +45,9 @@ class ConfigWdlNamespace(backendConfig: Config) {
     * The wdl namespace containing the submit, kill, and check alive tasks.
     */
   val wdlNamespace = {
-    WdlNamespace.loadUsingSource(wdlSource, None, None) match {
+    WdlNamespace.loadUsingSource(workflowSource, None, None) match {
       case Success(ns) => ns
-      case Failure(f) => throw new RuntimeException(s"Error parsing generated wdl:\n$wdlSource".stripMargin, f)
+      case Failure(f) => throw new RuntimeException(s"Error parsing generated wdl:\n$workflowSource".stripMargin, f)
     }
   }
 
@@ -61,7 +61,7 @@ class ConfigWdlNamespace(backendConfig: Config) {
 }
 
 object ConfigWdlNamespace {
-  private def makeWdlSource(taskName: String, command: String, declarations: String): WdlSource = {
+  private def makeWdlSource(taskName: String, command: String, declarations: String): WorkflowSource = {
     s"""
        |task $taskName {
        |$declarations
@@ -72,9 +72,9 @@ object ConfigWdlNamespace {
        |""".stripMargin
   }
 
-  private def makeTask(taskName: String, command: String, declarations: String): Task = {
-    val wdlSource = makeWdlSource(taskName, command, declarations)
-    val wdlNamespace = WdlNamespace.loadUsingSource(wdlSource, None, None).get
+  private def makeTask(taskName: String, command: String, declarations: String): WdlTask = {
+    val workflowSource = makeWdlSource(taskName, command, declarations)
+    val wdlNamespace = WdlNamespace.loadUsingSource(workflowSource, None, None).get
     wdlNamespace.findTask(taskName).getOrElse(throw new RuntimeException(s"Couldn't find task $taskName"))
   }
 
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/DeclarationValidation.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/DeclarationValidation.scala
index 587042d5e..5627be9c0 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/DeclarationValidation.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/DeclarationValidation.scala
@@ -1,10 +1,11 @@
 package cromwell.backend.impl.sfs.config
 
 import cromwell.backend.validation._
-import wdl4s.expression.NoFunctions
-import wdl4s.types._
-import wdl4s.values.WdlValue
-import wdl4s.{Declaration, NoLookup, WdlExpression}
+import wdl4s.wdl.expression.NoFunctions
+import wdl4s.wdl.types._
+import wdl4s.wdl.values.WdlValue
+import wdl4s.wdl.{Declaration, NoLookup, WdlExpression}
+import cromwell.backend.impl.sfs.config.ConfigConstants._
 
 /**
   * Creates instances of runtime attribute validations from WDL declarations.
@@ -25,10 +26,12 @@ object DeclarationValidation {
       // Docker and CPU are special keys understood by cromwell.
       case name if name == DockerValidation.instance.key =>
         new DeclarationValidation(declaration, DockerValidation.instance)
-      case name if name == CpuValidation.instance.key => new DeclarationValidation(declaration, CpuValidation.default)
+      case name if name == CpuValidation.instance.key => new DeclarationValidation(declaration, CpuValidation.instance)
       // See MemoryDeclarationValidation for more info
-      case name if MemoryDeclarationValidation.isMemoryDeclaration(name) =>
-        new MemoryDeclarationValidation(declaration)
+      case name if MemoryDeclarationValidation.isMemoryDeclaration(name, MemoryRuntimeAttribute, MemoryRuntimeAttributePrefix) =>
+        new MemoryDeclarationValidation(declaration, MemoryRuntimeAttribute, MemoryRuntimeAttributePrefix)
+      case name if MemoryDeclarationValidation.isMemoryDeclaration(name, DiskRuntimeAttribute, DiskRuntimeAttributePrefix) =>
+        new MemoryDeclarationValidation(declaration, DiskRuntimeAttribute, DiskRuntimeAttributePrefix)
       // All other declarations must be a Boolean, Float, Integer, or String.
       case _ =>
         val validatedRuntimeAttr = validator(declaration.wdlType, declaration.unqualifiedName)
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/MemoryDeclarationValidation.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/MemoryDeclarationValidation.scala
index 39fd88046..9ea3f9735 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/MemoryDeclarationValidation.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/MemoryDeclarationValidation.scala
@@ -1,13 +1,12 @@
 package cromwell.backend.impl.sfs.config
 
 import cromwell.backend.MemorySize
-import cromwell.backend.impl.sfs.config.ConfigConstants._
 import cromwell.backend.validation._
-import wdl4s.expression.NoFunctions
+import wdl4s.wdl.expression.NoFunctions
 import wdl4s.parser.MemoryUnit
-import wdl4s.types._
-import wdl4s.values.{WdlFloat, WdlInteger, WdlOptionalValue, WdlValue}
-import wdl4s.{Declaration, NoLookup, WdlExpression}
+import wdl4s.wdl.types._
+import wdl4s.wdl.values.{WdlFloat, WdlInteger, WdlOptionalValue, WdlValue}
+import wdl4s.wdl.{Declaration, NoLookup, WdlExpression}
 
 /**
   * Maps declarations of memory in WDL runtime attributes to the commands used to submit.
@@ -31,8 +30,8 @@ import wdl4s.{Declaration, NoLookup, WdlExpression}
   *
   * @param declaration The declaration used to create this memory validation.
   */
-class MemoryDeclarationValidation(declaration: Declaration)
-  extends DeclarationValidation(declaration, MemoryValidation.instance) {
+class MemoryDeclarationValidation(declaration: Declaration, attributeName: String, attributeNamePrefix: String)
+  extends DeclarationValidation(declaration, MemoryValidation.instance(attributeName)) {
 
   import MemoryDeclarationValidation._
 
@@ -64,7 +63,7 @@ class MemoryDeclarationValidation(declaration: Declaration)
   }
 
   private lazy val declarationMemoryUnit: MemoryUnit = {
-    val suffix = memoryUnitSuffix(declaration.unqualifiedName)
+    val suffix = memoryUnitSuffix(declaration.unqualifiedName, attributeName, attributeNamePrefix)
     val memoryUnitOption = MemoryUnit.values.find(_.suffixes.map(_.toLowerCase).contains(suffix.toLowerCase))
     memoryUnitOption match {
       case Some(memoryUnit) => memoryUnit
@@ -79,7 +78,7 @@ class MemoryDeclarationValidation(declaration: Declaration)
     * @return The value from the collection wrapped in `Some`, or `None` if the value wasn't found.
     */
   override def extractWdlValueOption(validatedRuntimeAttributes: ValidatedRuntimeAttributes): Option[WdlValue] = {
-    RuntimeAttributesValidation.extractOption(MemoryValidation.instance, validatedRuntimeAttributes) map
+    RuntimeAttributesValidation.extractOption(MemoryValidation.instance(attributeName), validatedRuntimeAttributes) map
       coerceMemorySize(declaration.wdlType)
   }
 
@@ -94,11 +93,11 @@ class MemoryDeclarationValidation(declaration: Declaration)
 }
 
 object MemoryDeclarationValidation {
-  def isMemoryDeclaration(name: String): Boolean = {
+  def isMemoryDeclaration(name: String, attributeName: String, attributeNamePrefix: String): Boolean = {
     name match {
-      case MemoryRuntimeAttribute => true
-      case prefixed if prefixed.startsWith(MemoryRuntimeAttributePrefix) =>
-        val suffix = memoryUnitSuffix(name)
+      case `attributeName` => true
+      case prefixed if prefixed.startsWith(attributeNamePrefix) =>
+        val suffix = memoryUnitSuffix(name, attributeName, attributeNamePrefix)
         MemoryUnit.values exists {
           _.suffixes.map(_.toLowerCase).contains(suffix)
         }
@@ -106,10 +105,10 @@ object MemoryDeclarationValidation {
     }
   }
 
-  private def memoryUnitSuffix(name: String) = {
-    if (name == MemoryRuntimeAttribute)
+  private def memoryUnitSuffix(name: String, attributeName: String, attributeNamePrefix: String) = {
+    if (name == attributeName)
       MemoryUnit.Bytes.suffixes.head
     else
-      name.substring(MemoryRuntimeAttributePrefix.length)
+      name.substring(attributeNamePrefix.length)
   }
 }
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala
index 272be2787..bc50ec13a 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala
@@ -11,8 +11,8 @@ import cromwell.backend.wdl.WdlFileMapper
 import cromwell.core.CromwellFatalExceptionMarker
 import cromwell.core.path.{DefaultPath, DefaultPathBuilder, Path, PathFactory}
 import lenthall.util.TryUtil
-import wdl4s.EvaluatedTaskInputs
-import wdl4s.values._
+import wdl4s.wdl.EvaluatedTaskInputs
+import wdl4s.wdl.values._
 
 import scala.collection.JavaConverters._
 import scala.language.postfixOps
@@ -54,7 +54,7 @@ object SharedFileSystem extends StrictLogging {
   private def localizePathViaHardLink(originalPath: Path, executionPath: Path): Try[Unit] = {
     val action = Try {
       executionPath.parent.createPermissionedDirectories()
-      executionPath.linkTo(originalPath)
+      originalPath.linkTo(executionPath)
     }.void
     logOnFailure(action, "hard link")
   }
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala
index 731ad6efa..cdc79b279 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemAsyncJobExecutionActor.scala
@@ -9,7 +9,7 @@ import cromwell.backend.standard.{StandardAsyncExecutionActor, StandardAsyncJob}
 import cromwell.backend.validation._
 import cromwell.core.path.{DefaultPathBuilder, Path}
 import cromwell.core.retry.SimpleExponentialBackoff
-import wdl4s.values.WdlFile
+import wdl4s.wdl.values.WdlFile
 
 import scala.concurrent.duration._
 
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemBackendLifecycleActorFactory.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemBackendLifecycleActorFactory.scala
index 85bd0f1d9..7e991a6ec 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemBackendLifecycleActorFactory.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemBackendLifecycleActorFactory.scala
@@ -1,6 +1,7 @@
 package cromwell.backend.sfs
 
 import cromwell.backend.standard._
+import cromwell.backend.standard.callcaching.StandardCacheHitCopyingActor
 
 /**
   * A factory that can be extended for any shared file system implementation.
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemCacheHitCopyingActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemCacheHitCopyingActor.scala
index 396a33849..b29816600 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemCacheHitCopyingActor.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemCacheHitCopyingActor.scala
@@ -1,11 +1,32 @@
 package cromwell.backend.sfs
 
-import cromwell.backend.standard.{StandardCacheHitCopyingActor, StandardCacheHitCopyingActorParams}
-import cromwell.core.path.Path
+import cromwell.backend.standard.callcaching.StandardCacheHitCopyingActor.PathPair
+import cromwell.backend.standard.callcaching.{StandardCacheHitCopyingActor, StandardCacheHitCopyingActorParams}
+import cromwell.filesystems.gcs.batch.GcsBatchCommandBuilder
+import lenthall.util.TryUtil
+import cats.instances.try_._
+import cats.syntax.functor._
+
+import scala.util.{Failure, Try}
 
 class SharedFileSystemCacheHitCopyingActor(standardParams: StandardCacheHitCopyingActorParams)
-  extends StandardCacheHitCopyingActor(standardParams) with SharedFileSystemJobCachingActorHelper {
-  override protected def duplicate(source: Path, destination: Path): Unit = {
-    sharedFileSystem.cacheCopy(source, destination).get
+  extends StandardCacheHitCopyingActor(standardParams) with SharedFileSystemJobCachingActorHelper with GcsBatchCommandBuilder {
+  override protected def duplicate(copyPairs: Set[PathPair]): Option[Try[Unit]] = Option {
+    val copies = copyPairs map {
+      case (source, destination) => 
+        sharedFileSystem.cacheCopy(source, destination)
+    }
+    
+    TryUtil.sequence(copies.toList).void recoverWith {
+      case failure =>
+        // If one or more of the copies failed, we want to delete all the files that were successfully copied
+        // before that. Especially if they've been symlinked, leaving them could lead to rewriting the original
+        // files when the job gets re-run
+        // TODO: this could be done more generally in the StandardCacheHitCopyingActor
+        copyPairs foreach {
+          case (_, dst) => dst.delete(swallowIOExceptions = true)
+        }
+        Failure(failure)
+    }
   }
 }
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemInitializationActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemInitializationActor.scala
index 6ca29cafe..dbbca727c 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemInitializationActor.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemInitializationActor.scala
@@ -1,28 +1,33 @@
 package cromwell.backend.sfs
 
 import cats.data.Validated.{Invalid, Valid}
+import cats.instances.future._
+import cats.instances.list._
+import cats.syntax.traverse._
 import cromwell.backend.BackendInitializationData
 import cromwell.backend.io.WorkflowPaths
 import cromwell.backend.standard.{StandardExpressionFunctions, StandardInitializationActor, StandardInitializationActorParams}
 import cromwell.backend.wfs.WorkflowPathBuilder
-import cromwell.core.path.{DefaultPathBuilderFactory, PathBuilder, PathBuilderFactory}
+import cromwell.core.path.{DefaultPathBuilder, PathBuilder}
 import cromwell.filesystems.gcs.{GcsPathBuilderFactory, GoogleConfiguration}
 import lenthall.exception.MessageAggregation
 import net.ceedubs.ficus.Ficus._
 
 import scala.concurrent.Future
-import scala.util.Try
 
 class SharedFileSystemInitializationActor(standardParams: StandardInitializationActorParams)
   extends StandardInitializationActor(standardParams) {
 
+  private implicit val system = context.system
+
   /**
     * If the backend sets a gcs authentication mode, try to create a PathBuilderFactory with it.
     */
   lazy val gcsPathBuilderFactory: Option[GcsPathBuilderFactory] = {
     configurationDescriptor.backendConfig.as[Option[String]]("filesystems.gcs.auth") map { configAuth =>
-      GoogleConfiguration(configurationDescriptor.globalConfig).auth(configAuth) match {
-        case Valid(auth) => GcsPathBuilderFactory(auth)
+      val googleConfiguration = GoogleConfiguration(configurationDescriptor.globalConfig)
+      googleConfiguration.auth(configAuth) match {
+        case Valid(auth) => GcsPathBuilderFactory(auth, googleConfiguration.applicationName)
         case Invalid(error) => throw new MessageAggregation {
           override def exceptionContext: String = "Failed to parse gcs auth configuration"
 
@@ -32,23 +37,21 @@ class SharedFileSystemInitializationActor(standardParams: StandardInitialization
     }
   }
 
-  lazy val pathBuilderFactories: List[PathBuilderFactory] =
-    List(gcsPathBuilderFactory, Option(DefaultPathBuilderFactory)).flatten
-
-  override lazy val pathBuilders: List[PathBuilder] =
-    pathBuilderFactories map { _.withOptions(workflowDescriptor.workflowOptions)(context.system) }
+  override lazy val pathBuilders: Future[List[PathBuilder]] =
+    gcsPathBuilderFactory.toList.traverse(_.withOptions(workflowDescriptor.workflowOptions)).map(_ ++ Option(DefaultPathBuilder))
 
-  override lazy val workflowPaths: WorkflowPaths =
-    WorkflowPathBuilder.workflowPaths(configurationDescriptor, workflowDescriptor, pathBuilders)
+  override lazy val workflowPaths: Future[WorkflowPaths] = pathBuilders map {
+    WorkflowPathBuilder.workflowPaths(configurationDescriptor, workflowDescriptor, _)
+  }
 
   override lazy val expressionFunctions: Class[_ <: StandardExpressionFunctions] =
     classOf[SharedFileSystemExpressionFunctions]
 
   override def beforeAll(): Future[Option[BackendInitializationData]] = {
-    Future.fromTry(Try {
-      publishWorkflowRoot(workflowPaths.workflowRoot.pathAsString)
-      workflowPaths.workflowRoot.createPermissionedDirectories()
-      Option(initializationData)
-    })
+    initializationData map { data =>
+      publishWorkflowRoot(data.workflowPaths.workflowRoot.pathAsString)
+      data.workflowPaths.workflowRoot.createPermissionedDirectories()
+      Option(data)
+    }
   }
 }
diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategySpec.scala b/supportedBackends/sfs/src/test/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategySpec.scala
index 588d1a5c4..e65debcc6 100644
--- a/supportedBackends/sfs/src/test/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategySpec.scala
+++ b/supportedBackends/sfs/src/test/scala/cromwell/backend/impl/sfs/config/ConfigHashingStrategySpec.scala
@@ -4,15 +4,15 @@ import java.util.UUID
 
 import akka.event.LoggingAdapter
 import com.typesafe.config.{ConfigFactory, ConfigValueFactory}
-import cromwell.backend.callcaching.FileHashingActor.SingleFileHashRequest
 import cromwell.backend.io.WorkflowPaths
 import cromwell.backend.standard.StandardInitializationData
+import cromwell.backend.standard.callcaching.StandardFileHashingActor.SingleFileHashRequest
 import cromwell.core.path.{DefaultPathBuilder, Path}
 import org.apache.commons.codec.digest.DigestUtils
 import org.scalatest.prop.TableDrivenPropertyChecks
 import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers}
 import org.specs2.mock.Mockito
-import wdl4s.values.WdlFile
+import wdl4s.wdl.values.WdlFile
 
 import scala.util.Success
 
@@ -38,7 +38,6 @@ class ConfigHashingStrategySpec extends FlatSpec with Matchers with TableDrivenP
 
   def mockRequest(withSibling: Boolean, symlink: Boolean) = {
     if (withSibling && md5File.notExists) md5File.write(md5FileHash)
-    val request = mock[SingleFileHashRequest]
     val requestFile = if (symlink) {
       val symLink: Path = symLinksDir./(s"symlink-${randomName()}")
       symLink.symbolicLinkTo(file)
@@ -51,10 +50,7 @@ class ConfigHashingStrategySpec extends FlatSpec with Matchers with TableDrivenP
     val initData = mock[StandardInitializationData]
     initData.workflowPaths returns workflowPaths
 
-    request.file returns WdlFile(requestFile.pathAsString)
-    request.initializationData returns Option(initData)
-
-    request
+    SingleFileHashRequest(null, null, WdlFile(requestFile.pathAsString), Option(initData))
   }
 
   def makeStrategy(strategy: String, checkSibling: Option[Boolean] = None) = {
diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/impl/sfs/config/MemoryDeclarationValidationSpec.scala b/supportedBackends/sfs/src/test/scala/cromwell/backend/impl/sfs/config/MemoryDeclarationValidationSpec.scala
index 3cc444c53..a1b7928eb 100644
--- a/supportedBackends/sfs/src/test/scala/cromwell/backend/impl/sfs/config/MemoryDeclarationValidationSpec.scala
+++ b/supportedBackends/sfs/src/test/scala/cromwell/backend/impl/sfs/config/MemoryDeclarationValidationSpec.scala
@@ -6,7 +6,8 @@ import cromwell.backend.validation.{RuntimeAttributesKeys, ValidatedRuntimeAttri
 import org.scalatest.prop.TableDrivenPropertyChecks
 import org.scalatest.{FlatSpec, Matchers}
 import wdl4s.parser.MemoryUnit
-import wdl4s.values.{WdlFloat, WdlInteger}
+import wdl4s.wdl.values.{WdlFloat, WdlInteger}
+import ConfigConstants._
 
 class MemoryDeclarationValidationSpec extends FlatSpec with Matchers with TableDrivenPropertyChecks {
   behavior of "MemoryDeclarationValidation"
@@ -40,7 +41,8 @@ class MemoryDeclarationValidationSpec extends FlatSpec with Matchers with TableD
 
       val configWdlNamespace = new ConfigWdlNamespace(config)
       val runtimeDeclaration = configWdlNamespace.runtimeDeclarations.head
-      val memoryDeclarationValidation = new MemoryDeclarationValidation(runtimeDeclaration)
+      val memoryDeclarationValidation = new MemoryDeclarationValidation(runtimeDeclaration,
+        MemoryRuntimeAttribute, MemoryRuntimeAttributePrefix)
       val attributes = runtimeAmount
         .map(amount => RuntimeAttributesKeys.MemoryKey -> MemorySize(amount.toDouble, MemoryUnit.GB))
         .toMap
@@ -52,7 +54,8 @@ class MemoryDeclarationValidationSpec extends FlatSpec with Matchers with TableD
       val expectedDefault = expectedDefaultAmount
         .map(amount => WdlInteger(MemorySize(amount.toDouble, MemoryUnit.GB).bytes.toInt))
 
-      MemoryDeclarationValidation.isMemoryDeclaration(runtimeDeclaration.unqualifiedName) should be(true)
+      MemoryDeclarationValidation.isMemoryDeclaration(runtimeDeclaration.unqualifiedName,
+        MemoryRuntimeAttribute, MemoryRuntimeAttributePrefix) should be(true)
       default should be(expectedDefault)
       extracted should be(expectedExtracted)
     }
@@ -92,7 +95,8 @@ class MemoryDeclarationValidationSpec extends FlatSpec with Matchers with TableD
 
       val configWdlNamespace = new ConfigWdlNamespace(config)
       val runtimeDeclaration = configWdlNamespace.runtimeDeclarations.head
-      MemoryDeclarationValidation.isMemoryDeclaration(runtimeDeclaration.unqualifiedName) should be(false)
+      MemoryDeclarationValidation.isMemoryDeclaration(runtimeDeclaration.unqualifiedName,
+        MemoryRuntimeAttribute, MemoryRuntimeAttributePrefix) should be(false)
     }
   }
 }
diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemInitializationActorSpec.scala b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemInitializationActorSpec.scala
index 085ad173d..a08b3a3b9 100644
--- a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemInitializationActorSpec.scala
+++ b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemInitializationActorSpec.scala
@@ -2,14 +2,15 @@ package cromwell.backend.sfs
 
 import akka.actor.Props
 import akka.testkit.{EventFilter, ImplicitSender, TestDuration}
+import com.typesafe.config.ConfigFactory
 import cromwell.backend.BackendSpec._
 import cromwell.backend.BackendWorkflowInitializationActor.Initialize
 import cromwell.backend.standard.DefaultInitializationActorParams
-import cromwell.backend.{BackendConfigurationDescriptor, BackendWorkflowDescriptor}
+import cromwell.backend.{BackendConfigurationDescriptor, BackendWorkflowDescriptor, TestConfig}
 import cromwell.core.TestKitSuite
 import cromwell.core.logging.LoggingTest._
 import org.scalatest.{Matchers, WordSpecLike}
-import wdl4s.TaskCall
+import wdl4s.wdl.WdlTaskCall
 
 import scala.concurrent.duration._
 
@@ -36,9 +37,9 @@ class SharedFileSystemInitializationActorSpec extends TestKitSuite("SharedFileSy
       |}
     """.stripMargin
 
-  private def getActorRef(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[TaskCall],
+  private def getActorRef(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[WdlTaskCall],
                           conf: BackendConfigurationDescriptor) = {
-    val params = DefaultInitializationActorParams(workflowDescriptor, calls, emptyActor, conf)
+    val params = DefaultInitializationActorParams(workflowDescriptor, emptyActor, calls, emptyActor, conf, restarting = false)
     val props = Props(new SharedFileSystemInitializationActor(params))
     system.actorOf(props, "SharedFileSystemInitializationActor")
   }
@@ -47,7 +48,7 @@ class SharedFileSystemInitializationActorSpec extends TestKitSuite("SharedFileSy
     "log a warning message when there are unsupported runtime attributes" in {
       within(Timeout) {
         val workflowDescriptor = buildWorkflowDescriptor(HelloWorld, runtime = """runtime { unsupported: 1 }""")
-        val conf = emptyBackendConfig
+        val conf = BackendConfigurationDescriptor(TestConfig.sampleBackendRuntimeConfig, ConfigFactory.empty())
         val backend = getActorRef(workflowDescriptor, workflowDescriptor.workflow.taskCalls, conf)
         val pattern = "Key/s [unsupported] is/are not supported by backend. " +
           "Unsupported attributes will not be part of job executions."
diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala
index a0f8d44a2..84a6a84a3 100644
--- a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala
+++ b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala
@@ -1,6 +1,6 @@
 package cromwell.backend.sfs
 
-import akka.testkit.TestDuration
+import akka.testkit.{TestDuration, TestProbe}
 import com.typesafe.config.ConfigFactory
 import cromwell.backend.BackendJobExecutionActor.{AbortedResponse, JobFailedNonRetryableResponse, JobSucceededResponse}
 import cromwell.backend.BackendLifecycleActor.AbortJobCommand
@@ -12,15 +12,15 @@ import cromwell.backend.sfs.TestLocalAsyncJobExecutionActor._
 import cromwell.backend.standard.StandardValidatedRuntimeAttributesBuilder
 import cromwell.core.Tags._
 import cromwell.core._
-import cromwell.core.callcaching.CallCachingEligible
+import cromwell.core.callcaching.NoDocker
 import cromwell.core.path.{DefaultPathBuilder, Path}
-import cromwell.services.keyvalue.KeyValueServiceActor.{KvJobKey, KvPair, ScopedKey}
+import cromwell.services.keyvalue.KeyValueServiceActor._
 import lenthall.exception.AggregatedException
 import org.scalatest.concurrent.PatienceConfiguration.Timeout
 import org.scalatest.prop.TableDrivenPropertyChecks
 import org.scalatest.{Assertion, FlatSpecLike, OptionValues}
-import wdl4s.types._
-import wdl4s.values._
+import wdl4s.wdl.types._
+import wdl4s.wdl.values._
 
 import scala.concurrent.duration._
 
@@ -30,16 +30,16 @@ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSyst
   behavior of "SharedFileSystemJobExecutionActor"
 
   lazy val runtimeAttributeDefinitions: Set[RuntimeAttributeDefinition] =
-    StandardValidatedRuntimeAttributesBuilder.default.definitions.toSet
+    StandardValidatedRuntimeAttributesBuilder.default(Some(TestConfig.optionalRuntimeConfig)).definitions.toSet
 
   def executeSpec(docker: Boolean): Any = {
     val expectedOutputs: CallOutputs = Map(
       "salutation" -> JobOutput(WdlString("Hello you !"))
     )
-    val expectedResponse = JobSucceededResponse(mock[BackendJobDescriptorKey], Some(0), expectedOutputs, None, Seq.empty)
+    val expectedResponse = JobSucceededResponse(mock[BackendJobDescriptorKey], Some(0), expectedOutputs, None, Seq.empty, None)
     val runtime = if (docker) """runtime { docker: "ubuntu:latest" }""" else ""
     val workflowDescriptor = buildWorkflowDescriptor(HelloWorld, runtime = runtime)
-    val workflow = TestWorkflow(workflowDescriptor, emptyBackendConfig, expectedResponse)
+    val workflow = TestWorkflow(workflowDescriptor, TestConfig.backendRuntimeConfigDescriptor, expectedResponse)
     val backend = createBackend(jobDescriptorFromSingleCallWorkflow(workflow.workflowDescriptor, Map.empty, WorkflowOptions.empty, runtimeAttributeDefinitions), workflow.config)
     testWorkflow(workflow, backend)
   }
@@ -55,27 +55,29 @@ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSyst
   it should "send back an execution failure if the task fails" in {
     val expectedResponse =
       JobFailedNonRetryableResponse(mock[BackendJobDescriptorKey], WrongReturnCode("wf_goodbye.goodbye:NA:1", 1, None), Option(1))
-    val workflow = TestWorkflow(buildWorkflowDescriptor(GoodbyeWorld), emptyBackendConfig, expectedResponse)
+    val workflow = TestWorkflow(buildWorkflowDescriptor(GoodbyeWorld), TestConfig.backendRuntimeConfigDescriptor, expectedResponse)
     val backend = createBackend(jobDescriptorFromSingleCallWorkflow(workflow.workflowDescriptor, Map.empty, WorkflowOptions.empty, runtimeAttributeDefinitions), workflow.config)
     testWorkflow(workflow, backend)
   }
 
   def localizationSpec(docker: Boolean): Assertion = {
-    def templateConf(localizers: String) = BackendConfigurationDescriptor(
-      ConfigFactory.parseString(
-        s"""|{
-            |  root = "local-cromwell-executions"
-            |  filesystems {
-            |    local {
-            |      localization = [
-            |        $localizers
-            |      ]
-            |    }
-            |  }
-            |}
-            |""".stripMargin),
-      ConfigFactory.parseString("{}")
-    )
+    def templateConf(localizers: String) = BackendConfigurationDescriptor(ConfigFactory.parseString(
+                s"""|{
+                    |  root = "local-cromwell-executions"
+                    |  filesystems {
+                    |    local {
+                    |      localization = [
+                    |        $localizers
+                    |      ]
+                    |    }
+                    |  }
+                    |  default-runtime-attributes {
+                    |  cpu: 1
+                    |  failOnStderr: false
+                    |  continueOnReturnCode: 0
+                    |  }
+                    |}
+                    |""".stripMargin), ConfigFactory.parseString("{}"))
 
     val hardConf = templateConf("hard-link")
     val symConf = templateConf("soft-link")
@@ -109,9 +111,9 @@ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSyst
       val workflowDescriptor = buildWorkflowDescriptor(InputFiles, inputs, runtime = runtime)
       val backend = createBackend(jobDescriptorFromSingleCallWorkflow(workflowDescriptor, inputs, WorkflowOptions.empty, runtimeAttributeDefinitions), conf)
       val jobDescriptor: BackendJobDescriptor = jobDescriptorFromSingleCallWorkflow(workflowDescriptor, inputs, WorkflowOptions.empty, runtimeAttributeDefinitions)
-      val expectedResponse = JobSucceededResponse(jobDescriptor.key, Some(0), expectedOutputs, None, Seq.empty)
+      val expectedResponse = JobSucceededResponse(jobDescriptor.key, Some(0), expectedOutputs, None, Seq.empty, None)
 
-      val jobPaths = new JobPathsWithDocker(jobDescriptor.key, workflowDescriptor, conf.backendConfig)
+      val jobPaths = JobPathsWithDocker(jobDescriptor.key, workflowDescriptor, conf.backendConfig)
 
       whenReady(backend.execute) { executionResponse =>
         assertResponse(executionResponse, expectedResponse)
@@ -142,7 +144,7 @@ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSyst
   it should "abort a job and kill a process" in {
     val workflowDescriptor = buildWorkflowDescriptor(Sleep10)
     val jobDescriptor: BackendJobDescriptor = jobDescriptorFromSingleCallWorkflow(workflowDescriptor, Map.empty, WorkflowOptions.empty, runtimeAttributeDefinitions)
-    val backendRef = createBackendRef(jobDescriptor, emptyBackendConfig)
+    val backendRef = createBackendRef(jobDescriptor, TestConfig.backendRuntimeConfigDescriptor)
     val backend = backendRef.underlyingActor
 
     val execute = backend.execute
@@ -156,10 +158,10 @@ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSyst
   def recoverSpec(completed: Boolean, writeReturnCode: Boolean = true): Assertion = {
     val workflowDescriptor = buildWorkflowDescriptor(HelloWorld)
     val jobDescriptor: BackendJobDescriptor = jobDescriptorFromSingleCallWorkflow(workflowDescriptor, Map.empty, WorkflowOptions.empty, runtimeAttributeDefinitions)
-    val backendRef = createBackendRef(jobDescriptor, emptyBackendConfig)
+    val backendRef = createBackendRef(jobDescriptor, TestConfig.backendRuntimeConfigDescriptor)
     val backend = backendRef.underlyingActor
 
-    val jobPaths = new JobPathsWithDocker(jobDescriptor.key, workflowDescriptor, ConfigFactory.empty)
+    val jobPaths = JobPathsWithDocker(jobDescriptor.key, workflowDescriptor, ConfigFactory.empty)
     jobPaths.callExecutionRoot.createPermissionedDirectories()
     jobPaths.stdout.write("Hello stubby ! ")
     jobPaths.stderr.touch()
@@ -180,14 +182,17 @@ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSyst
         pidField.get(p).toString
       }
 
-    val execute = backend.recover
+    def execute = backend.recover
 
     val kvJobKey =
       KvJobKey(jobDescriptor.key.call.fullyQualifiedName, jobDescriptor.key.index, jobDescriptor.key.attempt)
     val scopedKey = ScopedKey(workflowDescriptor.id, kvJobKey, SharedFileSystemAsyncJobExecutionActor.JobIdKey)
     val kvPair = KvPair(scopedKey, Option(pid))
 
-    backendRef ! kvPair
+    val previousKvPutter = TestProbe()
+    val kvPutReq = KvPut(kvPair)
+    backendRef.underlyingActor.serviceRegistryActor.tell(msg = kvPutReq, sender = previousKvPutter.ref)
+    previousKvPutter.expectMsg(KvPutSuccess(kvPutReq))
 
     whenReady(execute, Timeout(10.seconds.dilated)) { executionResponse =>
       if (writeReturnCode) {
@@ -231,10 +236,10 @@ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSyst
       val runtimeAttributes = RuntimeAttributeDefinition.addDefaultsToAttributes(runtimeAttributeDefinitions, WorkflowOptions.empty)(call.task.runtimeAttributes.attrs)
 
       val jobDescriptor: BackendJobDescriptor =
-        BackendJobDescriptor(workflowDescriptor, BackendJobDescriptorKey(call, Option(shard), 1), runtimeAttributes, fqnMapToDeclarationMap(symbolMaps), CallCachingEligible)
-      val backend = createBackend(jobDescriptor, emptyBackendConfig)
+        BackendJobDescriptor(workflowDescriptor, BackendJobDescriptorKey(call, Option(shard), 1), runtimeAttributes, fqnMapToDeclarationMap(symbolMaps), NoDocker, Map.empty)
+      val backend = createBackend(jobDescriptor, TestConfig.backendRuntimeConfigDescriptor)
       val response =
-        JobSucceededResponse(mock[BackendJobDescriptorKey], Some(0), Map("out" -> JobOutput(WdlInteger(shard))), None, Seq.empty)
+        JobSucceededResponse(mock[BackendJobDescriptorKey], Some(0), Map("out" -> JobOutput(WdlInteger(shard))), None, Seq.empty, None)
       executeJobAndAssertOutputs(backend, response)
     }
   }
@@ -246,8 +251,8 @@ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSyst
     }
     val workflowDescriptor = buildWorkflowDescriptor(OutputProcess, inputs)
     val jobDescriptor: BackendJobDescriptor = jobDescriptorFromSingleCallWorkflow(workflowDescriptor, inputs, WorkflowOptions.empty, runtimeAttributeDefinitions)
-    val backend = createBackend(jobDescriptor, emptyBackendConfig)
-    val jobPaths = new JobPathsWithDocker(jobDescriptor.key, workflowDescriptor, emptyBackendConfig.backendConfig)
+    val backend = createBackend(jobDescriptor, TestConfig.backendRuntimeConfigDescriptor)
+    val jobPaths = JobPathsWithDocker(jobDescriptor.key, workflowDescriptor, TestConfig.backendRuntimeConfigDescriptor.backendConfig)
     val expectedA = WdlFile(jobPaths.callExecutionRoot.resolve("a").toAbsolutePath.pathAsString)
     val expectedB = WdlFile(jobPaths.callExecutionRoot.resolve("dir").toAbsolutePath.resolve("b").pathAsString)
     val expectedOutputs = Map(
@@ -255,7 +260,7 @@ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSyst
       "o2" -> JobOutput(WdlArray(WdlArrayType(WdlFileType), Seq(expectedA, expectedB))),
       "o3" -> JobOutput(WdlFile(inputFile))
     )
-    val expectedResponse = JobSucceededResponse(jobDescriptor.key, Some(0), expectedOutputs, None, Seq.empty)
+    val expectedResponse = JobSucceededResponse(jobDescriptor.key, Some(0), expectedOutputs, None, Seq.empty, None)
 
     executeJobAndAssertOutputs(backend, expectedResponse)
   }
@@ -263,7 +268,7 @@ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSyst
   it should "fail post processing if an output file is not found" in {
     val expectedResponse = JobFailedNonRetryableResponse(mock[BackendJobDescriptorKey],
       AggregatedException("Could not process output, file not found:", Seq.empty), Option(0))
-    val workflow = TestWorkflow(buildWorkflowDescriptor(MissingOutputProcess), emptyBackendConfig, expectedResponse)
+    val workflow = TestWorkflow(buildWorkflowDescriptor(MissingOutputProcess), TestConfig.backendRuntimeConfigDescriptor, expectedResponse)
     val backend = createBackend(jobDescriptorFromSingleCallWorkflow(workflow.workflowDescriptor, Map.empty, WorkflowOptions.empty, runtimeAttributeDefinitions), workflow.config)
     testWorkflow(workflow, backend)
   }
diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemSpec.scala b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemSpec.scala
index ec1db2314..404c2bf03 100644
--- a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemSpec.scala
+++ b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemSpec.scala
@@ -7,7 +7,7 @@ import cromwell.core.path.{DefaultPathBuilder, Path}
 import org.scalatest.prop.TableDrivenPropertyChecks
 import org.scalatest.{FlatSpec, Matchers}
 import org.specs2.mock.Mockito
-import wdl4s.values.WdlFile
+import wdl4s.wdl.values.WdlFile
 
 class SharedFileSystemSpec extends FlatSpec with Matchers with Mockito with TableDrivenPropertyChecks with BackendSpec {
 
diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/TestLocalAsyncJobExecutionActor.scala b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/TestLocalAsyncJobExecutionActor.scala
index 3d9f63e54..0935c7dc1 100644
--- a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/TestLocalAsyncJobExecutionActor.scala
+++ b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/TestLocalAsyncJobExecutionActor.scala
@@ -2,10 +2,12 @@ package cromwell.backend.sfs
 
 import akka.actor.{ActorSystem, Props}
 import akka.testkit.TestActorRef
-import cromwell.backend.standard._
 import cromwell.backend.io.WorkflowPathsWithDocker
+import cromwell.backend.standard._
 import cromwell.backend.validation.{DockerValidation, RuntimeAttributesValidation}
 import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptor}
+import cromwell.core.SimpleIoActor
+import cromwell.services.keyvalue.InMemoryKvServiceActor
 
 class TestLocalAsyncJobExecutionActor(override val standardParams: StandardAsyncExecutionActorParams)
   extends BackgroundAsyncJobExecutionActor {
@@ -21,6 +23,8 @@ class TestLocalAsyncJobExecutionActor(override val standardParams: StandardAsync
       SharedFileSystemCommand("/bin/bash", script)
     }
   }
+
+  override def dockerImageUsed: Option[String] = None
 }
 
 object TestLocalAsyncJobExecutionActor {
@@ -31,15 +35,23 @@ object TestLocalAsyncJobExecutionActor {
 
   def createBackendRef(jobDescriptor: BackendJobDescriptor, configurationDescriptor: BackendConfigurationDescriptor)
                       (implicit system: ActorSystem): TestActorRef[StandardSyncExecutionActor] = {
-    val emptyActor = system.actorOf(Props.empty)
+    val serviceRegistryActor = system.actorOf(Props(new InMemoryKvServiceActor)) // We only really need the KV store for now
+    val ioActor = system.actorOf(SimpleIoActor.props)
     val workflowPaths = new WorkflowPathsWithDocker(jobDescriptor.workflowDescriptor, configurationDescriptor.backendConfig)
     val initializationData = new StandardInitializationData(workflowPaths,
-      StandardValidatedRuntimeAttributesBuilder.default.withValidation(DockerValidation.optional),
+      StandardValidatedRuntimeAttributesBuilder.default(configurationDescriptor.backendRuntimeConfig).withValidation(DockerValidation.optional),
       classOf[SharedFileSystemExpressionFunctions])
     val asyncClass = classOf[TestLocalAsyncJobExecutionActor]
 
-    val params = DefaultStandardSyncExecutionActorParams(SharedFileSystemAsyncJobExecutionActor.JobIdKey, emptyActor,
-      jobDescriptor, configurationDescriptor, Option(initializationData), None, asyncClass)
+    val params = DefaultStandardSyncExecutionActorParams(
+      jobIdKey = SharedFileSystemAsyncJobExecutionActor.JobIdKey,
+      serviceRegistryActor = serviceRegistryActor,
+      ioActor = ioActor,
+      jobDescriptor = jobDescriptor,
+      configurationDescriptor = configurationDescriptor,
+      backendInitializationDataOption = Option(initializationData),
+      backendSingletonActorOption = None,
+      asyncJobExecutionActorClass = asyncClass)
 
     TestActorRef(new StandardSyncExecutionActor(params))
   }
diff --git a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkBackendFactory.scala b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkBackendFactory.scala
index 6fd75323e..c93868f65 100644
--- a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkBackendFactory.scala
+++ b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkBackendFactory.scala
@@ -5,25 +5,27 @@ import cromwell.backend._
 import cromwell.backend.io.JobPathsWithDocker
 import cromwell.backend.sfs.SharedFileSystemExpressionFunctions
 import cromwell.core.CallContext
-import wdl4s.TaskCall
-import wdl4s.expression.WdlStandardLibraryFunctions
+import wdl4s.wdl.WdlTaskCall
+import wdl4s.wdl.expression.WdlStandardLibraryFunctions
 
 case class SparkBackendFactory(name: String, configurationDescriptor: BackendConfigurationDescriptor, actorSystem: ActorSystem) extends BackendLifecycleActorFactory {
-  override def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[TaskCall], serviceRegistryActor: ActorRef): Option[Props] = {
+  override def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor, ioActor: ActorRef,
+                                                calls: Set[WdlTaskCall], serviceRegistryActor: ActorRef, restarting: Boolean): Option[Props] = {
     Option(SparkInitializationActor.props(workflowDescriptor, calls, configurationDescriptor, serviceRegistryActor))
   }
 
   override def jobExecutionActorProps(jobDescriptor: BackendJobDescriptor,
                                       initializationData: Option[BackendInitializationData],
                                       serviceRegistryActor: ActorRef,
+                                      ioActor: ActorRef,
                                       backendSingletonActor: Option[ActorRef]): Props = {
     SparkJobExecutionActor.props(jobDescriptor, configurationDescriptor)
   }
 
   override def expressionLanguageFunctions(workflowDescriptor: BackendWorkflowDescriptor, jobKey: BackendJobDescriptorKey,
                                            initializationData: Option[BackendInitializationData]): WdlStandardLibraryFunctions = {
-    val jobPaths = new JobPathsWithDocker(jobKey, workflowDescriptor, configurationDescriptor.backendConfig)
-    val callContext = new CallContext(
+    val jobPaths = JobPathsWithDocker(jobKey, workflowDescriptor, configurationDescriptor.backendConfig)
+    val callContext = CallContext(
       jobPaths.callExecutionRoot,
       jobPaths.stdout.toAbsolutePath.toString,
       jobPaths.stderr.toAbsolutePath.toString
diff --git a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkClusterProcess.scala b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkClusterProcess.scala
index 0edffad09..701cb21a2 100644
--- a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkClusterProcess.scala
+++ b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkClusterProcess.scala
@@ -1,14 +1,17 @@
 package cromwell.backend.impl.spark
 
 import akka.actor.ActorSystem
+import akka.http.scaladsl.Http
+import akka.http.scaladsl.model.{HttpRequest, HttpResponse, StatusCodes}
+import akka.http.scaladsl.unmarshalling.Unmarshal
+import akka.stream.ActorMaterializer
 import com.typesafe.scalalogging.Logger
 import cromwell.backend.impl.spark.SparkClusterProcess.{SparkJobSubmissionResponse, TerminalStatus}
 import cromwell.core.path.Obsolete._
 import cromwell.core.path.Path
 import org.slf4j.LoggerFactory
-import spray.client.pipelining._
-import spray.http.{HttpRequest, HttpResponse, StatusCodes}
 import spray.json.{DefaultJsonProtocol, JsonParser}
+import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._
 
 import scala.concurrent.duration._
 import scala.concurrent.{ExecutionContext, Future, Promise}
@@ -34,12 +37,6 @@ object SparkClusterProcess {
 
 }
 
-trait SparkClusterRestClient {
-  def sendAndReceive: SendReceive
-
-  def makeHttpRequest(httpRequest: HttpRequest): Future[HttpResponse]
-}
-
 trait SparkClusterProcessMonitor {
   def startMonitoringSparkClusterJob(jobPath: Path, jsonFile: String): Future[TerminalStatus]
 
@@ -55,20 +52,18 @@ trait SparkClusterJobParser {
 }
 
 class SparkClusterProcess(implicit system: ActorSystem) extends SparkProcess
-  with SparkClusterRestClient with SparkClusterJobParser with SparkClusterProcessMonitor {
+  with SparkClusterJobParser with SparkClusterProcessMonitor {
 
   import SparkClusterProcess._
   import SparkClusterJsonProtocol._
-  import spray.httpx.SprayJsonSupport._
 
   implicit lazy val ec: ExecutionContext = system.dispatcher
+  implicit val materializer = ActorMaterializer()
   lazy val completionPromise = Promise[TerminalStatus]()
   lazy val monitorPromise = Promise[Unit]()
   val tag = this.getClass.getSimpleName
   lazy val logger = Logger(LoggerFactory.getLogger(getClass.getName))
 
-  override def sendAndReceive: SendReceive = sendReceive
-
   override def startMonitoringSparkClusterJob(jobPath: Path, jsonFile: String): Future[TerminalStatus] = {
     Future(parseJsonForSubmissionIdAndStatus(jobPath.resolve(jsonFile))) onComplete {
       case Success(resp: SparkJobSubmissionResponse) =>
@@ -130,15 +125,13 @@ class SparkClusterProcess(implicit system: ActorSystem) extends SparkProcess
     }
 
     val request = sparkClusterMasterHostName match {
-      case Some(master) =>
-        Get(s"http://$master:6066/v1/submissions/status/$subId")
-      case None =>
-        Get(s"http://spark-master:6066/v1/submissions/status/$subId")
+      case Some(master) => HttpRequest(uri = s"http://$master:6066/v1/submissions/status/$subId")
+      case None => HttpRequest(uri = s"http://spark-master:6066/v1/submissions/status/$subId")
     }
 
     makeHttpRequest(request) flatMap { v =>
       v.status match {
-        case StatusCodes.OK => Future(v ~> unmarshal[SparkDriverStateQueryResponse])
+        case StatusCodes.OK => Unmarshal(v).to[SparkDriverStateQueryResponse]
         case _ =>
           val msg = s"Unexpected response received in response from Spark rest api. Response: $v"
           logger.error("{} reason: {}", tag, msg)
@@ -163,8 +156,8 @@ class SparkClusterProcess(implicit system: ActorSystem) extends SparkProcess
     JsonParser(line).convertTo[SparkJobSubmissionResponse]
   }
 
-  override def makeHttpRequest(httpRequest: HttpRequest): Future[HttpResponse] = {
+  def makeHttpRequest(httpRequest: HttpRequest): Future[HttpResponse] = {
     val headers = httpRequest.headers
-    sendAndReceive(httpRequest.withHeaders(headers))
+    Http().singleRequest(httpRequest.withHeaders(headers))
   }
 }
diff --git a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkInitializationActor.scala b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkInitializationActor.scala
index 1ff66aa24..d2f47c3e6 100644
--- a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkInitializationActor.scala
+++ b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkInitializationActor.scala
@@ -6,9 +6,10 @@ import cromwell.backend.validation.RuntimeAttributesDefault
 import cromwell.backend.validation.RuntimeAttributesKeys._
 import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendWorkflowDescriptor, BackendWorkflowInitializationActor}
 import cromwell.core.WorkflowOptions
-import wdl4s.TaskCall
-import wdl4s.types.{WdlBooleanType, WdlIntegerType, WdlStringType}
-import wdl4s.values.WdlValue
+import cromwell.core.Dispatcher.BackendDispatcher
+import wdl4s.wdl.WdlTaskCall
+import wdl4s.wdl.types.{WdlBooleanType, WdlIntegerType, WdlStringType}
+import wdl4s.wdl.values.WdlValue
 
 import scala.concurrent.Future
 import scala.util.Try
@@ -18,14 +19,14 @@ object SparkInitializationActor {
     SparkRuntimeAttributes.NumberOfExecutorsKey, SparkRuntimeAttributes.AppMainClassKey)
 
   def props(workflowDescriptor: BackendWorkflowDescriptor,
-            calls: Set[TaskCall],
+            calls: Set[WdlTaskCall],
             configurationDescriptor: BackendConfigurationDescriptor,
             serviceRegistryActor: ActorRef): Props =
-    Props(new SparkInitializationActor(workflowDescriptor, calls, configurationDescriptor, serviceRegistryActor))
+    Props(new SparkInitializationActor(workflowDescriptor, calls, configurationDescriptor, serviceRegistryActor)).withDispatcher(BackendDispatcher)
 }
 
 class SparkInitializationActor(override val workflowDescriptor: BackendWorkflowDescriptor,
-                               override val calls: Set[TaskCall],
+                               override val calls: Set[WdlTaskCall],
                                override val configurationDescriptor: BackendConfigurationDescriptor,
                                override val serviceRegistryActor: ActorRef) extends BackendWorkflowInitializationActor {
 
diff --git a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkJobExecutionActor.scala b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkJobExecutionActor.scala
index 80c73baa3..4b0f9700b 100644
--- a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkJobExecutionActor.scala
+++ b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkJobExecutionActor.scala
@@ -44,7 +44,7 @@ class SparkJobExecutionActor(override val jobDescriptor: BackendJobDescriptor,
   private val sparkDeployMode = configurationDescriptor.backendConfig.getString("deployMode").toLowerCase
   override val sharedFileSystemConfig = fileSystemsConfig.getConfig("local")
   private val workflowDescriptor = jobDescriptor.workflowDescriptor
-  private val jobPaths = new JobPathsWithDocker(jobDescriptor.key, workflowDescriptor, configurationDescriptor.backendConfig)
+  private val jobPaths = JobPathsWithDocker(jobDescriptor.key, workflowDescriptor, configurationDescriptor.backendConfig)
 
   // Files
   private val executionDir = jobPaths.callExecutionRoot
@@ -133,7 +133,7 @@ class SparkJobExecutionActor(override val jobDescriptor: BackendJobDescriptor,
 
   private def processSuccess(rc: Int) = {
     evaluateOutputs(callEngineFunction, outputMapper(jobPaths)) match {
-      case Success(outputs) => JobSucceededResponse(jobDescriptor.key, Some(rc), outputs, None, Seq.empty)
+      case Success(outputs) => JobSucceededResponse(jobDescriptor.key, Some(rc), outputs, None, Seq.empty, dockerImageUsed = None)
       case Failure(e) =>
         val message = Option(e.getMessage) map {
           ": " + _
diff --git a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkProcess.scala b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkProcess.scala
index 74ac26560..a3576ea83 100644
--- a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkProcess.scala
+++ b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkProcess.scala
@@ -42,7 +42,7 @@ echo $$? > rc
   def sparkSubmitCommand(attributes: Map[String, Any]): String = {
     val sparkHome = Try(sys.env("SPARK_HOME")) match {
       case Success(s) => Option(s)
-      case Failure(ex) =>
+      case Failure(_) =>
         logger.warn(s"Spark home does not exist picking up default command")
         None
     }
diff --git a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkRuntimeAttributes.scala b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkRuntimeAttributes.scala
index 8df2b34b4..073069c52 100644
--- a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkRuntimeAttributes.scala
+++ b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkRuntimeAttributes.scala
@@ -10,8 +10,8 @@ import cromwell.backend.validation.RuntimeAttributesValidation._
 import cromwell.core._
 import lenthall.validation.ErrorOr._
 import lenthall.exception.MessageAggregation
-import wdl4s.types.{WdlBooleanType, WdlIntegerType, WdlStringType, WdlType}
-import wdl4s.values.{WdlBoolean, WdlInteger, WdlString, WdlValue}
+import wdl4s.wdl.types.{WdlBooleanType, WdlIntegerType, WdlStringType, WdlType}
+import wdl4s.wdl.values.{WdlBoolean, WdlInteger, WdlString, WdlValue}
 
 
 object SparkRuntimeAttributes {
diff --git a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkClusterProcessSpec.scala b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkClusterProcessSpec.scala
index d03779f48..07301fad5 100644
--- a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkClusterProcessSpec.scala
+++ b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkClusterProcessSpec.scala
@@ -1,5 +1,6 @@
 package cromwell.backend.impl.spark
 
+import akka.http.scaladsl.model._
 import akka.testkit.ImplicitSender
 import cromwell.backend.impl.spark.SparkClusterProcess.SparkClusterJsonProtocol._
 import cromwell.backend.impl.spark.SparkClusterProcess._
@@ -13,7 +14,6 @@ import org.scalatest.concurrent.PatienceConfiguration.Timeout
 import org.scalatest.concurrent.ScalaFutures
 import org.scalatest.mockito.MockitoSugar
 import org.scalatest.{BeforeAndAfter, Matchers, WordSpecLike}
-import spray.http._
 import spray.json._
 
 import scala.concurrent.duration._
@@ -73,9 +73,9 @@ class SparkClusterProcessSpec extends TestKitSuite("SparkClusterProcess")
   private val mockRunningClusterResponse = SparkDriverStateQueryResponse(action = "SubmissionStatusResponse", driverState = "RUNNING", serverSparkVersion = "1.6.1",
     submissionId = "driver-20160803181054-0000", success = true, workerHostPort = "10.0.1.55:43834", workerId = "worker-20160801162431-10.0.1.55-43834")
 
-  private val mockSuccessHttpResponse = HttpResponse(StatusCodes.OK, HttpEntity(ContentTypes.`application/json`, mockSuccessClusterResponse.toJson.toString))
-  private val mockRunningHttpResponse = HttpResponse(StatusCodes.OK, HttpEntity(ContentTypes.`application/json`, mockRunningClusterResponse.toJson.toString))
-  private val mockFailedHttpResponse = HttpResponse(StatusCodes.OK, HttpEntity(ContentTypes.`application/json`, mockFailedClusterResponse.toJson.toString))
+  private val mockSuccessHttpResponse = HttpResponse(StatusCodes.OK, entity = HttpEntity(ContentTypes.`application/json`, mockSuccessClusterResponse.toJson.toString))
+  private val mockRunningHttpResponse = HttpResponse(StatusCodes.OK, entity = HttpEntity(ContentTypes.`application/json`, mockRunningClusterResponse.toJson.toString))
+  private val mockFailedHttpResponse = HttpResponse(StatusCodes.OK, entity = HttpEntity(ContentTypes.`application/json`, mockFailedClusterResponse.toJson.toString))
   private val mockBadHttpResponse = HttpResponse(StatusCodes.BadRequest)
 
   "SparkCluster process" should {
diff --git a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkInitializationActorSpec.scala b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkInitializationActorSpec.scala
index de58b7c06..1eac555bc 100644
--- a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkInitializationActorSpec.scala
+++ b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkInitializationActorSpec.scala
@@ -3,10 +3,10 @@ package cromwell.backend.impl.spark
 import akka.testkit.{EventFilter, ImplicitSender, TestDuration}
 import cromwell.backend.BackendSpec._
 import cromwell.backend.BackendWorkflowInitializationActor.Initialize
-import cromwell.backend.{BackendConfigurationDescriptor, BackendWorkflowDescriptor}
+import cromwell.backend.{BackendConfigurationDescriptor, BackendWorkflowDescriptor, TestConfig}
 import cromwell.core.TestKitSuite
 import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpecLike}
-import wdl4s._
+import wdl4s.wdl._
 
 import scala.concurrent.duration._
 
@@ -33,7 +33,7 @@ class SparkInitializationActorSpec  extends  TestKitSuite("SparkInitializationAc
       |}
     """.stripMargin
 
-  private def getSparkBackend(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[TaskCall], conf: BackendConfigurationDescriptor) = {
+  private def getSparkBackend(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[WdlTaskCall], conf: BackendConfigurationDescriptor) = {
     system.actorOf(SparkInitializationActor.props(workflowDescriptor, calls, conf, emptyActor))
   }
 
@@ -42,7 +42,7 @@ class SparkInitializationActorSpec  extends  TestKitSuite("SparkInitializationAc
       within(Timeout) {
         EventFilter.warning(message = s"Key/s [memory] is/are not supported by SparkBackend. Unsupported attributes will not be part of jobs executions.", occurrences = 1) intercept {
           val workflowDescriptor = buildWorkflowDescriptor(HelloWorld, runtime = """runtime { memory: 1 %s: "%s"}""".format("appMainClass", "test"))
-          val backend = getSparkBackend(workflowDescriptor, workflowDescriptor.workflow.taskCalls, emptyBackendConfig)
+          val backend = getSparkBackend(workflowDescriptor, workflowDescriptor.workflow.taskCalls, TestConfig.emptyBackendConfigDescriptor)
           backend ! Initialize
         }
       }
diff --git a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkJobExecutionActorSpec.scala b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkJobExecutionActorSpec.scala
index 0020977f8..ac98705df 100644
--- a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkJobExecutionActorSpec.scala
+++ b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkJobExecutionActorSpec.scala
@@ -16,8 +16,8 @@ import org.mockito.Mockito._
 import org.scalatest.concurrent.PatienceConfiguration.Timeout
 import org.scalatest.mockito.MockitoSugar
 import org.scalatest.{BeforeAndAfter, Matchers, WordSpecLike}
-import wdl4s._
-import wdl4s.values.WdlValue
+import wdl4s.wdl._
+import wdl4s.wdl.values.WdlValue
 
 import scala.concurrent.Future
 import scala.concurrent.duration._
@@ -438,15 +438,15 @@ class SparkJobExecutionActorSpec extends TestKitSuite("SparkJobExecutionActor")
   }
 
   private def cleanUpJob(jobPaths: JobPathsWithDocker): Unit = {
-    File(jobPaths.workflowRoot).delete(true)
+    File(jobPaths.workflowPaths.workflowRoot).delete(true)
     ()
   }
 
-  private def prepareJob(wdlSource: WdlSource = helloWorldWdl, runtimeString: String = passOnStderr, inputFiles: Option[Map[String, WdlValue]] = None, isCluster: Boolean = false): TestJobDescriptor = {
-    val backendWorkflowDescriptor = buildWorkflowDescriptor(wdl = wdlSource, inputs = inputFiles.getOrElse(Map.empty), runtime = runtimeString)
+  private def prepareJob(workflowSource: WorkflowSource = helloWorldWdl, runtimeString: String = passOnStderr, inputFiles: Option[Map[String, WdlValue]] = None, isCluster: Boolean = false): TestJobDescriptor = {
+    val backendWorkflowDescriptor = buildWorkflowDescriptor(workflowSource = workflowSource, inputs = inputFiles.getOrElse(Map.empty), runtime = runtimeString)
     val backendConfigurationDescriptor = if (isCluster) BackendConfigurationDescriptor(backendClusterConfig, ConfigFactory.load) else BackendConfigurationDescriptor(backendClientConfig, ConfigFactory.load)
     val jobDesc = jobDescriptorFromSingleCallWorkflow(backendWorkflowDescriptor, inputFiles.getOrElse(Map.empty), WorkflowOptions.empty, Set.empty)
-    val jobPaths = if (isCluster) new JobPathsWithDocker(jobDesc.key, backendWorkflowDescriptor, backendClusterConfig) else new JobPathsWithDocker(jobDesc.key, backendWorkflowDescriptor, backendClientConfig)
+    val jobPaths = if (isCluster) JobPathsWithDocker(jobDesc.key, backendWorkflowDescriptor, backendClusterConfig) else JobPathsWithDocker(jobDesc.key, backendWorkflowDescriptor, backendClientConfig)
     val executionDir = jobPaths.callExecutionRoot
     val stdout = File(executionDir.toString, "stdout")
     stdout.createIfNotExists(asDirectory = false, createParents = true)
diff --git a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkRuntimeAttributesSpec.scala b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkRuntimeAttributesSpec.scala
index d7c0340d8..d570015c3 100644
--- a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkRuntimeAttributesSpec.scala
+++ b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkRuntimeAttributesSpec.scala
@@ -7,10 +7,10 @@ import cromwell.core.{WorkflowId, WorkflowOptions}
 import lenthall.util.TryUtil
 import org.scalatest.{Matchers, WordSpecLike}
 import spray.json.{JsBoolean, JsNumber, JsObject, JsValue}
-import wdl4s.WdlExpression._
-import wdl4s.expression.NoFunctions
-import wdl4s.values.WdlValue
-import wdl4s.{Call, _}
+import wdl4s.wdl.WdlExpression._
+import wdl4s.wdl.expression.NoFunctions
+import wdl4s.wdl.values.WdlValue
+import wdl4s.wdl._
 
 class SparkRuntimeAttributesSpec extends WordSpecLike with Matchers {
 
@@ -85,7 +85,7 @@ class SparkRuntimeAttributesSpec extends WordSpecLike with Matchers {
 
   }
 
-  private def buildWorkflowDescriptor(wdl: WdlSource,
+  private def buildWorkflowDescriptor(wdl: WorkflowSource,
                                       inputs: Map[String, WdlValue] = Map.empty,
                                       options: WorkflowOptions = WorkflowOptions(JsObject(Map.empty[String, JsValue])),
                                       runtime: String) = {
@@ -98,10 +98,10 @@ class SparkRuntimeAttributesSpec extends WordSpecLike with Matchers {
     )
   }
 
-  private def createRuntimeAttributes(wdlSource: WdlSource, runtimeAttributes: String) = {
-    val workflowDescriptor = buildWorkflowDescriptor(wdlSource, runtime = runtimeAttributes)
+  private def createRuntimeAttributes(workflowSource: WorkflowSource, runtimeAttributes: String) = {
+    val workflowDescriptor = buildWorkflowDescriptor(workflowSource, runtime = runtimeAttributes)
 
-    def createLookup(call: Call): ScopedLookupFunction = {
+    def createLookup(call: WdlCall): ScopedLookupFunction = {
       val knownInputs = workflowDescriptor.knownValues
       call.lookupFunction(knownInputs, NoFunctions)
     }
diff --git a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesAsyncBackendJobExecutionActor.scala b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesAsyncBackendJobExecutionActor.scala
index 67c45f175..1e8b69c7e 100644
--- a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesAsyncBackendJobExecutionActor.scala
+++ b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesAsyncBackendJobExecutionActor.scala
@@ -2,21 +2,23 @@ package cromwell.backend.impl.tes
 
 import java.nio.file.FileAlreadyExistsException
 
+import akka.http.scaladsl.Http
+import akka.http.scaladsl.model._
 import cromwell.backend.BackendJobLifecycleActor
 import cromwell.backend.async.{ExecutionHandle, FailedNonRetryableExecutionHandle, PendingExecutionHandle}
 import cromwell.backend.impl.tes.TesResponseJsonFormatter._
 import cromwell.backend.standard.{StandardAsyncExecutionActor, StandardAsyncExecutionActorParams, StandardAsyncJob}
 import cromwell.core.path.{DefaultPathBuilder, Path}
 import cromwell.core.retry.SimpleExponentialBackoff
-import spray.client.pipelining._
-import spray.http.HttpRequest
-import spray.httpx.SprayJsonSupport._
-import spray.httpx.unmarshalling._
-import wdl4s.expression.NoFunctions
-import wdl4s.values.WdlFile
+import wdl4s.wdl.expression.NoFunctions
+import wdl4s.wdl.values.WdlFile
+import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._
+import akka.http.scaladsl.marshalling.Marshal
+import akka.http.scaladsl.unmarshalling.{Unmarshal, Unmarshaller}
+import akka.stream.ActorMaterializer
 
 import scala.concurrent.duration._
-import scala.concurrent.{ExecutionContext, Future}
+import scala.concurrent.Future
 import scala.language.postfixOps
 import scala.util.{Failure, Success}
 
@@ -42,6 +44,8 @@ object TesAsyncBackendJobExecutionActor {
 
 class TesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyncExecutionActorParams)
   extends BackendJobLifecycleActor with StandardAsyncExecutionActor with TesJobCachingActorHelper {
+  implicit val actorSystem = context.system
+  implicit val materializer = ActorMaterializer()
 
   override type StandardAsyncRunInfo = Any
 
@@ -58,21 +62,20 @@ class TesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn
     maxInterval = 30 seconds,
     multiplier = 1.1
   )
-
-  override lazy val retryable: Boolean = false
+  
+  private lazy val realDockerImageUsed: String = jobDescriptor.maybeCallCachingEligible.dockerHash.getOrElse(runtimeAttributes.dockerImage)
+  override lazy val dockerImageUsed: Option[String] = Option(realDockerImageUsed)
 
   private val tesEndpoint = workflowDescriptor.workflowOptions.getOrElse("endpoint", tesConfiguration.endpointURL)
 
-  override lazy val jobTag = jobDescriptor.key.tag
-
-  private def pipeline[T: FromResponseUnmarshaller]: HttpRequest => Future[T] = sendReceive ~> unmarshal[T]
+  override lazy val jobTag: String = jobDescriptor.key.tag
 
   // Utility for converting a WdlValue so that the path is localized to the
   // container's filesystem.
   override def mapCommandLineWdlFile(wdlFile: WdlFile): WdlFile = {
     val localPath = DefaultPathBuilder.get(wdlFile.valueString).toAbsolutePath
     localPath match {
-      case p if p.startsWith(tesJobPaths.DockerRoot) =>
+      case p if p.startsWith(tesJobPaths.workflowPaths.DockerRoot) =>
         val containerPath = p.pathAsString
         WdlFile(containerPath)
       case p if p.startsWith(tesJobPaths.callExecutionRoot) =>
@@ -91,37 +94,40 @@ class TesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn
     }
   }
 
-  def createTaskMessage(): TesTaskMessage = {
-    val task = TesTask(jobDescriptor, configurationDescriptor, jobLogger, tesJobPaths, runtimeAttributes, commandDirectory, backendEngineFunctions)
-
-    tesJobPaths.script.write(commandScriptContents)
+  def createTaskMessage(): Task = {
+    val task = TesTask(jobDescriptor, configurationDescriptor, jobLogger, tesJobPaths,
+      runtimeAttributes, commandDirectory, commandScriptContents, backendEngineFunctions,
+      realDockerImageUsed)
 
-    TesTaskMessage(
+    Task(
+      None,
+      None,
       Option(task.name),
       Option(task.description),
       Option(task.project),
-      Option(task.inputs),
+      Option(task.inputs(commandLineValueMapper)),
       Option(task.outputs),
-      task.resources,
-      task.dockerExecutor
+      Option(task.resources),
+      task.executors,
+      None,
+      None,
+      None
     )
   }
 
-  override def executeAsync()(implicit ec: ExecutionContext): Future[ExecutionHandle] = {
+  override def executeAsync(): Future[ExecutionHandle] = {
     // create call exec dir
     tesJobPaths.callExecutionRoot.createPermissionedDirectories()
     val taskMessage = createTaskMessage()
 
-    val submitTask = pipeline[TesPostResponse]
-      .apply(Post(tesEndpoint, taskMessage))
-
-    submitTask.map {
-      response =>
-        val jobID = response.value
-        PendingExecutionHandle(jobDescriptor, StandardAsyncJob(jobID), None, previousStatus = None)
-    }
+    for {
+      entity <- Marshal(taskMessage).to[RequestEntity]
+      ctr <- makeRequest[CreateTaskResponse](HttpRequest(method = HttpMethods.POST, uri = tesEndpoint, entity = entity))
+    } yield PendingExecutionHandle(jobDescriptor, StandardAsyncJob(ctr.id), None, previousStatus = None)
   }
 
+  override def recoverAsync(jobId: StandardAsyncJob) = executeAsync()
+
   override def tryAbort(job: StandardAsyncJob): Unit = {
 
     val returnCodeTmp = jobPaths.returnCode.plusExt("kill")
@@ -134,31 +140,28 @@ class TesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn
         returnCodeTmp.delete(true)
     }
 
-    val abortRequest = pipeline[TesGetResponse]
-      .apply(Delete(s"$tesEndpoint/${job.jobId}"))
-    abortRequest onComplete {
+    makeRequest[CancelTaskResponse](HttpRequest(method = HttpMethods.POST, uri = s"$tesEndpoint/${job.jobId}:cancel")) onComplete {
       case Success(_) => jobLogger.info("{} Aborted {}", tag: Any, job.jobId)
       case Failure(ex) => jobLogger.warn("{} Failed to abort {}: {}", tag, job.jobId, ex.getMessage)
     }
+
     ()
   }
 
-  override def pollStatusAsync(handle: StandardAsyncPendingExecutionHandle)(implicit ec: ExecutionContext): Future[TesRunStatus] = {
-    val pollTask = pipeline[TesGetResponse].apply(Get(s"$tesEndpoint/${handle.pendingJob.jobId}"))
-
-    pollTask.map {
+  override def pollStatusAsync(handle: StandardAsyncPendingExecutionHandle): Future[TesRunStatus] = {
+    makeRequest[MinimalTaskView](HttpRequest(uri = s"$tesEndpoint/${handle.pendingJob.jobId}?view=MINIMAL")) map {
       response =>
         val state = response.state
         state match {
-          case s if s.contains("Complete") =>
+          case s if s.contains("COMPLETE") =>
             jobLogger.info(s"Job ${handle.pendingJob.jobId} is complete")
             Complete
 
-          case s if s.contains("Cancel") =>
+          case s if s.contains("CANCELED") =>
             jobLogger.info(s"Job ${handle.pendingJob.jobId} was canceled")
             FailedOrError
 
-          case s if s.contains("Error") =>
+          case s if s.contains("ERROR") =>
             jobLogger.info(s"TES reported an error for Job ${handle.pendingJob.jobId}")
             FailedOrError
 
@@ -183,21 +186,25 @@ class TesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn
       case _ => false
     }
   }
-
-  private def hostAbsoluteFilePath(wdlFile: WdlFile): Path = {
-    tesJobPaths.callExecutionRoot.resolve(wdlFile.value)
-  }
   
   private val outputWdlFiles: Seq[WdlFile] = jobDescriptor.call.task
     .findOutputFiles(jobDescriptor.fullyQualifiedInputs, NoFunctions)
     .filter(o => !DefaultPathBuilder.get(o.valueString).isAbsolute)
 
   override def mapOutputWdlFile(wdlFile: WdlFile): WdlFile = {
+    val absPath: Path = tesJobPaths.callExecutionRoot.resolve(wdlFile.valueString)
     wdlFile match {
-      case fileNotFound if !hostAbsoluteFilePath(fileNotFound).exists && outputWdlFiles.contains(fileNotFound) =>
+      case fileNotFound if !absPath.exists && outputWdlFiles.contains(fileNotFound) =>
         throw new RuntimeException("Could not process output, file not found: " +
-          s"${hostAbsoluteFilePath(wdlFile).pathAsString}")
-      case _ => WdlFile(hostAbsoluteFilePath(wdlFile).pathAsString)
+          s"${absPath.pathAsString}")
+      case _ => WdlFile(absPath.pathAsString)
     }
   }
+
+  private def makeRequest[A](request: HttpRequest)(implicit um: Unmarshaller[ResponseEntity, A]): Future[A] = {
+    for {
+      response <- Http().singleRequest(request)
+      data <- Unmarshal(response.entity).to[A]
+    } yield data
+  }
 }
diff --git a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesBackendFileHashing.scala b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesBackendFileHashing.scala
deleted file mode 100644
index d6f8d3454..000000000
--- a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesBackendFileHashing.scala
+++ /dev/null
@@ -1,21 +0,0 @@
-package cromwell.backend.impl.tes
-
-import akka.event.LoggingAdapter
-import cromwell.backend.callcaching.FileHashingActor.SingleFileHashRequest
-import cromwell.core.path.DefaultPathBuilder
-import cromwell.util.TryWithResource._
-
-import scala.language.postfixOps
-import scala.util.Try
-
-private[tes] object TesBackendFileHashing {
-  def getMd5Result(request: SingleFileHashRequest, log: LoggingAdapter): Try[String] = {
-    val path = DefaultPathBuilder.build(request.file.valueString) recover {
-      case failure => throw new RuntimeException("Failed to construct path to hash", failure)
-    } get
-
-    tryWithResource(() => path.newInputStream) { inputStream =>
-      org.apache.commons.codec.digest.DigestUtils.md5Hex(inputStream)
-    }
-  }
-}
diff --git a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesBackendLifecycleActorFactory.scala b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesBackendLifecycleActorFactory.scala
index 41c03f083..d4b18a35b 100644
--- a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesBackendLifecycleActorFactory.scala
+++ b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesBackendLifecycleActorFactory.scala
@@ -2,11 +2,10 @@ package cromwell.backend.impl.tes
 
 import akka.actor.ActorRef
 import cromwell.backend._
-import cromwell.backend.callcaching.FileHashingActor.FileHashingFunction
 import cromwell.backend.standard._
 import cromwell.core.JobExecutionToken.JobExecutionTokenType
 import net.ceedubs.ficus.Ficus._
-import wdl4s.TaskCall
+import wdl4s.wdl.WdlTaskCall
 
 case class TesBackendLifecycleActorFactory(name: String, configurationDescriptor: BackendConfigurationDescriptor)
   extends StandardLifecycleActorFactory {
@@ -25,10 +24,8 @@ case class TesBackendLifecycleActorFactory(name: String, configurationDescriptor
     JobExecutionTokenType(name, concurrentJobLimit)
   }
 
-  override def workflowInitializationActorParams(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[TaskCall],
-                                                 serviceRegistryActor: ActorRef): StandardInitializationActorParams = {
+  override def workflowInitializationActorParams(workflowDescriptor: BackendWorkflowDescriptor, ioActor: ActorRef, calls: Set[WdlTaskCall],
+                                                 serviceRegistryActor: ActorRef, restarting: Boolean): StandardInitializationActorParams = {
     TesInitializationActorParams(workflowDescriptor, calls, tesConfiguration, serviceRegistryActor)
   }
-
-  override lazy val fileHashingFunction: Option[FileHashingFunction] = Option(FileHashingFunction(TesBackendFileHashing.getMd5Result))
 }
diff --git a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesConfiguration.scala b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesConfiguration.scala
index 65f4deec6..9a499dde5 100644
--- a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesConfiguration.scala
+++ b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesConfiguration.scala
@@ -4,4 +4,5 @@ import cromwell.backend.BackendConfigurationDescriptor
 
 class TesConfiguration(val configurationDescriptor: BackendConfigurationDescriptor) {
   val endpointURL = configurationDescriptor.backendConfig.getString("endpoint")
+  val runtimeConfig = configurationDescriptor.backendRuntimeConfig
 }
diff --git a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesInitializationActor.scala b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesInitializationActor.scala
index 773522f43..dc055c898 100644
--- a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesInitializationActor.scala
+++ b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesInitializationActor.scala
@@ -1,17 +1,24 @@
 package cromwell.backend.impl.tes
 
 import akka.actor.ActorRef
+import cats.data.Validated.{Invalid, Valid}
+import cats.instances.future._
+import cats.instances.list._
+import cats.syntax.traverse._
 import cromwell.backend.standard._
 import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendWorkflowDescriptor}
-import wdl4s.TaskCall
+import cromwell.core.path.{DefaultPathBuilder, PathBuilder}
+import cromwell.filesystems.gcs.{GcsPathBuilderFactory, GoogleConfiguration}
+import lenthall.exception.MessageAggregation
+import net.ceedubs.ficus.Ficus._
+import wdl4s.wdl.WdlTaskCall
 
 import scala.concurrent.Future
-import scala.util.Try
 
 case class TesInitializationActorParams
 (
   workflowDescriptor: BackendWorkflowDescriptor,
-  calls: Set[TaskCall],
+  calls: Set[WdlTaskCall],
   tesConfiguration: TesConfiguration,
   serviceRegistryActor: ActorRef
 ) extends StandardInitializationActorParams {
@@ -22,18 +29,41 @@ class TesInitializationActor(params: TesInitializationActorParams)
   extends StandardInitializationActor(params) {
 
   private val tesConfiguration = params.tesConfiguration
+  
+  private implicit val system = context.system
 
-  override lazy val workflowPaths: TesWorkflowPaths =
-    new TesWorkflowPaths(workflowDescriptor, tesConfiguration.configurationDescriptor.backendConfig, pathBuilders)
+  /**
+    * If the backend sets a gcs authentication mode, try to create a PathBuilderFactory with it.
+    */
+  lazy val gcsPathBuilderFactory: Option[GcsPathBuilderFactory] = {
+    configurationDescriptor.backendConfig.as[Option[String]]("filesystems.gcs.auth") map { configAuth =>
+      val googleConfiguration = GoogleConfiguration(configurationDescriptor.globalConfig)
+      googleConfiguration.auth(configAuth) match {
+        case Valid(auth) => GcsPathBuilderFactory(auth, googleConfiguration.applicationName)
+        case Invalid(error) => throw new MessageAggregation {
+          override def exceptionContext: String = "Failed to parse gcs auth configuration"
+
+          override def errorMessages: Traversable[String] = error.toList
+        }
+      }
+    }
+  }
+
+  override lazy val pathBuilders: Future[List[PathBuilder]] =
+    gcsPathBuilderFactory.toList.traverse(_.withOptions(workflowDescriptor.workflowOptions)).map(_ ++ Option(DefaultPathBuilder))
+
+  override lazy val workflowPaths: Future[TesWorkflowPaths] = pathBuilders map {
+    new TesWorkflowPaths(workflowDescriptor, tesConfiguration.configurationDescriptor.backendConfig, _)
+  }
 
   override lazy val runtimeAttributesBuilder: StandardValidatedRuntimeAttributesBuilder =
-    TesRuntimeAttributes.runtimeAttributesBuilder
+    TesRuntimeAttributes.runtimeAttributesBuilder(tesConfiguration.runtimeConfig)
 
   override def beforeAll(): Future[Option[BackendInitializationData]] = {
-    Future.fromTry(Try {
-      publishWorkflowRoot(workflowPaths.workflowRoot.toString)
-      workflowPaths.workflowRoot.createPermissionedDirectories()
-      Option(TesBackendInitializationData(workflowPaths, runtimeAttributesBuilder, tesConfiguration))
-    })
+    workflowPaths map { paths =>
+      publishWorkflowRoot(paths.workflowRoot.toString)
+      paths.workflowRoot.createPermissionedDirectories()
+      Option(TesBackendInitializationData(paths, runtimeAttributesBuilder, tesConfiguration))
+    }
   }
 }
diff --git a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesJobCachingActorHelper.scala b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesJobCachingActorHelper.scala
index 0f12d802f..58828915f 100644
--- a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesJobCachingActorHelper.scala
+++ b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesJobCachingActorHelper.scala
@@ -18,5 +18,5 @@ trait TesJobCachingActorHelper extends StandardCachingActorHelper {
 
   lazy val tesConfiguration: TesConfiguration = initializationData.tesConfiguration
 
-  lazy val runtimeAttributes = TesRuntimeAttributes(validatedRuntimeAttributes)
+  lazy val runtimeAttributes = TesRuntimeAttributes(validatedRuntimeAttributes, tesConfiguration.runtimeConfig)
 }
diff --git a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesJobPaths.scala b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesJobPaths.scala
index 928fd0fa8..b270d0864 100644
--- a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesJobPaths.scala
+++ b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesJobPaths.scala
@@ -1,38 +1,40 @@
 package cromwell.backend.impl.tes
 
 import com.typesafe.config.Config
-import cromwell.backend.io.{JobPaths, WorkflowPaths}
 import cromwell.backend.{BackendJobDescriptorKey, BackendWorkflowDescriptor}
-import cromwell.core.path.{DefaultPathBuilder, Path, PathBuilder}
+import cromwell.backend.io.{JobPaths, WorkflowPaths}
+import cromwell.core.path._
+
+object TesJobPaths {
+  def apply(jobKey: BackendJobDescriptorKey,
+  workflowDescriptor: BackendWorkflowDescriptor,
+  config: Config,
+  pathBuilders: List[PathBuilder] = WorkflowPaths.DefaultPathBuilders) = {
+    val workflowPaths = TesWorkflowPaths(workflowDescriptor, config, pathBuilders)
+    new TesJobPaths(workflowPaths, jobKey)
+  }
+}
 
-class TesJobPaths(val jobKey: BackendJobDescriptorKey,
-                  workflowDescriptor: BackendWorkflowDescriptor,
-                  config: Config,
-                  pathBuilders: List[PathBuilder] = WorkflowPaths.DefaultPathBuilders) extends TesWorkflowPaths(
-  workflowDescriptor, config, pathBuilders) with JobPaths {
+case class TesJobPaths private[tes] (override val workflowPaths: TesWorkflowPaths,
+                  jobKey: BackendJobDescriptorKey) extends JobPaths {
 
   import JobPaths._
 
   override lazy val callExecutionRoot = {
     callRoot.resolve("execution")
   }
-  val callDockerRoot = callPathBuilder(dockerWorkflowRoot, jobKey)
+  val callDockerRoot = callPathBuilder(workflowPaths.dockerWorkflowRoot, jobKey)
   val callExecutionDockerRoot = callDockerRoot.resolve("execution")
   val callInputsDockerRoot = callDockerRoot.resolve("inputs")
   val callInputsRoot = callRoot.resolve("inputs")
 
-  //TODO move to TesConfiguration
-  private def prefixScheme(path: String): String = "file://" + path
-
-  def storageInput(path: String): String = prefixScheme(path)
-
   // Given an output path, return a path localized to the storage file system
   def storageOutput(path: String): String = {
-    prefixScheme(callExecutionRoot.resolve(path).toString)
+    callExecutionRoot.resolve(path).toString
   }
 
   def containerInput(path: String): String = {
-    cleanContainerInputPath(callInputsDockerRoot, DefaultPathBuilder.get(path))
+    cleanContainerInputPath(callInputsDockerRoot, path)
   }
 
   // Given an output path, return a path localized to the container file system
@@ -46,12 +48,15 @@ class TesJobPaths(val jobKey: BackendJobDescriptorKey,
     cwd.resolve(path).toString
   }
 
-  private def cleanContainerInputPath(inputDir: Path, path: Path): String = {
-    path.toAbsolutePath match {
-      case p if p.startsWith(callExecutionRoot) =>
-        callExecutionDockerRoot.resolve(p.getFileName.toString).toString
+  private def cleanContainerInputPath(inputDir: Path, path: String): String = {
+    path match {
+      case p if p.startsWith("gs:") =>
+        inputDir.resolve(p.replaceFirst("gs:/?/?", "")).pathAsString
+      case p if p.startsWith(callExecutionRoot.pathAsString) =>
+        val f = DefaultPathBuilder.get(p)
+        callExecutionDockerRoot.resolve(f.name).pathAsString
       case p =>
-        inputDir + p.toString
+        inputDir.resolve(p).pathAsString
     }
   }
 }
diff --git a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesResponseJsonFormatter.scala b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesResponseJsonFormatter.scala
index 111f7e198..79a2be702 100644
--- a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesResponseJsonFormatter.scala
+++ b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesResponseJsonFormatter.scala
@@ -2,24 +2,20 @@ package cromwell.backend.impl.tes
 
 import spray.json._
 
-final case class TesPostResponse(value: String)
-
-final case class TesGetResponse(
-                                 jobID: String,
-                                 task: TesTaskMessage,
-                                 state: String,
-                                 logs: Option[Seq[JobLogs]]
-                               )
-
+final case class CreateTaskResponse(id: String)
+final case class MinimalTaskView(id: String, state: String)
+final case class CancelTaskResponse()
 
 object TesResponseJsonFormatter extends DefaultJsonProtocol {
-  implicit val volumeFormat = jsonFormat5(Volume)
   implicit val resourcesFormat = jsonFormat5(Resources)
   implicit val taskParameterFormat = jsonFormat6(TaskParameter)
   implicit val portsFormat = jsonFormat2(Ports)
-  implicit val dockerExecutorFormat = jsonFormat7(DockerExecutor)
-  implicit val tesTaskMessageFormat = jsonFormat7(TesTaskMessage)
-  implicit val jobLogsFormat = jsonFormat8(JobLogs)
-  implicit val tesPostResponseFormat = jsonFormat1(TesPostResponse)
-  implicit val tesGetResponseFormat = jsonFormat4(TesGetResponse)
+  implicit val executorFormat = jsonFormat8(Executor)
+  implicit val executorLogFormat = jsonFormat7(ExecutorLog)
+  implicit val outputFileLogFormat = jsonFormat3(OutputFileLog)
+  implicit val taskLogFormat = jsonFormat5(TaskLog)
+  implicit val taskFormat = jsonFormat12(Task)
+  implicit val minimalTaskView = jsonFormat2(MinimalTaskView)
+  implicit val createTaskResponseFormat = jsonFormat1(CreateTaskResponse)
+  implicit val cancelTaskResponseFormat = jsonFormat0(CancelTaskResponse)
 }
diff --git a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesRuntimeAttributes.scala b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesRuntimeAttributes.scala
index 06069c12b..b6b09446a 100644
--- a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesRuntimeAttributes.scala
+++ b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesRuntimeAttributes.scala
@@ -1,61 +1,59 @@
 package cromwell.backend.impl.tes
 
 import cats.syntax.validated._
+import com.typesafe.config.Config
 import cromwell.backend.MemorySize
 import cromwell.backend.standard.StandardValidatedRuntimeAttributesBuilder
 import cromwell.backend.validation._
 import lenthall.validation.ErrorOr.ErrorOr
-import wdl4s.parser.MemoryUnit
-import wdl4s.values.{WdlInteger, WdlString, WdlValue}
+import wdl4s.wdl.values.{WdlString, WdlValue}
 
 case class TesRuntimeAttributes(continueOnReturnCode: ContinueOnReturnCode,
                                 dockerImage: String,
                                 dockerWorkingDir: Option[String],
                                 failOnStderr: Boolean,
-                                cpu: Int,
-                                memory: MemorySize,
-                                disk: MemorySize)
+                                cpu: Option[Int],
+                                memory: Option[MemorySize],
+                                disk: Option[MemorySize])
 
 object TesRuntimeAttributes {
 
   val DockerWorkingDirKey = "dockerWorkingDir"
+  val DiskSizeKey = "disk"
 
-  private val MemoryDefaultValue = "2 GB"
+  private def cpuValidation(runtimeConfig: Option[Config]): OptionalRuntimeAttributesValidation[Int] = CpuValidation.optional
 
-  val DiskSizeKey = "disk"
-  private val DiskSizeDefaultValue = "2 GB"
+  private def failOnStderrValidation(runtimeConfig: Option[Config]) = FailOnStderrValidation.default(runtimeConfig)
 
-  private val cpuValidation: RuntimeAttributesValidation[Int] = CpuValidation.default
+  private def continueOnReturnCodeValidation(runtimeConfig: Option[Config]) = ContinueOnReturnCodeValidation.default(runtimeConfig)
 
-  private val diskSizeValidation: RuntimeAttributesValidation[MemorySize] =
-    DiskSizeValidation.withDefaultDiskSize(MemorySize.parse(DiskSizeDefaultValue).get)
+  private def diskSizeValidation(runtimeConfig: Option[Config]): OptionalRuntimeAttributesValidation[MemorySize] = MemoryValidation.optional(DiskSizeKey)
+
+  private def memoryValidation(runtimeConfig: Option[Config]): OptionalRuntimeAttributesValidation[MemorySize] = MemoryValidation.optional(RuntimeAttributesKeys.MemoryKey)
 
   private val dockerValidation: RuntimeAttributesValidation[String] = DockerValidation.instance
 
   private val dockerWorkingDirValidation: OptionalRuntimeAttributesValidation[String] = DockerWorkingDirValidation.optional
 
-  private val memoryValidation: RuntimeAttributesValidation[MemorySize] =
-    MemoryValidation.withDefaultMemory(MemorySize.parse(MemoryDefaultValue).get)
-
-  def runtimeAttributesBuilder: StandardValidatedRuntimeAttributesBuilder =
-    StandardValidatedRuntimeAttributesBuilder.default.withValidation(
-      cpuValidation,
-      memoryValidation,
-      diskSizeValidation,
+  def runtimeAttributesBuilder(backendRuntimeConfig: Option[Config]): StandardValidatedRuntimeAttributesBuilder =
+    StandardValidatedRuntimeAttributesBuilder.default(backendRuntimeConfig).withValidation(
+      cpuValidation(backendRuntimeConfig),
+      memoryValidation(backendRuntimeConfig),
+      diskSizeValidation(backendRuntimeConfig),
       dockerValidation,
       dockerWorkingDirValidation
     )
 
-  def apply(validatedRuntimeAttributes: ValidatedRuntimeAttributes): TesRuntimeAttributes = {
+  def apply(validatedRuntimeAttributes: ValidatedRuntimeAttributes, backendRuntimeConfig: Option[Config]): TesRuntimeAttributes = {
     val docker: String = RuntimeAttributesValidation.extract(dockerValidation, validatedRuntimeAttributes)
     val dockerWorkingDir: Option[String] = RuntimeAttributesValidation.extractOption(dockerWorkingDirValidation.key, validatedRuntimeAttributes)
-    val cpu: Int = RuntimeAttributesValidation.extract(cpuValidation, validatedRuntimeAttributes)
-    val memory: MemorySize = RuntimeAttributesValidation.extract(memoryValidation, validatedRuntimeAttributes)
-    val disk: MemorySize = RuntimeAttributesValidation.extract(diskSizeValidation, validatedRuntimeAttributes)
+    val cpu: Option[Int] = RuntimeAttributesValidation.extractOption(cpuValidation(backendRuntimeConfig).key, validatedRuntimeAttributes)
+    val memory: Option[MemorySize] = RuntimeAttributesValidation.extractOption(memoryValidation(backendRuntimeConfig).key, validatedRuntimeAttributes)
+    val disk: Option[MemorySize] = RuntimeAttributesValidation.extractOption(diskSizeValidation(backendRuntimeConfig).key, validatedRuntimeAttributes)
     val failOnStderr: Boolean =
-      RuntimeAttributesValidation.extract(FailOnStderrValidation.default, validatedRuntimeAttributes)
+      RuntimeAttributesValidation.extract(failOnStderrValidation(backendRuntimeConfig), validatedRuntimeAttributes)
     val continueOnReturnCode: ContinueOnReturnCode =
-      RuntimeAttributesValidation.extract(ContinueOnReturnCodeValidation.default, validatedRuntimeAttributes)
+      RuntimeAttributesValidation.extract(continueOnReturnCodeValidation(backendRuntimeConfig), validatedRuntimeAttributes)
 
     new TesRuntimeAttributes(
       continueOnReturnCode,
@@ -81,53 +79,3 @@ class DockerWorkingDirValidation extends StringRuntimeAttributesValidation(TesRu
   }
 }
 
-object DiskSizeValidation {
-  lazy val instance: RuntimeAttributesValidation[MemorySize] = new DiskSizeValidation
-  lazy val optional: OptionalRuntimeAttributesValidation[MemorySize] = instance.optional
-
-  def withDefaultDiskSize(memorySize: MemorySize): RuntimeAttributesValidation[MemorySize] =
-    instance.withDefault(WdlInteger(memorySize.bytes.toInt))
-
-  private val wrongAmountFormat =
-    s"Expecting ${TesRuntimeAttributes.DiskSizeKey} runtime attribute value greater than 0 but got %s"
-  private val wrongTypeFormat =
-    s"Expecting ${TesRuntimeAttributes.DiskSizeKey} runtime attribute to be an Integer or String with format '8 GB'." +
-      s" Exception: %s"
-
-  def validateDiskSizeString(wdlString: WdlString): ErrorOr[MemorySize] =
-    validateDiskSizeString(wdlString.value)
-
-  def validateDiskSizeString(value: String): ErrorOr[MemorySize] = {
-    MemorySize.parse(value) match {
-      case scala.util.Success(memorySize: MemorySize) if memorySize.amount > 0 =>
-        memorySize.to(MemoryUnit.GB).validNel
-      case scala.util.Success(memorySize: MemorySize) =>
-        wrongAmountFormat.format(memorySize.amount).invalidNel
-      case scala.util.Failure(throwable) =>
-        wrongTypeFormat.format(throwable.getMessage).invalidNel
-    }
-  }
-
-  def validateDiskSizeInteger(wdlInteger: WdlInteger): ErrorOr[MemorySize] =
-    validateDiskSizeInteger(wdlInteger.value)
-
-  def validateDiskSizeInteger(value: Int): ErrorOr[MemorySize] = {
-    if (value <= 0)
-      wrongAmountFormat.format(value).invalidNel
-    else
-      MemorySize(value.toDouble, MemoryUnit.Bytes).to(MemoryUnit.GB).validNel
-  }
-}
-
-class DiskSizeValidation extends MemoryValidation {
-  override def key = TesRuntimeAttributes.DiskSizeKey
-
-  override protected def validateValue: PartialFunction[WdlValue, ErrorOr[MemorySize]] = {
-    case WdlInteger(value) => DiskSizeValidation.validateDiskSizeInteger(value)
-    case WdlString(value) => DiskSizeValidation.validateDiskSizeString(value)
-  }
-
-  override def missingValueMessage: String = DiskSizeValidation.wrongTypeFormat.format("Not supported WDL type value")
-}
-
-
diff --git a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesTask.scala b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesTask.scala
index f6645f009..f106d0b3e 100644
--- a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesTask.scala
+++ b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesTask.scala
@@ -4,10 +4,10 @@ import cromwell.backend.standard.StandardExpressionFunctions
 import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptor}
 import cromwell.core.logging.JobLogger
 import cromwell.core.path.{DefaultPathBuilder, Path}
-import wdl4s.FullyQualifiedName
-import wdl4s.expression.NoFunctions
+import wdl4s.wdl.FullyQualifiedName
+import wdl4s.wdl.expression.NoFunctions
 import wdl4s.parser.MemoryUnit
-import wdl4s.values.{WdlFile, WdlGlobFile, WdlSingleFile}
+import wdl4s.wdl.values.{WdlFile, WdlGlobFile, WdlSingleFile, WdlValue}
 
 final case class TesTask(jobDescriptor: BackendJobDescriptor,
                          configurationDescriptor: BackendConfigurationDescriptor,
@@ -15,35 +15,48 @@ final case class TesTask(jobDescriptor: BackendJobDescriptor,
                          tesPaths: TesJobPaths,
                          runtimeAttributes: TesRuntimeAttributes,
                          containerWorkDir: Path,
-                         backendEngineFunctions: StandardExpressionFunctions) {
+                         commandScriptContents: String,
+                         backendEngineFunctions: StandardExpressionFunctions,
+                         dockerImageUsed: String) {
 
   private val workflowDescriptor = jobDescriptor.workflowDescriptor
   private val workflowName = workflowDescriptor.workflow.unqualifiedName
   private val fullyQualifiedTaskName = jobDescriptor.call.fullyQualifiedName
-  val name = fullyQualifiedTaskName
-  val description = jobDescriptor.toString
+  val name: String = fullyQualifiedTaskName
+  val description: String = jobDescriptor.toString
 
   // TODO validate "project" field of workflowOptions
   val project = {
-    workflowDescriptor.workflowOptions.getOrElse("project", workflowName)
+    workflowDescriptor.workflowOptions.getOrElse("project", "")
   }
 
   // contains the script to be executed
   private val commandScript = TaskParameter(
     Option("commandScript"),
     Option(fullyQualifiedTaskName + ".commandScript"),
-    tesPaths.storageInput(tesPaths.script.toString),
+    None,
     tesPaths.callExecutionDockerRoot.resolve("script").toString,
-    "File",
-    Option(false)
+    Option("FILE"),
+    Option(commandScriptContents)
+  )
+
+  private val commandScriptOut = commandScript.copy(
+    url = Option(tesPaths.script.toString),
+    contents = None
   )
 
-  private val writeFunctionFiles: Map[FullyQualifiedName, Seq[WdlFile]] = jobDescriptor
-    .call
-    .task
-    .evaluateFilesFromCommand(jobDescriptor.fullyQualifiedInputs, backendEngineFunctions)
-    .map {
-    case (expression, file) => expression.toWdlString -> Seq(file)
+  private def writeFunctionFiles(commandLineValueMapper: WdlValue => WdlValue): Map[FullyQualifiedName, Seq[WdlFile]] = {
+    val commandLineMappedInputs = jobDescriptor.inputDeclarations map {
+      case (declaration, value) => declaration.fullyQualifiedName -> commandLineValueMapper(value)
+    }
+
+    jobDescriptor
+      .call
+      .task
+      .evaluateFilesFromCommand(commandLineMappedInputs, backendEngineFunctions)
+      .map {
+        case (expression, file) => expression.toWdlString -> Seq(file)
+      }
   }
 
   private val callInputFiles: Map[FullyQualifiedName, Seq[WdlFile]] = jobDescriptor
@@ -52,16 +65,16 @@ final case class TesTask(jobDescriptor: BackendJobDescriptor,
       _.collectAsSeq { case w: WdlFile => w }
     }
 
-  val inputs: Seq[TaskParameter] = (callInputFiles ++ writeFunctionFiles)
+  def inputs(commandLineValueMapper: WdlValue => WdlValue): Seq[TaskParameter] = (callInputFiles ++ writeFunctionFiles(commandLineValueMapper))
     .flatMap {
       case (fullyQualifiedName, files) => files.zipWithIndex.map {
         case (f, index) => TaskParameter(
           Option(fullyQualifiedName + "." + index),
           Option(workflowName + "." + fullyQualifiedName + "." + index),
-          tesPaths.storageInput(f.value),
+          Option(f.value),
           tesPaths.containerInput(f.value),
-          "File",
-          Option(false)
+          Option("FILE"),
+          None
         )
       }
     }.toList ++ Seq(commandScript)
@@ -72,10 +85,10 @@ final case class TesTask(jobDescriptor: BackendJobDescriptor,
       TaskParameter(
         Option(f),
         Option(fullyQualifiedTaskName + "." + f),
-        tesPaths.storageOutput(f),
+        Option(tesPaths.storageOutput(f)),
         tesPaths.containerOutput(containerWorkDir, f),
-        "File",
-        Option(false)
+        Option("FILE"),
+        None
       )
   }
 
@@ -102,10 +115,10 @@ final case class TesTask(jobDescriptor: BackendJobDescriptor,
           TaskParameter(
             Option(fullyQualifiedTaskName + ".output." + index),
             Option(fullyQualifiedTaskName + ".output." + index),
-            tesPaths.storageOutput(outputFile),
+            Option(tesPaths.storageOutput(outputFile)),
             tesPaths.containerOutput(containerWorkDir, outputFile),
-            "File",
-            Option(false)
+            Option("FILE"),
+            None
           )
         )
       case (g: WdlGlobFile, index) =>
@@ -118,117 +131,104 @@ final case class TesTask(jobDescriptor: BackendJobDescriptor,
           TaskParameter(
             Option(globDirName),
             Option(fullyQualifiedTaskName + "." + globDirName),
-            tesPaths.storageOutput(globDirectory),
+            Option(tesPaths.storageOutput(globDirectory)),
             tesPaths.containerOutput(containerWorkDir, globDirectory),
-            "Directory",
-            Option(false)
+            Option("DIRECTORY"),
+            None
           ),
           TaskParameter(
             Option(globListName),
             Option(fullyQualifiedTaskName + "." + globListName),
-            tesPaths.storageOutput(globListFile),
+            Option(tesPaths.storageOutput(globListFile)),
             tesPaths.containerOutput(containerWorkDir, globListFile),
-            "File",
-            Option(false)
+            Option("FILE"),
+            None
           )
         )
     }
 
-  val outputs: Seq[TaskParameter] = wdlOutputs ++ standardOutputs
-
-  // TODO all volumes currently get the same disk requirements
-  private val workingDirVolume = runtimeAttributes
-    .dockerWorkingDir
-    .map(path => Volume(
-      Option(path),
-      runtimeAttributes.disk.to(MemoryUnit.GB).amount.toInt,
-      None,
-      path,
-      Option(false)
-    ))
-
-  val volumes = Seq(
-    Volume(
-      Option(tesPaths.callInputsDockerRoot.toString),
-      runtimeAttributes.disk.to(MemoryUnit.GB).amount.toInt,
-      None,
-      tesPaths.callInputsDockerRoot.toString,
-      // inputs in read-only volume
-      Option(true)
-    ),
-    Volume(
-      Option(tesPaths.callExecutionDockerRoot.toString),
-      runtimeAttributes.disk.to(MemoryUnit.GB).amount.toInt,
-      None,
-      tesPaths.callExecutionDockerRoot.toString,
-      Option(false)
-    )
-  ) ++ workingDirVolume
+  val outputs: Seq[TaskParameter] = wdlOutputs ++ standardOutputs ++ Seq(commandScriptOut)
+
+  private val disk :: ram :: _ = Seq(runtimeAttributes.disk, runtimeAttributes.memory) map {
+    case Some(x) =>
+      Option(x.to(MemoryUnit.GB).amount)
+    case None =>
+      None
+  }
 
   val resources = Resources(
     runtimeAttributes.cpu,
-    runtimeAttributes.memory.to(MemoryUnit.GB).amount.toInt,
+    ram,
+    disk,
     Option(false),
-    volumes,
     None
   )
 
-  val dockerExecutor = Seq(DockerExecutor(
-    runtimeAttributes.dockerImage,
+  val executors = Seq(Executor(
+    dockerImageUsed,
     Seq("/bin/bash", commandScript.path),
     runtimeAttributes.dockerWorkingDir,
     Option(tesPaths.containerOutput(containerWorkDir, "stdout")),
     Option(tesPaths.containerOutput(containerWorkDir, "stderr")),
     None,
+    None,
     None
   ))
 }
 
 // Field requirements in classes below based off GA4GH schema
-
-final case class TesTaskMessage(name: Option[String],
-                                description: Option[String],
-                                projectId: Option[String],
-                                inputs: Option[Seq[TaskParameter]],
-                                outputs: Option[Seq[TaskParameter]],
-                                resources: Resources,
-                                docker: Seq[DockerExecutor])
-
-final case class DockerExecutor(imageName: String,
-                                cmd: Seq[String],
-                                workdir: Option[String],
-                                stdout: Option[String],
-                                stderr: Option[String],
-                                stdin: Option[String],
-                                ports: Option[Seq[Ports]])
+final case class Task(id: Option[String],
+                      state: Option[String],
+                      name: Option[String],
+                      description: Option[String],
+                      project: Option[String],
+                      inputs: Option[Seq[TaskParameter]],
+                      outputs: Option[Seq[TaskParameter]],
+                      resources: Option[Resources],
+                      executors: Seq[Executor],
+                      volumes: Option[Seq[String]],
+                      tags: Option[Map[String, String]],
+                      logs: Option[Seq[TaskLog]])
+
+final case class Executor(image_name: String,
+                          cmd: Seq[String],
+                          workdir: Option[String],
+                          stdout: Option[String],
+                          stderr: Option[String],
+                          stdin: Option[String],
+                          environ: Option[Map[String, String]],
+                          ports: Option[Seq[Ports]])
 
 final case class TaskParameter(name: Option[String],
                                description: Option[String],
-                               location: String,
+                               url: Option[String],
                                path: String,
-                               `class`: String,
-                               create: Option[Boolean])
+                               `type`: Option[String],
+                               contents: Option[String])
 
-final case class Resources(minimumCpuCores: Int,
-                           minimumRamGb: Int,
+final case class Resources(cpu_cores: Option[Int],
+                           ram_gb: Option[Double],
+                           size_gb: Option[Double],
                            preemptible: Option[Boolean],
-                           volumes: Seq[Volume],
                            zones: Option[Seq[String]])
 
-final case class Volume(name: Option[String],
-                        sizeGb: Int,
-                        source: Option[String],
-                        mountPoint: String,
-                        readonly: Option[Boolean])
-
-final case class JobLogs(cmd: Option[Seq[String]],
-                         startTime: Option[String],
-                         endTime: Option[String],
-                         stdout: Option[String],
-                         stderr: Option[String],
-                         exitCode: Option[Int],
-                         hostIP: Option[String],
-                         ports: Option[Seq[Ports]])
+final case class OutputFileLog(url: String,
+                               path: String,
+                               size_bytes: Int)
+
+final case class TaskLog(start_time: Option[String],
+                         end_time: Option[String],
+                         metadata: Option[Map[String, String]],
+                         logs: Option[Seq[ExecutorLog]],
+                         outputs: Option[Seq[OutputFileLog]])
+
+final case class ExecutorLog(start_time: Option[String],
+                             end_time: Option[String],
+                             stdout: Option[String],
+                             stderr: Option[String],
+                             exit_code: Option[Int],
+                             host_ip: Option[String],
+                             ports: Option[Seq[Ports]])
 
 final case class Ports(host: Option[String],
-                       container: String)
\ No newline at end of file
+                       container: String)
diff --git a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesWorkflowPaths.scala b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesWorkflowPaths.scala
index 0fcd9c4f9..c85cee1a0 100644
--- a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesWorkflowPaths.scala
+++ b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesWorkflowPaths.scala
@@ -6,7 +6,7 @@ import cromwell.backend.{BackendJobDescriptorKey, BackendWorkflowDescriptor}
 import cromwell.core.path.{PathBuilder, PathFactory}
 import net.ceedubs.ficus.Ficus._
 
-class TesWorkflowPaths(override val workflowDescriptor: BackendWorkflowDescriptor,
+case class TesWorkflowPaths(override val workflowDescriptor: BackendWorkflowDescriptor,
                        override val config: Config,
                        override val pathBuilders: List[PathBuilder] = WorkflowPaths.DefaultPathBuilders) extends WorkflowPaths {
 
@@ -17,8 +17,10 @@ class TesWorkflowPaths(override val workflowDescriptor: BackendWorkflowDescripto
   }
   val dockerWorkflowRoot = workflowPathBuilder(DockerRoot)
 
-  override def toJobPaths(jobKey: BackendJobDescriptorKey,
-                          jobWorkflowDescriptor: BackendWorkflowDescriptor): TesJobPaths = {
-    new TesJobPaths(jobKey, jobWorkflowDescriptor, config, pathBuilders)
+  override def toJobPaths(workflowPaths: WorkflowPaths,
+                          jobKey: BackendJobDescriptorKey): TesJobPaths = {
+    new TesJobPaths(workflowPaths.asInstanceOf[TesWorkflowPaths], jobKey)
   }
+
+  override protected def withDescriptor(workflowDescriptor: BackendWorkflowDescriptor): WorkflowPaths = this.copy(workflowDescriptor = workflowDescriptor)
 }
diff --git a/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesInitializationActorSpec.scala b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesInitializationActorSpec.scala
index 0f033bbb1..f54d2040c 100644
--- a/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesInitializationActorSpec.scala
+++ b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesInitializationActorSpec.scala
@@ -12,7 +12,7 @@ import cromwell.backend.{BackendConfigurationDescriptor, BackendWorkflowDescript
 import cromwell.core.TestKitSuite
 import cromwell.core.logging.LoggingTest._
 import org.scalatest.{Matchers, WordSpecLike}
-import wdl4s.TaskCall
+import wdl4s.wdl.WdlTaskCall
 
 import scala.concurrent.duration._
 
@@ -50,10 +50,21 @@ class TesInitializationActorSpec extends TestKitSuite("TesInitializationActorSpe
       |// Polling for completion backs-off gradually for slower-running jobs.
       |// This is the maximum polling interval (in seconds):
       |maximum-polling-interval = 600
+      |
+      |default-runtime-attributes {
+      |    cpu: 1
+      |    failOnStderr: false
+      |    continueOnReturnCode: 0
+      |    memory: "2 GB"
+      |    disk: "2 GB"
+      |    # The keys below have been commented out as they are optional runtime attributes.
+      |    # dockerWorkingDir
+      |    # docker
+      |}
       |""".stripMargin
 
 
-  private def getActorRef(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[TaskCall],
+  private def getActorRef(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[WdlTaskCall],
                           conf: BackendConfigurationDescriptor) = {
     val params = TesInitializationActorParams(workflowDescriptor, calls, new TesConfiguration(conf), emptyActor)
     val props = Props(new TesInitializationActor(params))
diff --git a/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesJobPathsSpec.scala b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesJobPathsSpec.scala
index a5c1e6eeb..85fc95b89 100644
--- a/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesJobPathsSpec.scala
+++ b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesJobPathsSpec.scala
@@ -1,40 +1,18 @@
 package cromwell.backend.impl.tes
 
 import better.files._
-import com.typesafe.config.ConfigFactory
-import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptorKey, BackendSpec}
+import cromwell.backend.{BackendJobDescriptorKey, BackendSpec}
 import org.scalatest.{FlatSpec, Matchers}
-import wdl4s.TaskCall
+import wdl4s.wdl.WdlTaskCall
 
 class TesJobPathsSpec extends FlatSpec with Matchers with BackendSpec {
 
-  val configString =
-    """
-      |root = "local-cromwell-executions"
-      |dockerRoot = "/cromwell-executions"
-      |
-      |filesystems {
-      |  local {
-      |    localization: [
-      |      "hard-link", "soft-link", "copy"
-      |    ]
-      |  }
-      |  gcs {
-      |    auth = "application-default"
-      |  }
-      |}
-      |""".stripMargin
-
-  val globalConfig = ConfigFactory.load()
-  val backendConfig =  ConfigFactory.parseString(configString)
-  val defaultBackendConfigDescriptor = BackendConfigurationDescriptor(backendConfig, globalConfig)
-
   "JobPaths" should "provide correct paths for a job" in {
 
     val wd = buildWorkflowDescriptor(TestWorkflows.HelloWorld)
-    val call: TaskCall = wd.workflow.taskCalls.head
+    val call: WdlTaskCall = wd.workflow.taskCalls.head
     val jobKey = BackendJobDescriptorKey(call, None, 1)
-    val jobPaths = new TesJobPaths(jobKey, wd, backendConfig)
+    val jobPaths = TesJobPaths(jobKey, wd, TesTestConfig.backendConfig)
     val id = wd.id
     jobPaths.callRoot.toString shouldBe
       File(s"local-cromwell-executions/wf_hello/$id/call-hello").pathAsString
@@ -56,17 +34,17 @@ class TesJobPathsSpec extends FlatSpec with Matchers with BackendSpec {
       File(s"/cromwell-executions/wf_hello/$id/call-hello/execution").pathAsString
 
     val jobKeySharded = BackendJobDescriptorKey(call, Option(0), 1)
-    val jobPathsSharded = new TesJobPaths(jobKeySharded, wd, backendConfig)
+    val jobPathsSharded = TesJobPaths(jobKeySharded, wd, TesTestConfig.backendConfig)
     jobPathsSharded.callExecutionRoot.toString shouldBe
       File(s"local-cromwell-executions/wf_hello/$id/call-hello/shard-0/execution").pathAsString
 
     val jobKeyAttempt = BackendJobDescriptorKey(call, None, 2)
-    val jobPathsAttempt = new TesJobPaths(jobKeyAttempt, wd, backendConfig)
+    val jobPathsAttempt = TesJobPaths(jobKeyAttempt, wd, TesTestConfig.backendConfig)
     jobPathsAttempt.callExecutionRoot.toString shouldBe
       File(s"local-cromwell-executions/wf_hello/$id/call-hello/attempt-2/execution").pathAsString
 
     val jobKeyShardedAttempt = BackendJobDescriptorKey(call, Option(0), 2)
-    val jobPathsShardedAttempt = new TesJobPaths(jobKeyShardedAttempt, wd, backendConfig)
+    val jobPathsShardedAttempt = TesJobPaths(jobKeyShardedAttempt, wd, TesTestConfig.backendConfig)
     jobPathsShardedAttempt.callExecutionRoot.toString shouldBe
       File(s"local-cromwell-executions/wf_hello/$id/call-hello/shard-0/attempt-2/execution").pathAsString
   }
diff --git a/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesRuntimeAttributesSpec.scala b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesRuntimeAttributesSpec.scala
index 5fb711925..346739e5c 100644
--- a/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesRuntimeAttributesSpec.scala
+++ b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesRuntimeAttributesSpec.scala
@@ -1,14 +1,13 @@
 package cromwell.backend.impl.tes
 
 import cromwell.backend.validation.ContinueOnReturnCodeSet
-import cromwell.backend.{MemorySize, RuntimeAttributeDefinition}
+import cromwell.backend.{BackendConfigurationDescriptor, MemorySize, RuntimeAttributeDefinition, TestConfig}
 import cromwell.core.WorkflowOptions
 import org.scalatest.{Matchers, WordSpecLike}
 import org.slf4j.helpers.NOPLogger
 import spray.json._
-import wdl4s.parser.MemoryUnit
-import wdl4s.types.{WdlArrayType, WdlIntegerType, WdlStringType}
-import wdl4s.values.{WdlArray, WdlBoolean, WdlInteger, WdlString, WdlValue}
+import wdl4s.wdl.types.{WdlArrayType, WdlIntegerType, WdlStringType}
+import wdl4s.wdl.values.{WdlArray, WdlBoolean, WdlInteger, WdlString, WdlValue}
 
 class TesRuntimeAttributesSpec extends WordSpecLike with Matchers {
 
@@ -17,9 +16,9 @@ class TesRuntimeAttributesSpec extends WordSpecLike with Matchers {
     "ubuntu:latest",
     None,
     false,
-    1,
-    MemorySize(2, MemoryUnit.GB),
-    MemorySize(2, MemoryUnit.GB)
+    None,
+    None,
+    None
   )
 
   val expectedDefaultsPlusUbuntuDocker = expectedDefaults.copy(dockerImage = "ubuntu:latest")
@@ -84,12 +83,12 @@ class TesRuntimeAttributesSpec extends WordSpecLike with Matchers {
 
     "validate a valid cpu entry" in assertSuccess(
       Map("docker" -> WdlString("ubuntu:latest"), "cpu" -> WdlInteger(2)),
-      expectedDefaultsPlusUbuntuDocker.copy(cpu = 2)
+      expectedDefaultsPlusUbuntuDocker.copy(cpu = Option(2))
     )
 
     "validate a valid cpu string entry" in {
       val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest"), "cpu" -> WdlString("2"))
-      val expectedRuntimeAttributes = expectedDefaultsPlusUbuntuDocker.copy(cpu = 2)
+      val expectedRuntimeAttributes = expectedDefaultsPlusUbuntuDocker.copy(cpu = Option(2))
       assertSuccess(runtimeAttributes, expectedRuntimeAttributes)
     }
 
@@ -100,7 +99,7 @@ class TesRuntimeAttributesSpec extends WordSpecLike with Matchers {
 
     "validate a valid memory entry" in {
       val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest"), "memory" -> WdlString("1 GB"))
-      val expectedRuntimeAttributes = expectedDefaults.copy(memory = MemorySize.parse("1 GB").get)
+      val expectedRuntimeAttributes = expectedDefaults.copy(memory = Option(MemorySize.parse("1 GB").get))
       assertSuccess(runtimeAttributes, expectedRuntimeAttributes)
     }
 
@@ -109,13 +108,13 @@ class TesRuntimeAttributesSpec extends WordSpecLike with Matchers {
       assertFailure(runtimeAttributes, "Expecting memory runtime attribute to be an Integer or String with format '8 GB'")
     }
 
-    "validate a valid disks entry" in {
+    "validate a valid disk entry" in {
       val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest"), "disk" -> WdlString("1 GB"))
-      val expectedRuntimeAttributes = expectedDefaults.copy(disk = MemorySize.parse("1 GB").get)
+      val expectedRuntimeAttributes = expectedDefaults.copy(disk = Option(MemorySize.parse("1 GB").get))
       assertSuccess(runtimeAttributes, expectedRuntimeAttributes)
     }
 
-    "fail to validate an invalid disks entry" in {
+    "fail to validate an invalid disk entry" in {
       val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest"), "disk" -> WdlString("blah"))
       assertFailure(runtimeAttributes, "Expecting disk runtime attribute to be an Integer or String with format '8 GB'")
     }
@@ -137,12 +136,15 @@ class TesRuntimeAttributesSpec extends WordSpecLike with Matchers {
     )
   }
 
+  private val mockConfigurationDescriptor = BackendConfigurationDescriptor(TesTestConfig.backendConfig, TestConfig.globalConfig)
+  private val mockTesConfiguration = new TesConfiguration(mockConfigurationDescriptor)
+
   private def assertSuccess(runtimeAttributes: Map[String, WdlValue],
                             expectedRuntimeAttributes: TesRuntimeAttributes,
                             workflowOptions: WorkflowOptions = emptyWorkflowOptions): Unit = {
 
     try {
-      val actualRuntimeAttributes = toTesRuntimeAttributes(runtimeAttributes, workflowOptions)
+      val actualRuntimeAttributes = toTesRuntimeAttributes(runtimeAttributes, workflowOptions, mockTesConfiguration)
       assert(actualRuntimeAttributes == expectedRuntimeAttributes)
     } catch {
       case ex: RuntimeException => fail(s"Exception was not expected but received: ${ex.getMessage}")
@@ -154,7 +156,7 @@ class TesRuntimeAttributesSpec extends WordSpecLike with Matchers {
                             exMsg: String,
                             workflowOptions: WorkflowOptions = emptyWorkflowOptions): Unit = {
     try {
-      toTesRuntimeAttributes(runtimeAttributes, workflowOptions)
+      toTesRuntimeAttributes(runtimeAttributes, workflowOptions, mockTesConfiguration)
       fail("A RuntimeException was expected.")
     } catch {
       case ex: RuntimeException => assert(ex.getMessage.contains(exMsg))
@@ -164,15 +166,17 @@ class TesRuntimeAttributesSpec extends WordSpecLike with Matchers {
 
   private val emptyWorkflowOptions = WorkflowOptions.fromMap(Map.empty).get
   private val staticRuntimeAttributeDefinitions: Set[RuntimeAttributeDefinition] =
-    TesRuntimeAttributes.runtimeAttributesBuilder.definitions.toSet
+    TesRuntimeAttributes.runtimeAttributesBuilder(mockTesConfiguration.runtimeConfig).definitions.toSet
 
 
   private def toTesRuntimeAttributes(runtimeAttributes: Map[String, WdlValue],
-                                     workflowOptions: WorkflowOptions): TesRuntimeAttributes = {
-    val runtimeAttributesBuilder = TesRuntimeAttributes.runtimeAttributesBuilder
+                                     workflowOptions: WorkflowOptions,
+                                     tesConfiguration: TesConfiguration): TesRuntimeAttributes = {
+    val runtimeAttributesBuilder = TesRuntimeAttributes.runtimeAttributesBuilder(tesConfiguration.runtimeConfig)
     val defaultedAttributes = RuntimeAttributeDefinition.addDefaultsToAttributes(
       staticRuntimeAttributeDefinitions, workflowOptions)(runtimeAttributes)
     val validatedRuntimeAttributes = runtimeAttributesBuilder.build(defaultedAttributes, NOPLogger.NOP_LOGGER)
-    TesRuntimeAttributes(validatedRuntimeAttributes)
+    TesRuntimeAttributes(validatedRuntimeAttributes, tesConfiguration.runtimeConfig
+    )
   }
-}
\ No newline at end of file
+}
diff --git a/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesTestConfig.scala b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesTestConfig.scala
new file mode 100644
index 000000000..eb3085706
--- /dev/null
+++ b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesTestConfig.scala
@@ -0,0 +1,27 @@
+package cromwell.backend.impl.tes
+
+import com.typesafe.config.ConfigFactory
+
+object TesTestConfig {
+
+  private val backendConfigString =
+    """
+      |root = "local-cromwell-executions"
+      |dockerRoot = "/cromwell-executions"
+      |endpoint = "http://127.0.0.1:9000/v1/jobs"
+      |
+      |default-runtime-attributes {
+      |  cpu: 1
+      |  failOnStderr: false
+      |  continueOnReturnCode: 0
+      |  memory: "2 GB"
+      |  disk: "2 GB"
+      |  # The keys below have been commented out as they are optional runtime attributes.
+      |  # dockerWorkingDir
+      |  # docker
+      |}
+      |""".stripMargin
+
+  val backendConfig = ConfigFactory.parseString(backendConfigString)
+}
+
diff --git a/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesWorkflowPathsSpec.scala b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesWorkflowPathsSpec.scala
index 591b65193..1fbefeec4 100644
--- a/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesWorkflowPathsSpec.scala
+++ b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesWorkflowPathsSpec.scala
@@ -1,36 +1,16 @@
 package cromwell.backend.impl.tes
 
 import better.files._
-import com.typesafe.config.ConfigFactory
 import cromwell.backend.{BackendJobBreadCrumb, BackendSpec, BackendWorkflowDescriptor}
 import cromwell.core.{JobKey, WorkflowId}
 import org.scalatest.{FlatSpec, Matchers}
-import wdl4s.{Call, Workflow}
+import wdl4s.wdl.{WdlCall, WdlWorkflow}
 
 class TesWorkflowPathsSpec extends FlatSpec with Matchers with BackendSpec {
-  val configString =
-    """
-      |root = "local-cromwell-executions"
-      |dockerRoot = "cromwell-executions"
-      |
-      |filesystems {
-      |  local {
-      |    localization: [
-      |      "hard-link", "soft-link", "copy"
-      |    ]
-      |  }
-      |  gcs {
-      |    auth = "application-default"
-      |  }
-      |}
-      |""".stripMargin
-
-  val globalConfig = ConfigFactory.load()
-  val backendConfig =  ConfigFactory.parseString(configString)
 
   "WorkflowPaths" should "provide correct paths for a workflow" in {
     val wd = buildWorkflowDescriptor(TestWorkflows.HelloWorld)
-    val workflowPaths = new TesWorkflowPaths(wd, backendConfig)
+    val workflowPaths = TesWorkflowPaths(wd, TesTestConfig.backendConfig)
     val id = wd.id
     workflowPaths.workflowRoot.toString shouldBe
       File(s"local-cromwell-executions/wf_hello/$id").pathAsString
@@ -40,22 +20,22 @@ class TesWorkflowPathsSpec extends FlatSpec with Matchers with BackendSpec {
 
   "WorkflowPaths" should "provide correct paths for a sub workflow" in {
     val rootWd = mock[BackendWorkflowDescriptor]
-    val rootWorkflow = mock[Workflow]
+    val rootWorkflow = mock[WdlWorkflow]
     val rootWorkflowId = WorkflowId.randomId()
     rootWorkflow.unqualifiedName returns "rootWorkflow"
     rootWd.workflow returns rootWorkflow
     rootWd.id returns rootWorkflowId
 
     val subWd = mock[BackendWorkflowDescriptor]
-    val subWorkflow = mock[Workflow]
+    val subWorkflow = mock[WdlWorkflow]
     val subWorkflowId = WorkflowId.randomId()
     subWorkflow.unqualifiedName returns "subWorkflow"
     subWd.workflow returns subWorkflow
     subWd.id returns subWorkflowId
     
-    val call1 = mock[Call]
+    val call1 = mock[WdlCall]
     call1.unqualifiedName returns "call1"
-    val call2 = mock[Call]
+    val call2 = mock[WdlCall]
     call2.unqualifiedName returns "call2"
     
     val jobKey = new JobKey {
@@ -68,7 +48,7 @@ class TesWorkflowPathsSpec extends FlatSpec with Matchers with BackendSpec {
     subWd.breadCrumbs returns List(BackendJobBreadCrumb(rootWorkflow, rootWorkflowId, jobKey))
     subWd.id returns subWorkflowId
     
-    val workflowPaths = new TesWorkflowPaths(subWd, backendConfig)
+    val workflowPaths = new TesWorkflowPaths(subWd, TesTestConfig.backendConfig)
     workflowPaths.workflowRoot.toString shouldBe File(s"local-cromwell-executions/rootWorkflow/$rootWorkflowId/call-call1/shard-1/attempt-2/subWorkflow/$subWorkflowId").pathAsString
     workflowPaths.dockerWorkflowRoot.toString shouldBe s"/cromwell-executions/rootWorkflow/$rootWorkflowId/call-call1/shard-1/attempt-2/subWorkflow/$subWorkflowId"
   }