From fd15a89c57ff74fb94d27ffc4b579e0abf59158e Mon Sep 17 00:00:00 2001 From: Ruchi Munshi Date: Tue, 11 Apr 2017 16:22:39 -0400 Subject: [PATCH 001/134] Update cromwell version from 26 to 27 --- project/Version.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Version.scala b/project/Version.scala index 5d7c067e5..0d9839c3d 100644 --- a/project/Version.scala +++ b/project/Version.scala @@ -4,7 +4,7 @@ import sbt._ object Version { // Upcoming release, or current if we're on a master / hotfix branch - val cromwellVersion = "26" + val cromwellVersion = "27" // Adapted from SbtGit.versionWithGit def cromwellVersionWithGit: Seq[Setting[_]] = From 1906bdfa979efd372a1e55692e6a04d158e17db4 Mon Sep 17 00:00:00 2001 From: kshakir Date: Wed, 12 Apr 2017 13:28:24 -0400 Subject: [PATCH 002/134] Updated to Slick 3.2. (#2104) Removed deadlock workaround. Minimal updates to fix depracated usage. Old Slick 3.1 style names are still used in scala, ex: `val driver: JdbcProfile`. Added notes about SQL converters handling empty LOBs. --- CHANGELOG.md | 31 +++-- README.md | 7 +- core/src/main/resources/reference.conf | 2 +- core/src/test/resources/application.conf | 2 +- .../migration/src/main/resources/changelog.xml | 2 +- .../cromwell/database/slick/SlickDatabase.scala | 41 +------ .../slick/tables/DataAccessComponent.scala | 2 +- .../database/slick/tables/DriverComponent.scala | 2 +- .../cromwell/database/sql/SqlConverters.scala | 23 +++- project/Dependencies.scala | 2 +- .../compose/cromwell/app-config/application.conf | 2 +- .../jes-cromwell/jes-config/application.conf | 2 +- .../cromwell/services/ServicesStoreSpec.scala | 133 +++++++-------------- 13 files changed, 98 insertions(+), 153 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2174c82f1..e28de5e73 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,25 @@ # Cromwell Change Log +## 27 + + +* The update to Slick 3.2 requires a database stanza to +[switch](http://slick.lightbend.com/doc/3.2.0/upgrade.html#profiles-vs-drivers) from using `driver` to `profile`. + +```hocon +database { + #driver = "slick.driver.MySQLDriver$" #old + profile = "slick.jdbc.MySQLProfile$" #new + db { + driver = "com.mysql.jdbc.Driver" + url = "jdbc:mysql://host/cromwell?rewriteBatchedStatements=true" + user = "user" + password = "pass" + connectionTimeout = 5000 + } +} +``` + ## 26 ### Breaking Changes @@ -43,17 +63,6 @@ system.io { } ``` -* Added a `script-epilogue` configuration option to adjust the logic that runs at the end of the scripts which wrap call executions. - This option is adjustable on a per-backend basis. If unspecified, the default value is `sync`. - -### WDL Features - -With Cromwell 26, Cromwell will support `if x then y else z` expressions (see: https://github.com/broadinstitute/wdl/blob/develop/SPEC.md#if-then-else). For example: -``` -Boolean b = true -String s = if b then "value if True" else "value if False" -``` - ## 25 ### External Contributors diff --git a/README.md b/README.md index 989a6c609..358c454b7 100644 --- a/README.md +++ b/README.md @@ -497,8 +497,7 @@ Then, edit the configuration file `database` stanza, as follows: ``` database { - - driver = "slick.driver.MySQLDriver$" + profile = "slick.jdbc.MySQLProfile$" db { driver = "com.mysql.jdbc.Driver" url = "jdbc:mysql://host/cromwell?rewriteBatchedStatements=true" @@ -506,10 +505,6 @@ database { password = "pass" connectionTimeout = 5000 } - - test { - ... - } } ``` diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf index ad3982a4e..787303136 100644 --- a/core/src/main/resources/reference.conf +++ b/core/src/main/resources/reference.conf @@ -488,7 +488,7 @@ services { database { # hsql default - driver = "slick.driver.HsqldbDriver$" + profile = "slick.jdbc.HsqldbProfile$" db { driver = "org.hsqldb.jdbcDriver" url = "jdbc:hsqldb:mem:${uniqueSchema};shutdown=false;hsqldb.tx=mvcc" diff --git a/core/src/test/resources/application.conf b/core/src/test/resources/application.conf index 1ef667470..f45d06517 100644 --- a/core/src/test/resources/application.conf +++ b/core/src/test/resources/application.conf @@ -22,7 +22,7 @@ database.db.connectionTimeout = 3000 database-test-mysql { # Run the following to (optionally) drop and (re-)create the database: # mysql -utravis -e "DROP DATABASE IF EXISTS cromwell_test" && mysql -utravis -e "CREATE DATABASE cromwell_test" - driver = "slick.driver.MySQLDriver$" + profile = "slick.jdbc.MySQLProfile$" db { driver = "com.mysql.jdbc.Driver" url = "jdbc:mysql://localhost/cromwell_test?useSSL=false" diff --git a/database/migration/src/main/resources/changelog.xml b/database/migration/src/main/resources/changelog.xml index b3c05e026..c0a72170c 100644 --- a/database/migration/src/main/resources/changelog.xml +++ b/database/migration/src/main/resources/changelog.xml @@ -80,7 +80,7 @@ WHY In slick schemas, `index(unique = false)` always creates an index. Depending on the database type, `index(unique = true)` sometimes creates a unique _constraint_, not actually an index. - https://github.com/slick/slick/blob/3.1.1/slick/src/main/scala/slick/driver/HsqldbDriver.scala#L126 + https://github.com/slick/slick/blob/3.2.0/slick/src/main/scala/slick/jdbc/HsqldbProfile.scala#L127 -*- diff --git a/database/sql/src/main/scala/cromwell/database/slick/SlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/SlickDatabase.scala index 860ad65b3..000637a5c 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/SlickDatabase.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/SlickDatabase.scala @@ -1,18 +1,17 @@ package cromwell.database.slick import java.sql.Connection -import java.util.concurrent.{ExecutorService, Executors} import com.typesafe.config.Config import cromwell.database.slick.tables.DataAccessComponent import cromwell.database.sql.SqlDatabase import net.ceedubs.ficus.Ficus._ import org.slf4j.LoggerFactory -import slick.backend.DatabaseConfig -import slick.driver.JdbcProfile +import slick.basic.DatabaseConfig +import slick.jdbc.{JdbcCapabilities, JdbcProfile} import scala.concurrent.duration._ -import scala.concurrent.{Await, ExecutionContext, Future} +import scala.concurrent.{Await, Future} object SlickDatabase { /** @@ -64,7 +63,7 @@ class SlickDatabase(override val originalDatabaseConfig: Config) extends SqlData override val urlKey = SlickDatabase.urlKey(originalDatabaseConfig) private val slickConfig = DatabaseConfig.forConfig[JdbcProfile]("", databaseConfig) - val dataAccess = new DataAccessComponent(slickConfig.driver) + val dataAccess = new DataAccessComponent(slickConfig.profile) // Allows creation of a Database, plus implicits for running transactions import dataAccess.driver.api._ @@ -74,36 +73,10 @@ class SlickDatabase(override val originalDatabaseConfig: Config) extends SqlData SlickDatabase.log.info(s"Running with database $urlKey = ${databaseConfig.getString(urlKey)}") - /** - * Create a special execution context, a fixed thread pool, to run each of our composite database actions. Running - * each composite action as a runnable within the pool will ensure that-- at most-- the same number of actions are - * running as there are available connections. Thus there should never be a connection deadlock, as outlined in - * - https://github.com/slick/slick/issues/1274 - * - https://groups.google.com/d/msg/scalaquery/5MCUnwaJ7U0/NLLMotX9BQAJ - * - * Custom future thread pool based on: - * - http://stackoverflow.com/questions/15285284/how-to-configure-a-fine-tuned-thread-pool-for-futures#comment23278672_15285441 - * - * Database config parameter defaults based on: (expand the `forConfig` scaladoc for a full list of values) - * - http://slick.typesafe.com/doc/3.1.0/api/index.html#slick.jdbc.JdbcBackend$DatabaseFactoryDef@forConfig(path:String,config:com.typesafe.config.Config,driver:java.sql.Driver,classLoader:ClassLoader):JdbcBackend.this.Database - * - * Reuses the error reporter from the database's executionContext. - */ - private val actionThreadPool: ExecutorService = { - val dbNumThreads = databaseConfig.as[Option[Int]]("db.numThreads").getOrElse(20) - val dbMaximumPoolSize = databaseConfig.as[Option[Int]]("db.maxConnections").getOrElse(dbNumThreads * 5) - val actionThreadPoolSize = databaseConfig.as[Option[Int]]("actionThreadPoolSize").getOrElse(dbNumThreads) min dbMaximumPoolSize - Executors.newFixedThreadPool(actionThreadPoolSize) - } - - private val actionExecutionContext: ExecutionContext = ExecutionContext.fromExecutor( - actionThreadPool, database.executor.executionContext.reportFailure - ) - protected[this] lazy val insertBatchSize = databaseConfig.as[Option[Int]]("insert-batch-size").getOrElse(2000) protected[this] lazy val useSlickUpserts = - dataAccess.driver.capabilities.contains(JdbcProfile.capabilities.insertOrUpdate) + dataAccess.driver.capabilities.contains(JdbcCapabilities.insertOrUpdate) protected[this] def assertUpdateCount(description: String, updates: Int, expected: Int): DBIO[Unit] = { if (updates == expected) { @@ -123,12 +96,10 @@ class SlickDatabase(override val originalDatabaseConfig: Config) extends SqlData } override def close(): Unit = { - actionThreadPool.shutdown() database.close() } protected[this] def runTransaction[R](action: DBIO[R]): Future[R] = { - //database.run(action.transactionally) <-- https://github.com/slick/slick/issues/1274 - Future(Await.result(database.run(action.transactionally), Duration.Inf))(actionExecutionContext) + database.run(action.transactionally) } } diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/DataAccessComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/DataAccessComponent.scala index b0c70abf3..2cbaf9576 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/tables/DataAccessComponent.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/tables/DataAccessComponent.scala @@ -1,6 +1,6 @@ package cromwell.database.slick.tables -import slick.driver.JdbcProfile +import slick.jdbc.JdbcProfile class DataAccessComponent(val driver: JdbcProfile) extends DriverComponent diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/DriverComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/DriverComponent.scala index f3fba44d4..d5f786018 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/tables/DriverComponent.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/tables/DriverComponent.scala @@ -1,6 +1,6 @@ package cromwell.database.slick.tables -import slick.driver.JdbcProfile +import slick.jdbc.JdbcProfile trait DriverComponent { val driver: JdbcProfile diff --git a/database/sql/src/main/scala/cromwell/database/sql/SqlConverters.scala b/database/sql/src/main/scala/cromwell/database/sql/SqlConverters.scala index 9b95b348d..1799040cd 100644 --- a/database/sql/src/main/scala/cromwell/database/sql/SqlConverters.scala +++ b/database/sql/src/main/scala/cromwell/database/sql/SqlConverters.scala @@ -4,9 +4,6 @@ import java.sql.{Blob, Clob, Timestamp} import java.time.{OffsetDateTime, ZoneId} import javax.sql.rowset.serial.{SerialBlob, SerialClob} -import eu.timepit.refined.api.Refined -import eu.timepit.refined.collection.NonEmpty - object SqlConverters { // TODO: Storing times relative to system zone. Look into db/slick using OffsetDateTime, or storing datetimes as UTC? @@ -41,6 +38,17 @@ object SqlConverters { } implicit class StringToClobOption(val str: String) extends AnyVal { + /* + Many, many Clob implementations have problems with empty char arrays. + http://hg.openjdk.java.net/jdk8/jdk8/jdk/file/687fd7c7986d/src/share/classes/javax/sql/rowset/serial/SerialClob.java#l268 + http://hg.openjdk.java.net/jdk9/client/jdk/file/1158c3e5bd9c/src/java.sql.rowset/share/classes/javax/sql/rowset/serial/SerialClob.java#l270 + https://github.com/apache/derby/blob/10.13/java/engine/org/apache/derby/iapi/types/HarmonySerialClob.java#L109 + https://github.com/arteam/hsqldb/blob/2.3.4/src/org/hsqldb/jdbc/JDBCClob.java#L196 + */ + + import eu.timepit.refined.api.Refined + import eu.timepit.refined.collection.NonEmpty + def toClobOption: Option[Clob] = if (str.isEmpty) None else Option(new SerialClob(str.toCharArray)) def toClob(default: String Refined NonEmpty): Clob = { @@ -61,10 +69,17 @@ object SqlConverters { } implicit class BytesOptionToBlob(val bytesOption: Option[Array[Byte]]) extends AnyVal { + /* + Many, many Blob implementations (but fewer than Clob) have problems with empty byte arrays. + http://hg.openjdk.java.net/jdk8/jdk8/jdk/file/687fd7c7986d/src/share/classes/javax/sql/rowset/serial/SerialBlob.java#l178 + http://hg.openjdk.java.net/jdk9/client/jdk/file/1158c3e5bd9c/src/java.sql.rowset/share/classes/javax/sql/rowset/serial/SerialBlob.java#l178 + https://github.com/apache/derby/blob/10.13/java/engine/org/apache/derby/iapi/types/HarmonySerialBlob.java#L111 + OK! -> https://github.com/arteam/hsqldb/blob/2.3.4/src/org/hsqldb/jdbc/JDBCBlob.java#L184 + */ def toBlobOption: Option[Blob] = bytesOption.flatMap(_.toBlobOption) } - implicit class BytesToBlob(val bytes: Array[Byte]) extends AnyVal { + implicit class BytesToBlobOption(val bytes: Array[Byte]) extends AnyVal { def toBlobOption: Option[Blob] = if (bytes.isEmpty) None else Option(new SerialBlob(bytes)) } } diff --git a/project/Dependencies.scala b/project/Dependencies.scala index fc9196540..1db4fd4d4 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -14,7 +14,7 @@ object Dependencies { lazy val sprayJsonV = "1.3.2" lazy val akkaV = "2.4.16" lazy val akkaHttpV = "2.4.11" - lazy val slickV = "3.1.1" + lazy val slickV = "3.2.0" lazy val googleClientApiV = "1.22.0" lazy val googleGenomicsServicesApiV = "1.22.0" lazy val betterFilesV = "2.17.1" diff --git a/scripts/docker-compose-mysql/compose/cromwell/app-config/application.conf b/scripts/docker-compose-mysql/compose/cromwell/app-config/application.conf index 2869bc5fd..72feb2457 100644 --- a/scripts/docker-compose-mysql/compose/cromwell/app-config/application.conf +++ b/scripts/docker-compose-mysql/compose/cromwell/app-config/application.conf @@ -51,5 +51,5 @@ database { db.user = "cromwell" db.password = "cromwell" db.driver = "com.mysql.jdbc.Driver" - driver = "slick.driver.MySQLDriver$" + profile = "slick.jdbc.MySQLProfile$" } diff --git a/scripts/docker-compose-mysql/jes-cromwell/jes-config/application.conf b/scripts/docker-compose-mysql/jes-cromwell/jes-config/application.conf index aab9925ff..64b47218f 100644 --- a/scripts/docker-compose-mysql/jes-cromwell/jes-config/application.conf +++ b/scripts/docker-compose-mysql/jes-cromwell/jes-config/application.conf @@ -64,5 +64,5 @@ database { db.user = "cromwell" db.password = "cromwell" db.driver = "com.mysql.jdbc.Driver" - driver = "slick.driver.MySQLDriver$" + profile = "slick.jdbc.MySQLProfile$" } diff --git a/services/src/test/scala/cromwell/services/ServicesStoreSpec.scala b/services/src/test/scala/cromwell/services/ServicesStoreSpec.scala index 1233879ff..2d8a96454 100644 --- a/services/src/test/scala/cromwell/services/ServicesStoreSpec.scala +++ b/services/src/test/scala/cromwell/services/ServicesStoreSpec.scala @@ -6,7 +6,7 @@ import java.time.OffsetDateTime import javax.sql.rowset.serial.{SerialBlob, SerialClob, SerialException} import better.files._ -import com.typesafe.config.{Config, ConfigFactory} +import com.typesafe.config.ConfigFactory import cromwell.core.Tags._ import cromwell.core.WorkflowId import cromwell.database.migration.liquibase.LiquibaseUtils @@ -14,8 +14,6 @@ import cromwell.database.slick.SlickDatabase import cromwell.database.sql.SqlConverters._ import cromwell.database.sql.joins.JobStoreJoin import cromwell.database.sql.tables.{JobStoreEntry, JobStoreSimpletonEntry, WorkflowStoreEntry} -import eu.timepit.refined.api.Refined -import eu.timepit.refined.collection.NonEmpty import liquibase.diff.DiffResult import liquibase.diff.output.DiffOutputControl import liquibase.diff.output.changelog.DiffToChangeLog @@ -25,11 +23,11 @@ import org.scalatest.concurrent.PatienceConfiguration.Timeout import org.scalatest.concurrent.ScalaFutures import org.scalatest.time.{Millis, Seconds, Span} import org.scalatest.{FlatSpec, Matchers} -import slick.driver.JdbcProfile +import slick.jdbc.JdbcProfile import slick.jdbc.meta._ import scala.concurrent.duration._ -import scala.concurrent.{Await, ExecutionContext, Future, TimeoutException} +import scala.concurrent.{Await, ExecutionContext, Future} import scala.util.Try import scala.xml._ @@ -43,53 +41,6 @@ class ServicesStoreSpec extends FlatSpec with Matchers with ScalaFutures with St behavior of "ServicesStore" - it should "deadlock and timeout after 10 seconds" in { - /* - Tests that we still need our deadlock workaround in `database.run` to deal with problems in Slick 3.1. - If/when this test fails, the workaround should be removed. - */ - - class DeadlockingSlickDatabase(config: Config) extends SlickDatabase(config) { - - import dataAccess.driver.api._ - - override protected[this] def runTransaction[R](action: DBIO[R]): Future[R] = { - database.run(action.transactionally) //<-- https://github.com/slick/slick/issues/1274 - } - } - - // Test based on https://github.com/kwark/slick-deadlock/blob/82525fc/src/main/scala/SlickDeadlock.scala - val databaseConfig = ConfigFactory.parseString( - s"""|db.url = "jdbc:hsqldb:mem:$${uniqueSchema};shutdown=false;hsqldb.tx=mvcc" - |db.driver = "org.hsqldb.jdbcDriver" - |db.connectionTimeout = 3000 - |db.numThreads = 2 - |driver = "slick.driver.HsqldbDriver$$" - |""".stripMargin) - import ServicesStore.EnhancedSqlDatabase - for { - database <- new DeadlockingSlickDatabase(databaseConfig).initialized.autoClosed - } { - val futures = 1 to 20 map { _ => - val workflowUuid = WorkflowId.randomId().toString - val callFqn = "call.fqn" - val jobIndex = 1 - val jobAttempt = 1 - val jobSuccessful = false - val jobStoreEntry = JobStoreEntry(workflowUuid, callFqn, jobIndex, jobAttempt, jobSuccessful, None, None, None) - val jobStoreJoins = Seq(JobStoreJoin(jobStoreEntry, Seq())) - // NOTE: This test just needs to repeatedly read/write from a table that acts as a PK for a FK. - for { - _ <- database.addJobStores(jobStoreJoins) - queried <- database.queryJobStores(workflowUuid, callFqn, jobIndex, jobAttempt) - _ = queried.get.jobStoreEntry.workflowExecutionUuid should be(workflowUuid) - } yield () - } - - intercept[TimeoutException](Await.result(Future.sequence(futures), 10.seconds)) - } - } - it should "not deadlock" in { // Test based on https://github.com/kwark/slick-deadlock/blob/82525fc/src/main/scala/SlickDeadlock.scala val databaseConfig = ConfigFactory.parseString( @@ -97,7 +48,7 @@ class ServicesStoreSpec extends FlatSpec with Matchers with ScalaFutures with St |db.driver = "org.hsqldb.jdbcDriver" |db.connectionTimeout = 3000 |db.numThreads = 2 - |driver = "slick.driver.HsqldbDriver$$" + |profile = "slick.jdbc.HsqldbProfile$$" |""".stripMargin) import ServicesStore.EnhancedSqlDatabase for { @@ -307,11 +258,8 @@ class ServicesStoreSpec extends FlatSpec with Matchers with ScalaFutures with St } yield ()).futureValue } - it should "fail to store and retrieve empty clobs in MySQL" taggedAs DbmsTest in { - /* - Tests that we still need our empty clob workaround in `SqlConverters.toClob`. The current mysql jdbc driver throws - an exception when serializing an empty clob. If/when this test fails, the workaround should be removed. - */ + it should "fail to store and retrieve empty clobs" taggedAs DbmsTest in { + // See notes in StringToClobOption val emptyClob = new SerialClob(Array.empty[Char]) val workflowUuid = WorkflowId.randomId().toString @@ -325,46 +273,47 @@ class ServicesStoreSpec extends FlatSpec with Matchers with ScalaFutures with St val future = for { product <- dataAccess.database.run(getProduct) - _ = product match { + _ <- product match { + case "HSQL Database Engine" => + // HSQLDB doesn't crash because it calls getCharacterStream instead of getSubString. + dataAccess.addJobStores(jobStoreJoins) case "MySQL" => - val exception = intercept[SerialException] { - Await.result(dataAccess.addJobStores(jobStoreJoins), Duration.Inf) + dataAccess.addJobStores(jobStoreJoins).failed map { exception => + exception should be(a[SerialException]) + exception.getMessage should be("Invalid position in SerialClob object set") } - exception should be(a[SerialException]) - exception.getMessage should be("Invalid position in SerialClob object set") - case _ => () } } yield () future.futureValue } - it should "fail to store and retrieve empty blobs in MySQL" taggedAs DbmsTest in { - /* - Tests that we still need our empty blob workaround in `SqlConverters.toBlob`. The current mysql jdbc driver throws - an exception when serializing an empty blob. If/when this test fails, the workaround should be removed. - */ - import eu.timepit.refined._ - val nonEmptyString: String Refined NonEmpty = refineMV[NonEmpty]("{}") + it should "fail to store and retrieve empty blobs" taggedAs DbmsTest in { + // See notes in BytesToBlobOption + import eu.timepit.refined.auto._ + import eu.timepit.refined.collection._ + val clob = "".toClob(default = "{}") + val clobOption = "{}".toClobOption val emptyBlob = new SerialBlob(Array.empty[Byte]) val workflowUuid = WorkflowId.randomId().toString - val workflowStoreEntry = WorkflowStoreEntry(workflowUuid, "{}".toClobOption, "{}".toClobOption, "{}".toClobOption, - "Testing", OffsetDateTime.now.toSystemTimestamp, Option(emptyBlob), "{}".toClob(nonEmptyString)) + val workflowStoreEntry = WorkflowStoreEntry(workflowUuid, clobOption, clobOption, clobOption, + "Testing", OffsetDateTime.now.toSystemTimestamp, Option(emptyBlob), clob) val workflowStoreEntries = Seq(workflowStoreEntry) val future = for { product <- dataAccess.database.run(getProduct) - _ = product match { + _ <- product match { + case "HSQL Database Engine" => + // HSQLDB doesn't crash because it calls getBinaryStream instead of getBytes. + dataAccess.addWorkflowStoreEntries(workflowStoreEntries) case "MySQL" => - val exception = intercept[SerialException] { - Await.result(dataAccess.addWorkflowStoreEntries(workflowStoreEntries), Duration.Inf) + dataAccess.addWorkflowStoreEntries(workflowStoreEntries).failed map { exception => + exception should be(a[SerialException]) + exception.getMessage should + be("Invalid arguments: position cannot be less than 1 or greater than the length of the SerialBlob") } - exception should be(a[SerialException]) - exception.getMessage should - be("Invalid arguments: position cannot be less than 1 or greater than the length of the SerialBlob") - case _ => () } } yield () @@ -372,6 +321,7 @@ class ServicesStoreSpec extends FlatSpec with Matchers with ScalaFutures with St } it should "store and retrieve empty clobs" taggedAs DbmsTest in { + // See notes in StringToClobOption val workflowUuid = WorkflowId.randomId().toString val callFqn = "call.fqn" val jobIndex = 1 @@ -403,23 +353,28 @@ class ServicesStoreSpec extends FlatSpec with Matchers with ScalaFutures with St } it should "store and retrieve empty blobs" taggedAs DbmsTest in { - import eu.timepit.refined._ + // See notes in BytesToBlobOption + import eu.timepit.refined.auto._ + import eu.timepit.refined.collection._ - val nonEmptyString: String Refined NonEmpty = refineMV[NonEmpty]("{}") val testWorkflowState = "Testing" + val clob = "".toClob(default = "{}") + val clobOption = "{}".toClobOption val emptyWorkflowUuid = WorkflowId.randomId().toString - val emptyWorkflowStoreEntry = WorkflowStoreEntry(emptyWorkflowUuid, "{}".toClobOption, "{}".toClobOption, "{}".toClobOption, - testWorkflowState, OffsetDateTime.now.toSystemTimestamp, Option(Array.empty[Byte]).toBlobOption, "{}".toClob(nonEmptyString)) + val emptyWorkflowStoreEntry = WorkflowStoreEntry(emptyWorkflowUuid, clobOption, clobOption, + clobOption, testWorkflowState, OffsetDateTime.now.toSystemTimestamp, + Option(Array.empty[Byte]).toBlobOption, clob) val noneWorkflowUuid = WorkflowId.randomId().toString - val noneWorkflowStoreEntry = WorkflowStoreEntry(noneWorkflowUuid, "{}".toClobOption, "{}".toClobOption, "{}".toClobOption, - testWorkflowState, OffsetDateTime.now.toSystemTimestamp, None, "{}".toClob(nonEmptyString)) + val noneWorkflowStoreEntry = WorkflowStoreEntry(noneWorkflowUuid, clobOption, clobOption, + clobOption, testWorkflowState, OffsetDateTime.now.toSystemTimestamp, None, clob) val aByte = 'a'.toByte val aByteWorkflowUuid = WorkflowId.randomId().toString - val aByteWorkflowStoreEntry = WorkflowStoreEntry(aByteWorkflowUuid, "{}".toClobOption, "{}".toClobOption, "{}".toClobOption, - testWorkflowState, OffsetDateTime.now.toSystemTimestamp, Option(Array(aByte)).toBlobOption, "{}".toClob(nonEmptyString)) + val aByteWorkflowStoreEntry = WorkflowStoreEntry(aByteWorkflowUuid, clobOption, clobOption, + clobOption, testWorkflowState, OffsetDateTime.now.toSystemTimestamp, Option(Array(aByte)).toBlobOption, + clob) val workflowStoreEntries = Seq(emptyWorkflowStoreEntry, noneWorkflowStoreEntry, aByteWorkflowStoreEntry) @@ -476,7 +431,7 @@ object ServicesStoreSpec { |db.url = "jdbc:hsqldb:mem:$${uniqueSchema};shutdown=false;hsqldb.tx=mvcc" |db.driver = "org.hsqldb.jdbcDriver" |db.connectionTimeout = 3000 - |driver = "slick.driver.HsqldbDriver$$" + |profile = "slick.jdbc.HsqldbProfile$$" |liquibase.updateSchema = false |""".stripMargin) val database = new SlickDatabase(databaseConfig) From 91dd653d5d2cd982185589d80f096ad3caf866fc Mon Sep 17 00:00:00 2001 From: Jeff Gentry Date: Wed, 12 Apr 2017 14:28:41 -0400 Subject: [PATCH 003/134] EngineWorkflowFinalizationActor only had a single implementation, remove layer of indirection (#2155) --- .../lifecycle/CopyWorkflowOutputsActor.scala | 32 +++++++++++++++++----- .../EngineWorkflowFinalizationActor.scala | 30 -------------------- 2 files changed, 25 insertions(+), 37 deletions(-) delete mode 100644 engine/src/main/scala/cromwell/engine/workflow/lifecycle/EngineWorkflowFinalizationActor.scala diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/CopyWorkflowOutputsActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/CopyWorkflowOutputsActor.scala index 9575fb472..6297588cf 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/CopyWorkflowOutputsActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/CopyWorkflowOutputsActor.scala @@ -1,7 +1,9 @@ package cromwell.engine.workflow.lifecycle -import akka.actor.{ActorRef, Props} -import cromwell.backend.BackendWorkflowFinalizationActor.{FinalizationResponse, FinalizationSuccess} +import akka.actor.{Actor, ActorLogging, ActorRef, Props} +import akka.event.LoggingReceive +import cromwell.backend.BackendLifecycleActor.BackendWorkflowLifecycleActorResponse +import cromwell.backend.BackendWorkflowFinalizationActor.{FinalizationFailed, FinalizationResponse, FinalizationSuccess, Finalize} import cromwell.backend.{AllBackendInitializationData, BackendConfigurationDescriptor, BackendInitializationData, BackendLifecycleActorFactory} import cromwell.core.Dispatcher.IoDispatcher import cromwell.core.WorkflowOptions._ @@ -14,6 +16,7 @@ import cromwell.filesystems.gcs.batch.GcsBatchCommandBuilder import wdl4s.values.{WdlArray, WdlMap, WdlSingleFile, WdlValue} import scala.concurrent.{ExecutionContext, Future} +import scala.util.{Failure, Success} object CopyWorkflowOutputsActor { def props(workflowId: WorkflowId, ioActor: ActorRef, workflowDescriptor: EngineWorkflowDescriptor, workflowOutputs: CallOutputs, @@ -24,13 +27,25 @@ object CopyWorkflowOutputsActor { class CopyWorkflowOutputsActor(workflowId: WorkflowId, override val ioActor: ActorRef, val workflowDescriptor: EngineWorkflowDescriptor, workflowOutputs: CallOutputs, initializationData: AllBackendInitializationData) - extends EngineWorkflowFinalizationActor with PathFactory with AsyncIo with GcsBatchCommandBuilder { + extends Actor with ActorLogging with PathFactory with AsyncIo with GcsBatchCommandBuilder { implicit val ec = context.dispatcher override val pathBuilders = workflowDescriptor.pathBuilders - override def receive = ioReceive orElse super.receive - + override def receive = ioReceive orElse LoggingReceive { + case Finalize => performActionThenRespond(afterAll()(context.dispatcher), FinalizationFailed)(context.dispatcher) + } + + private def performActionThenRespond(operation: => Future[BackendWorkflowLifecycleActorResponse], + onFailure: (Throwable) => BackendWorkflowLifecycleActorResponse) + (implicit ec: ExecutionContext) = { + val respondTo: ActorRef = sender + operation onComplete { + case Success(r) => respondTo ! r + case Failure(t) => respondTo ! onFailure(t) + } + } + private def copyWorkflowOutputs(workflowOutputsFilePath: String): Future[Seq[Unit]] = { val workflowOutputsPath = buildPath(workflowOutputsFilePath) @@ -39,7 +54,7 @@ class CopyWorkflowOutputsActor(workflowId: WorkflowId, override val ioActor: Act val copies = outputFilePaths map { case (srcPath, dstPath) => dstPath.createDirectories() - copyAsync(srcPath, dstPath, overwrite = true) + copyAsync(srcPath, dstPath) } Future.sequence(copies) @@ -86,7 +101,10 @@ class CopyWorkflowOutputsActor(workflowId: WorkflowId, override val ioActor: Act backendFactory.getExecutionRootPath(workflowDescriptor.backendDescriptor, config.backendConfig, initializationData) } - final override def afterAll()(implicit ec: ExecutionContext): Future[FinalizationResponse] = { + /** + * Happens after everything else runs + */ + final def afterAll()(implicit ec: ExecutionContext): Future[FinalizationResponse] = { workflowDescriptor.getWorkflowOption(FinalWorkflowOutputsDir) match { case Some(outputs) => copyWorkflowOutputs(outputs) map { _ => FinalizationSuccess } case None => Future.successful(FinalizationSuccess) diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/EngineWorkflowFinalizationActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/EngineWorkflowFinalizationActor.scala deleted file mode 100644 index 401396cae..000000000 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/EngineWorkflowFinalizationActor.scala +++ /dev/null @@ -1,30 +0,0 @@ -package cromwell.engine.workflow.lifecycle - -import akka.actor.{Actor, ActorLogging, ActorRef} -import akka.event.LoggingReceive -import cromwell.backend.BackendLifecycleActor.BackendWorkflowLifecycleActorResponse -import cromwell.backend.BackendWorkflowFinalizationActor.{FinalizationFailed, FinalizationResponse, Finalize} - -import scala.concurrent.{ExecutionContext, Future} -import scala.util.{Failure, Success} - -trait EngineWorkflowFinalizationActor extends Actor with ActorLogging { - def receive: Receive = LoggingReceive { - case Finalize => performActionThenRespond(afterAll()(context.dispatcher), FinalizationFailed)(context.dispatcher) - } - - protected def performActionThenRespond(operation: => Future[BackendWorkflowLifecycleActorResponse], - onFailure: (Throwable) => BackendWorkflowLifecycleActorResponse) - (implicit ec: ExecutionContext) = { - val respondTo: ActorRef = sender - operation onComplete { - case Success(r) => respondTo ! r - case Failure(t) => respondTo ! onFailure(t) - } - } - - /** - * Happens after everything else runs - */ - def afterAll()(implicit ec: ExecutionContext): Future[FinalizationResponse] -} From 7947d4c54e3e07f9519c6eef3a1d94ac537572d7 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Mon, 10 Apr 2017 11:07:48 -0400 Subject: [PATCH 004/134] Upgrade test script --- scripts/test_upgrade/custom_labels.json | 3 + scripts/test_upgrade/defaultDocker.json | 5 + scripts/test_upgrade/scatter_files.wdl | 60 +++++++ .../test_upgrade/scatter_files_input_part1_ab.json | 5 + .../test_upgrade/scatter_files_input_part1_c.json | 3 + .../test_upgrade/scatter_files_input_part2.json | 3 + .../test_upgrade/scatter_files_input_part3.json | 3 + .../test_upgrade/scatter_files_input_part4.json | 3 + .../test_upgrade/scatter_files_input_part5.json | 3 + .../test_upgrade/scatter_files_input_part6.json | 3 + scripts/test_upgrade/test_upgrade.sh | 182 +++++++++++++++++++++ 11 files changed, 273 insertions(+) create mode 100644 scripts/test_upgrade/custom_labels.json create mode 100644 scripts/test_upgrade/defaultDocker.json create mode 100644 scripts/test_upgrade/scatter_files.wdl create mode 100644 scripts/test_upgrade/scatter_files_input_part1_ab.json create mode 100644 scripts/test_upgrade/scatter_files_input_part1_c.json create mode 100644 scripts/test_upgrade/scatter_files_input_part2.json create mode 100644 scripts/test_upgrade/scatter_files_input_part3.json create mode 100644 scripts/test_upgrade/scatter_files_input_part4.json create mode 100644 scripts/test_upgrade/scatter_files_input_part5.json create mode 100644 scripts/test_upgrade/scatter_files_input_part6.json create mode 100755 scripts/test_upgrade/test_upgrade.sh diff --git a/scripts/test_upgrade/custom_labels.json b/scripts/test_upgrade/custom_labels.json new file mode 100644 index 000000000..d3e90d2bf --- /dev/null +++ b/scripts/test_upgrade/custom_labels.json @@ -0,0 +1,3 @@ +{ + "test-upgrade-label": "goodvalue" +} diff --git a/scripts/test_upgrade/defaultDocker.json b/scripts/test_upgrade/defaultDocker.json new file mode 100644 index 000000000..f4713ec3e --- /dev/null +++ b/scripts/test_upgrade/defaultDocker.json @@ -0,0 +1,5 @@ +{ + "default_runtime_attributes": { + "docker": "library/ubuntu:latest" + } +} diff --git a/scripts/test_upgrade/scatter_files.wdl b/scripts/test_upgrade/scatter_files.wdl new file mode 100644 index 000000000..636faae37 --- /dev/null +++ b/scripts/test_upgrade/scatter_files.wdl @@ -0,0 +1,60 @@ +task mkFile { + + Int index + + command { + echo "content-${index}" + } + + output { + File f = stdout() + } + runtime { docker: "ubuntu:latest" } +} + +task catFile { + File f_in + + command { + sleep 50 + cat ${f_in} + } + + output { + File f = stdout() + } +} + +task gather { + Array[File] inputs + + command { + cat ${sep=" " inputs} + } + + output { + String result = stdout() + } + runtime { docker: "ubuntu:latest" } +} + +workflow scatter_files { + + Int scatter_width_part_1 + Int scatter_width_part_2 + Int scatter_width_part_3 + Int scatter_width_part_4 + Int scatter_width_part_5 + Int scatter_width_part_6 + + Int sum = scatter_width_part_1 + scatter_width_part_2 + scatter_width_part_3 + scatter_width_part_4 + scatter_width_part_5 + scatter_width_part_6 + + Array[Int] xs = range(sum / 4) + + scatter (x in xs) { + call mkFile { input: index = x } + call catFile { input: f_in = mkFile.f} + } + + call gather { input: inputs = catFile.f } +} diff --git a/scripts/test_upgrade/scatter_files_input_part1_ab.json b/scripts/test_upgrade/scatter_files_input_part1_ab.json new file mode 100644 index 000000000..a706d3f72 --- /dev/null +++ b/scripts/test_upgrade/scatter_files_input_part1_ab.json @@ -0,0 +1,5 @@ +[{ + "scatter_files.scatter_width_part_1": 2 +},{ + "scatter_files.scatter_width_part_1": 2 +}] diff --git a/scripts/test_upgrade/scatter_files_input_part1_c.json b/scripts/test_upgrade/scatter_files_input_part1_c.json new file mode 100644 index 000000000..1b04da787 --- /dev/null +++ b/scripts/test_upgrade/scatter_files_input_part1_c.json @@ -0,0 +1,3 @@ +{ + "scatter_files.scatter_width_part_1": 2 +} diff --git a/scripts/test_upgrade/scatter_files_input_part2.json b/scripts/test_upgrade/scatter_files_input_part2.json new file mode 100644 index 000000000..5208451da --- /dev/null +++ b/scripts/test_upgrade/scatter_files_input_part2.json @@ -0,0 +1,3 @@ +{ + "scatter_files.scatter_width_part_2": 2 +} diff --git a/scripts/test_upgrade/scatter_files_input_part3.json b/scripts/test_upgrade/scatter_files_input_part3.json new file mode 100644 index 000000000..a9611598c --- /dev/null +++ b/scripts/test_upgrade/scatter_files_input_part3.json @@ -0,0 +1,3 @@ +{ + "scatter_files.scatter_width_part_3": 2 +} diff --git a/scripts/test_upgrade/scatter_files_input_part4.json b/scripts/test_upgrade/scatter_files_input_part4.json new file mode 100644 index 000000000..e61982340 --- /dev/null +++ b/scripts/test_upgrade/scatter_files_input_part4.json @@ -0,0 +1,3 @@ +{ + "scatter_files.scatter_width_part_4": 2 +} diff --git a/scripts/test_upgrade/scatter_files_input_part5.json b/scripts/test_upgrade/scatter_files_input_part5.json new file mode 100644 index 000000000..4cdabf350 --- /dev/null +++ b/scripts/test_upgrade/scatter_files_input_part5.json @@ -0,0 +1,3 @@ +{ + "scatter_files.scatter_width_part_5": 2 +} diff --git a/scripts/test_upgrade/scatter_files_input_part6.json b/scripts/test_upgrade/scatter_files_input_part6.json new file mode 100644 index 000000000..d0d5e4ecd --- /dev/null +++ b/scripts/test_upgrade/scatter_files_input_part6.json @@ -0,0 +1,3 @@ +{ + "scatter_files.scatter_width_part_6": 2 +} diff --git a/scripts/test_upgrade/test_upgrade.sh b/scripts/test_upgrade/test_upgrade.sh new file mode 100755 index 000000000..2b47d0c32 --- /dev/null +++ b/scripts/test_upgrade/test_upgrade.sh @@ -0,0 +1,182 @@ +#!/bin/bash +# +# test_upgrade.sh +# +# What this script does: +# - Starts Cromwell using the "previous" cromwell JAR/configuration +# - Sends 3 jobs to Cromwell (2 batched, one single) +# - Waits 6 minutes +# - Sends the same 3 jobs again, in the same way +# - Shutdown Cromwell with jobs still running +# - Restarts Cromwell using the "new" cromwell JAR/configuration +# - Sends the same three jobs again +# - Waits for everything to complete +# +# What it doesn't do (yet... but maybe for the C27 release!): +# - Guarantee that any jobs are in any specific states at the point that it shuts down. +# - Check the status of jobs after completing. +# - Check that operations IDs weren't duplicated after restarting. +# - DRY it up with functions replacing the outrageous copy/pasting. +# + +PREVIOUS_CROMWELL_JAR=cromwell-25-31ae549-SNAP.jar +PREVIOUS_CROMWELL_CONF=jes.conf + +NEW_CROMWELL_JAR=cromwell-26-88630db-SNAP.jar +NEW_CROMWELL_CONF=jes.conf + +OUT_DIR=out + +PREVIOUS_CROMWELL_LOG="$OUT_DIR/previous_version.log" +NEW_CROMWELL_LOG="$OUT_DIR/new_version.log" + +START_DT=$(date "+%Y-%m-%dT%H:%M:%S.000-04:00") +echo "Starting $0 at: $START_DT" +# Now we've printed it, make it URL-safe +START_DT="${START_DT//:/%3A}" + +pgrep -q cromwell +ALREADY_RUNNING=$? +if [ $ALREADY_RUNNING -ne 1 ]; +then + echo "Oops! Cromwell (🐖 ) is already running!" + pgrep cromwell + exit 1 +fi + +rm -r $OUT_DIR +mkdir $OUT_DIR + +echo -n "Starting Cromwell (🐖 )..." +java -Dconfig.file="$PREVIOUS_CROMWELL_CONF" -jar "$PREVIOUS_CROMWELL_JAR" server &> "$PREVIOUS_CROMWELL_LOG" & +CROMWELL_PID=$! +echo "started (PID: $CROMWELL_PID)." + +echo -n "Waiting for (🐖 ) API..." +READY=-1 +while [ $READY -ne 0 ]; +do + sleep 1 + curl -X GET --header "Accept: application/json" "http://localhost:8000/api/workflows/v1/backends" &>/dev/null + READY=$? +done +echo "ready." + +for i in 1 2 +do + echo -n "submitting job ${i}a and ${i}b..." + RESULT_FILE="$OUT_DIR/submitResult_${i}_ab.json" + curl -X POST --header "Accept: application/json" "http://localhost:8000/api/workflows/v1/batch" \ + -F wdlSource=@scatter_files.wdl \ + -F workflowInputs=@scatter_files_input_part1_ab.json \ + -F workflowInputs_2=@scatter_files_input_part2.json \ + -F workflowInputs_3=@scatter_files_input_part3.json \ + -F workflowInputs_4=@scatter_files_input_part4.json \ + -F workflowInputs_5=@scatter_files_input_part5.json \ + -F workflowInputs_6=@scatter_files_input_part6.json \ + -F workflowOptions=@defaultDocker.json \ + -F customLabels=@custom_labels.json &> $"RESULT_FILE" + echo "done (Response in: $RESULT_FILE)." + echo -n "submitting job ${i}c..." + RESULT_FILE="$OUT_DIR/submitResult_${i}_c.json" + curl -X POST --header "Accept: application/json" "http://localhost:8000/api/workflows/v1/batch" \ + -F wdlSource=@scatter_files.wdl \ + -F workflowInputs=@scatter_files_input_part1_c.json \ + -F workflowInputs_2=@scatter_files_input_part2.json \ + -F workflowInputs_3=@scatter_files_input_part3.json \ + -F workflowInputs_4=@scatter_files_input_part4.json \ + -F workflowInputs_5=@scatter_files_input_part5.json \ + -F workflowInputs_6=@scatter_files_input_part6.json \ + -F workflowOptions=@defaultDocker.json \ + -F customLabels=@custom_labels.json &> $"RESULT_FILE" + echo "done (Response in: $RESULT_FILE)." + [ "$i" -eq 1 ] && sleep 360 +done + +# Step two: upgrade cromwell +kill $CROMWELL_PID + +pgrep -q cromwell +STILL_RUNNING=$? +while [ $STILL_RUNNING -eq 0 ]; +do + echo "Waiting for Cromwell(🐖 ) to exit..." + sleep 1 + pgrep -q cromwell + STILL_RUNNING=$? +done + +echo -n "Starting Cromwell(🐖 )... " +java -Dconfig.file=$NEW_CROMWELL_CONF -jar $NEW_CROMWELL_JAR server &> $NEW_CROMWELL_LOG & +CROMWELL_PID=$! +echo "started (PID=$CROMWELL_PID)." + +echo -n "Waiting for 🐖 API..." +READY=-1 +while [ "$READY" -ne "0" ]; +do + sleep 1 + curl -X GET --header "Accept: application/json" "http://localhost:8000/api/workflows/v1/backends" &>/dev/null + READY=$? +done +echo "ready." + +i=3 +echo -n "submitting job ${i}a and ${i}b..." +RESULT_FILE="$OUT_DIR/submitResult_${i}_ab.json" +curl -X POST --header "Accept: application/json" "http://localhost:8000/api/workflows/v1/batch" \ + -F wdlSource=@scatter_files.wdl \ + -F workflowInputs=@scatter_files_input_part1_ab.json \ + -F workflowInputs_2=@scatter_files_input_part2.json \ + -F workflowInputs_3=@scatter_files_input_part3.json \ + -F workflowInputs_4=@scatter_files_input_part4.json \ + -F workflowInputs_5=@scatter_files_input_part5.json \ + -F workflowInputs_6=@scatter_files_input_part6.json \ + -F workflowOptions=@defaultDocker.json \ + -F customLabels=@custom_labels.json &> $"RESULT_FILE" +echo "done (Response in: $RESULT_FILE)." +echo -n "submitting job ${i}c..." +RESULT_FILE="$OUT_DIR/submitResult_${i}_c.json" +curl -X POST --header "Accept: application/json" "http://localhost:8000/api/workflows/v1/batch" \ + -F wdlSource=@scatter_files.wdl \ + -F workflowInputs=@scatter_files_input_part1_c.json \ + -F workflowInputs_2=@scatter_files_input_part2.json \ + -F workflowInputs_3=@scatter_files_input_part3.json \ + -F workflowInputs_4=@scatter_files_input_part4.json \ + -F workflowInputs_5=@scatter_files_input_part5.json \ + -F workflowInputs_6=@scatter_files_input_part6.json \ + -F workflowOptions=@defaultDocker.json \ + -F customLabels=@custom_labels.json &> $"RESULT_FILE" +echo "done (Response in: $RESULT_FILE)." + +# Step 3: Wait until everything's done: +echo -n "Waiting for the run to complete at a rate of one 🐷 per minute..." +DONE=1 +while [ "$DONE" -ne "0" ]; +do + CURLED=$(curl -X GET --header "Accept: application/json" "http://localhost:8000/api/workflows/v1/query?start=${START_DT}&status=Running" 2>/dev/null) + grep -q ": \[\]" <<< "$CURLED" + FINISHED=$? + grep -q '"status": "fail"' <<< "$CURLED" + ERROR=$? + [ $ERROR -eq 0 ] && ( echo "Error: $CURLED" ) + if [ $ERROR -eq 0 ] || [ $FINISHED -eq 0 ]; + then + DONE=0 + else + echo -n ".🐷 ." + sleep 60 + fi +done +echo "...done" + +kill $CROMWELL_PID + +# Step 4: analyse logs to make sure things worked out: +echo "Previous version's operations IDs:" +grep operations "$PREVIOUS_CROMWELL_LOG" | sed "s/.*\ - \(.*\)/\1/g" + +echo +echo "New version's operations IDs:" +grep operations "$NEW_CROMWELL_LOG" | sed "s/.*\ - \(.*\)/\1/g" +# TODO... From 9460d4760f58c30308651692ed8cb8dbde0b1e73 Mon Sep 17 00:00:00 2001 From: Ruchi Date: Fri, 14 Apr 2017 15:06:17 -0400 Subject: [PATCH 005/134] Config-specified Default runtime attributes (#2121) grab runtime attribute defaults from reference.conf, add deprecation warnings, introduced the BadDefaultAttribute WdlValue. --- .../BackendWorkflowInitializationActor.scala | 2 +- .../backend/RuntimeAttributeDefinition.scala | 1 - .../src/main/scala/cromwell/backend/backend.scala | 9 +- .../standard/StandardInitializationActor.scala | 2 +- ...StandardValidatedRuntimeAttributesBuilder.scala | 5 +- .../ContinueOnReturnCodeValidation.scala | 12 +-- .../backend/validation/CpuValidation.scala | 13 +-- .../validation/FailOnStderrValidation.scala | 14 +++- .../backend/validation/MemoryValidation.scala | 20 +++-- .../validation/RuntimeAttributesValidation.scala | 49 +++++++++-- .../test/scala/cromwell/backend/BackendSpec.scala | 4 - .../BackendWorkflowInitializationActorSpec.scala | 45 +++++----- .../test/scala/cromwell/backend/TestConfig.scala | 44 ++++++++++ .../scala/cromwell/backend/io/JobPathsSpec.scala | 5 +- ...dardValidatedRuntimeAttributesBuilderSpec.scala | 12 +-- .../RuntimeAttributesValidationSpec.scala | 82 +++++++++++++++++++ core/src/main/resources/reference.conf | 27 +++++- .../htcondor/HtCondorInitializationActor.scala | 8 +- .../impl/htcondor/HtCondorRuntimeAttributes.scala | 2 +- .../htcondor/HtCondorInitializationActorSpec.scala | 4 +- .../cromwell/backend/impl/jes/JesAttributes.scala | 38 ++++----- .../backend/impl/jes/JesConfiguration.scala | 2 +- .../impl/jes/JesJobCachingActorHelper.scala | 3 +- .../backend/impl/jes/JesRuntimeAttributes.scala | 95 ++++++++++++---------- .../backend/impl/jes/JesAttributesSpec.scala | 14 +--- .../backend/impl/jes/JesConfigurationSpec.scala | 19 +++-- .../impl/jes/JesInitializationActorSpec.scala | 29 ++++++- .../impl/jes/JesRuntimeAttributesSpec.scala | 76 +++++++++++++---- .../cromwell/backend/impl/jes/JesTestConfig.scala | 35 ++++++++ .../impl/sfs/config/DeclarationValidation.scala | 2 +- .../SharedFileSystemInitializationActorSpec.scala | 5 +- .../SharedFileSystemJobExecutionActorSpec.scala | 50 ++++++------ .../sfs/TestLocalAsyncJobExecutionActor.scala | 2 +- .../impl/spark/SparkInitializationActorSpec.scala | 4 +- .../backend/impl/tes/TesConfiguration.scala | 1 + .../backend/impl/tes/TesInitializationActor.scala | 2 +- .../impl/tes/TesJobCachingActorHelper.scala | 2 +- .../backend/impl/tes/TesRuntimeAttributes.scala | 53 +++++++----- .../impl/tes/TesInitializationActorSpec.scala | 11 +++ .../backend/impl/tes/TesJobPathsSpec.scala | 32 ++------ .../impl/tes/TesRuntimeAttributesSpec.scala | 21 +++-- .../cromwell/backend/impl/tes/TesTestConfig.scala | 27 ++++++ .../backend/impl/tes/TesWorkflowPathsSpec.scala | 24 +----- 43 files changed, 634 insertions(+), 273 deletions(-) create mode 100644 backend/src/test/scala/cromwell/backend/TestConfig.scala create mode 100644 supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesTestConfig.scala diff --git a/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala b/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala index 5bf1d05fd..a7ad6f5c7 100644 --- a/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala +++ b/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala @@ -69,7 +69,7 @@ trait BackendWorkflowInitializationActor extends BackendWorkflowLifecycleActor w * return `true` in both cases. */ protected def continueOnReturnCodePredicate(valueRequired: Boolean)(wdlExpressionMaybe: Option[WdlValue]): Boolean = { - ContinueOnReturnCodeValidation.default.validateOptionalExpression(wdlExpressionMaybe) + ContinueOnReturnCodeValidation.default(configurationDescriptor.backendRuntimeConfig).validateOptionalExpression(wdlExpressionMaybe) } protected def runtimeAttributeValidators: Map[String, Option[WdlValue] => Boolean] diff --git a/backend/src/main/scala/cromwell/backend/RuntimeAttributeDefinition.scala b/backend/src/main/scala/cromwell/backend/RuntimeAttributeDefinition.scala index 0c9bf8727..506bbe4eb 100644 --- a/backend/src/main/scala/cromwell/backend/RuntimeAttributeDefinition.scala +++ b/backend/src/main/scala/cromwell/backend/RuntimeAttributeDefinition.scala @@ -47,7 +47,6 @@ object RuntimeAttributeDefinition { case (runtimeAttributeDefinition, Success(jsValue)) => runtimeAttributeDefinition.name -> jsValue.convertTo[WdlValue] case (RuntimeAttributeDefinition(name, Some(factoryDefault), _), _) => name -> factoryDefault } - specifiedAttributes ++ defaults } } diff --git a/backend/src/main/scala/cromwell/backend/backend.scala b/backend/src/main/scala/cromwell/backend/backend.scala index 5839f268b..4b2619ba0 100644 --- a/backend/src/main/scala/cromwell/backend/backend.scala +++ b/backend/src/main/scala/cromwell/backend/backend.scala @@ -65,7 +65,14 @@ case class BackendWorkflowDescriptor(id: WorkflowId, /** * For passing to a BackendActor construction time */ -case class BackendConfigurationDescriptor(backendConfig: Config, globalConfig: Config) +case class BackendConfigurationDescriptor(backendConfig: Config, globalConfig: Config) { + + lazy val backendRuntimeConfig = backendConfig.hasPath("default-runtime-attributes") match { + case true => Option(backendConfig.getConfig("default-runtime-attributes")) + case false => None + } + +} final case class AttemptedLookupResult(name: String, value: Try[WdlValue]) { def toPair = name -> value diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardInitializationActor.scala b/backend/src/main/scala/cromwell/backend/standard/StandardInitializationActor.scala index d3389bcc7..89e3c48b2 100644 --- a/backend/src/main/scala/cromwell/backend/standard/StandardInitializationActor.scala +++ b/backend/src/main/scala/cromwell/backend/standard/StandardInitializationActor.scala @@ -68,7 +68,7 @@ class StandardInitializationActor(val standardParams: StandardInitializationActo * @return runtime attributes builder with possible custom validations */ def runtimeAttributesBuilder: StandardValidatedRuntimeAttributesBuilder = - StandardValidatedRuntimeAttributesBuilder.default + StandardValidatedRuntimeAttributesBuilder.default(configurationDescriptor.backendRuntimeConfig) override protected lazy val runtimeAttributeValidators: Map[String, (Option[WdlValue]) => Boolean] = { runtimeAttributesBuilder.validatorMap diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilder.scala b/backend/src/main/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilder.scala index 20a649e00..37f037b57 100644 --- a/backend/src/main/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilder.scala +++ b/backend/src/main/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilder.scala @@ -1,5 +1,6 @@ package cromwell.backend.standard +import com.typesafe.config.Config import cromwell.backend.validation._ /** @@ -29,8 +30,8 @@ object StandardValidatedRuntimeAttributesBuilder { * * Additional runtime attribute validations may be added by calling `withValidation` on the default. */ - lazy val default: StandardValidatedRuntimeAttributesBuilder = { - val required = Seq(ContinueOnReturnCodeValidation.default, FailOnStderrValidation.default) + def default(backendRuntimeConfig: Option[Config]): StandardValidatedRuntimeAttributesBuilder = { + val required = Seq(ContinueOnReturnCodeValidation.default(backendRuntimeConfig), FailOnStderrValidation.default(backendRuntimeConfig)) val custom = Seq.empty StandardValidatedRuntimeAttributesBuilderImpl(custom, required) } diff --git a/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala b/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala index d0ca49339..03e9d5b1a 100644 --- a/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala @@ -4,6 +4,7 @@ import cats.data.Validated.{Invalid, Valid} import cats.instances.list._ import cats.syntax.traverse._ import cats.syntax.validated._ +import com.typesafe.config.Config import cromwell.backend.validation.RuntimeAttributesValidation._ import lenthall.validation.ErrorOr._ import wdl4s.types.{WdlArrayType, WdlIntegerType, WdlStringType, WdlType} @@ -17,15 +18,16 @@ import scala.util.Try * * `instance` returns an validation that errors when no attribute is specified. * - * The default returns a `ContinueOnReturnCodeSet(0)` when no attribute is specified. + * `configDefaultWdlValue` returns the value of the attribute as specified by the + * reference.conf file, coerced into a WdlValue. * - * `optional` can be used return the validated value as an `Option`, wrapped in a `Some`, if present, or `None` if not - * found. + * `default` a validation with the default value specified by the reference.conf file. */ object ContinueOnReturnCodeValidation { lazy val instance: RuntimeAttributesValidation[ContinueOnReturnCode] = new ContinueOnReturnCodeValidation - lazy val default: RuntimeAttributesValidation[ContinueOnReturnCode] = instance.withDefault(WdlInteger(0)) - lazy val optional: OptionalRuntimeAttributesValidation[ContinueOnReturnCode] = default.optional + def default(runtimeConfig: Option[Config]): RuntimeAttributesValidation[ContinueOnReturnCode] = instance.withDefault( + configDefaultWdlValue(runtimeConfig) getOrElse WdlInteger(0)) + def configDefaultWdlValue(runtimeConfig: Option[Config]): Option[WdlValue] = instance.configDefaultWdlValue(runtimeConfig) } class ContinueOnReturnCodeValidation extends RuntimeAttributesValidation[ContinueOnReturnCode] { diff --git a/backend/src/main/scala/cromwell/backend/validation/CpuValidation.scala b/backend/src/main/scala/cromwell/backend/validation/CpuValidation.scala index 5a5d09730..81cce8d0d 100644 --- a/backend/src/main/scala/cromwell/backend/validation/CpuValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/CpuValidation.scala @@ -1,6 +1,7 @@ package cromwell.backend.validation import cats.syntax.validated._ +import com.typesafe.config.Config import lenthall.validation.ErrorOr.ErrorOr import wdl4s.types.WdlIntegerType import wdl4s.values.{WdlInteger, WdlValue} @@ -10,15 +11,15 @@ import wdl4s.values.{WdlInteger, WdlValue} * * `instance` returns an validation that errors when no attribute is specified. * - * The default returns `1` when no attribute is specified. + * `default` a hardcoded default WdlValue for Cpu. * - * `optional` can be used return the validated value as an `Option`, wrapped in a `Some`, if present, or `None` if not - * found. + * `configDefaultWdlValue` returns the value of the attribute as specified by the + * reference.conf file, coerced into a WdlValue. */ -object CpuValidation extends { +object CpuValidation { lazy val instance: RuntimeAttributesValidation[Int] = new CpuValidation - lazy val default: RuntimeAttributesValidation[Int] = instance.withDefault(WdlInteger(1)) - lazy val optional: OptionalRuntimeAttributesValidation[Int] = default.optional + lazy val default: WdlValue = WdlInteger(1) + def configDefaultWdlValue(config: Option[Config]): Option[WdlValue] = instance.configDefaultWdlValue(config) } class CpuValidation extends IntRuntimeAttributesValidation(RuntimeAttributesKeys.CpuKey) { diff --git a/backend/src/main/scala/cromwell/backend/validation/FailOnStderrValidation.scala b/backend/src/main/scala/cromwell/backend/validation/FailOnStderrValidation.scala index 6a7ea6ee2..c57aaa246 100644 --- a/backend/src/main/scala/cromwell/backend/validation/FailOnStderrValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/FailOnStderrValidation.scala @@ -1,6 +1,7 @@ package cromwell.backend.validation -import wdl4s.values.WdlBoolean +import com.typesafe.config.Config +import wdl4s.values.{WdlBoolean, WdlValue} /** * Validates the "failOnStderr" runtime attribute as a Boolean or a String 'true' or 'false', returning the value as a @@ -8,12 +9,17 @@ import wdl4s.values.WdlBoolean * * `instance` returns an validation that errors when no attribute is specified. * - * The default returns `false` when no attribute is specified. + * `configDefaultWdlValue` returns the value of the attribute as specified by the + * reference.conf file, coerced into a WdlValue. + * + * `default` a validation with the default value specified by the reference.conf file. */ + object FailOnStderrValidation { lazy val instance: RuntimeAttributesValidation[Boolean] = new FailOnStderrValidation - lazy val default: RuntimeAttributesValidation[Boolean] = instance.withDefault(WdlBoolean(false)) - lazy val optional: OptionalRuntimeAttributesValidation[Boolean] = default.optional + def default(runtimeConfig: Option[Config]): RuntimeAttributesValidation[Boolean] = instance.withDefault( + configDefaultWdlValue(runtimeConfig) getOrElse WdlBoolean(false)) + def configDefaultWdlValue(runtimeConfig: Option[Config]): Option[WdlValue] = instance.configDefaultWdlValue(runtimeConfig) } class FailOnStderrValidation extends BooleanRuntimeAttributesValidation(RuntimeAttributesKeys.FailOnStderrKey) { diff --git a/backend/src/main/scala/cromwell/backend/validation/MemoryValidation.scala b/backend/src/main/scala/cromwell/backend/validation/MemoryValidation.scala index d493e357e..7af6271b9 100644 --- a/backend/src/main/scala/cromwell/backend/validation/MemoryValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/MemoryValidation.scala @@ -1,29 +1,39 @@ package cromwell.backend.validation import cats.syntax.validated._ +import com.typesafe.config.Config import cromwell.backend.MemorySize import lenthall.validation.ErrorOr._ import wdl4s.parser.MemoryUnit import wdl4s.types.{WdlIntegerType, WdlStringType} import wdl4s.values.{WdlInteger, WdlString, WdlValue} +import scala.util.{Failure, Success} + /** * Validates the "memory" runtime attribute as an Integer or String with format '8 GB', returning the value as a * `MemorySize`. * * `instance` returns an validation that errors when no attribute is specified. * - * There is no default, however `optional` can be used to validate the attribute and return the validated value as an - * `Option`, wrapped in an `Some`, if present, or `None` if not found. + * `configDefaultWdlValue` returns the value of the attribute as specified by the + * reference.conf file, coerced into a WdlValue. + * + * `optional` can be used to return the validated value as an `Option`, + * wrapped in a `Some`, if present, or `None` if not found. * * `withDefaultMemory` can be used to create a memory validation that defaults to a particular memory size. */ object MemoryValidation { lazy val instance: RuntimeAttributesValidation[MemorySize] = new MemoryValidation lazy val optional: OptionalRuntimeAttributesValidation[MemorySize] = instance.optional - - def withDefaultMemory(memorySize: MemorySize): RuntimeAttributesValidation[MemorySize] = - instance.withDefault(WdlInteger(memorySize.bytes.toInt)) + def configDefaultString(config: Option[Config]): Option[String] = instance.configDefaultValue(config) + def withDefaultMemory(memorySize: String): RuntimeAttributesValidation[MemorySize] = { + MemorySize.parse(memorySize) match { + case Success(memory) => instance.withDefault(WdlInteger(memory.bytes.toInt)) + case Failure(_) => instance.withDefault(BadDefaultAttribute(WdlString(memorySize.toString))) + } + } private[validation] val wrongAmountFormat = s"Expecting ${RuntimeAttributesKeys.MemoryKey} runtime attribute value greater than 0 but got %s" diff --git a/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala b/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala index f8ff1441a..5a868f416 100644 --- a/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala @@ -2,11 +2,12 @@ package cromwell.backend.validation import cats.data.{NonEmptyList, Validated} import cats.syntax.validated._ +import com.typesafe.config.Config import cromwell.backend.{MemorySize, RuntimeAttributeDefinition} import lenthall.validation.ErrorOr._ import org.slf4j.Logger import wdl4s.expression.PureStandardLibraryFunctions -import wdl4s.types.{WdlBooleanType, WdlIntegerType, WdlType} +import wdl4s.types.{WdlBooleanType, WdlIntegerType, WdlStringType, WdlType} import wdl4s.values._ import wdl4s.{NoLookup, WdlExpression} @@ -20,16 +21,16 @@ object RuntimeAttributesValidation { } def validateDocker(docker: Option[WdlValue], onMissingKey: => ErrorOr[Option[String]]): ErrorOr[Option[String]] = { - validateWithValidation(docker, DockerValidation.optional, onMissingKey) + validateWithValidation(docker, DockerValidation.instance.optional, onMissingKey) } def validateFailOnStderr(value: Option[WdlValue], onMissingKey: => ErrorOr[Boolean]): ErrorOr[Boolean] = { - validateWithValidation(value, FailOnStderrValidation.default, onMissingKey) + validateWithValidation(value, FailOnStderrValidation.instance, onMissingKey) } def validateContinueOnReturnCode(value: Option[WdlValue], onMissingKey: => ErrorOr[ContinueOnReturnCode]): ErrorOr[ContinueOnReturnCode] = { - validateWithValidation(value, ContinueOnReturnCodeValidation.default, onMissingKey) + validateWithValidation(value, ContinueOnReturnCodeValidation.instance, onMissingKey) } def validateMemory(value: Option[WdlValue], onMissingKey: => ErrorOr[MemorySize]): ErrorOr[MemorySize] = { @@ -37,7 +38,7 @@ object RuntimeAttributesValidation { } def validateCpu(cpu: Option[WdlValue], onMissingKey: => ErrorOr[Int]): ErrorOr[Int] = { - validateWithValidation(cpu, CpuValidation.default, onMissingKey) + validateWithValidation(cpu, CpuValidation.instance, onMissingKey) } private def validateWithValidation[T](valueOption: Option[WdlValue], @@ -213,6 +214,15 @@ object RuntimeAttributesValidation { } /** + * A wrapper class to classify config-based default runtime attributes + * that cannot be coerced into an acceptable WdlType. + */ +case class BadDefaultAttribute(badDefaultValue: WdlValue) extends WdlValue { + val wdlType = WdlStringType +} + + +/** * Performs a validation on a runtime attribute and returns some value. * * @tparam ValidatedType The type of the validated value. @@ -371,6 +381,35 @@ trait RuntimeAttributesValidation[ValidatedType] { final def withDefault(wdlValue: WdlValue): RuntimeAttributesValidation[ValidatedType] = RuntimeAttributesValidation.withDefault(this, wdlValue) + /** + * Returns the value of the default runtime attribute of a + * validation key as specified in the reference.conf. Given + * a value, this method coerces it into an optional + * WdlValue. In case the value cannot be succesfully coerced + * the value is wrapped as a "BadDefaultAttributeValue" type that + * is failed downstream by the ValidatedRuntimeAttributesBuilder. + * + * @param optionalRuntimeConfig Optional default runtime attributes config of a particular backend. + * @return The new version of this validation. + */ + final def configDefaultWdlValue(optionalRuntimeConfig: Option[Config]): Option[WdlValue] = { + optionalRuntimeConfig flatMap { config => + val value = config.getValue(key).unwrapped() + coercion.collectFirst({ + case wdlType if wdlType.coerceRawValue(value).isSuccess => { + wdlType.coerceRawValue(value).get + } + }) orElse Option(BadDefaultAttribute(WdlString(value.toString))) + } + } + + final def configDefaultValue(optionalRuntimeConfig: Option[Config]): Option[String] = { + optionalRuntimeConfig match { + case Some(config) if config.hasPath(key) => Option(config.getValue(key).unwrapped().toString) + case _ => None + } + } + /* Methods below provide aliases to expose protected methods to the package. Allows wrappers to wire their overrides to invoke the corresponding method on the inner object. diff --git a/backend/src/test/scala/cromwell/backend/BackendSpec.scala b/backend/src/test/scala/cromwell/backend/BackendSpec.scala index e0632ec1c..b3b3c3fc2 100644 --- a/backend/src/test/scala/cromwell/backend/BackendSpec.scala +++ b/backend/src/test/scala/cromwell/backend/BackendSpec.scala @@ -1,6 +1,5 @@ package cromwell.backend -import com.typesafe.config.ConfigFactory import cromwell.backend.BackendJobExecutionActor.{BackendJobExecutionResponse, JobFailedNonRetryableResponse, JobFailedRetryableResponse, JobSucceededResponse} import cromwell.backend.io.TestWorkflows._ import cromwell.core.callcaching.CallCachingEligible @@ -117,9 +116,6 @@ trait BackendSpec extends ScalaFutures with Matchers with Mockito { } } - lazy val emptyBackendConfig = BackendConfigurationDescriptor( - ConfigFactory.parseString("{}"), ConfigFactory.load()) - def firstJobDescriptorKey(workflowDescriptor: BackendWorkflowDescriptor): BackendJobDescriptorKey = { val call = workflowDescriptor.workflow.taskCalls.head BackendJobDescriptorKey(call, None, 1) diff --git a/backend/src/test/scala/cromwell/backend/BackendWorkflowInitializationActorSpec.scala b/backend/src/test/scala/cromwell/backend/BackendWorkflowInitializationActorSpec.scala index 82e62d641..d26df12f6 100644 --- a/backend/src/test/scala/cromwell/backend/BackendWorkflowInitializationActorSpec.scala +++ b/backend/src/test/scala/cromwell/backend/BackendWorkflowInitializationActorSpec.scala @@ -2,6 +2,7 @@ package cromwell.backend import akka.actor.ActorRef import akka.testkit.TestActorRef +import com.typesafe.config.ConfigFactory import cromwell.backend.validation.{ContinueOnReturnCodeFlag, ContinueOnReturnCodeSet, ContinueOnReturnCodeValidation, RuntimeAttributesKeys} import cromwell.core.{TestKitSuite, WorkflowOptions} import org.scalatest.prop.TableDrivenPropertyChecks @@ -30,9 +31,11 @@ class BackendWorkflowInitializationActorSpec extends TestKitSuite("BackendWorkfl testPredicateBackendWorkflowInitializationActor.continueOnReturnCodePredicate(valueRequired = false) } + val optionalConfig = Option(TestConfig.optionalRuntimeConfig) + it should "continueOnReturnCodePredicate" in { testContinueOnReturnCode(None) should be(true) - ContinueOnReturnCodeValidation.default.validateOptionalExpression(None) should be(true) + ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalExpression(None) should be(true) val booleanRows = Table( "value", @@ -66,9 +69,9 @@ class BackendWorkflowInitializationActorSpec extends TestKitSuite("BackendWorkfl val wdlValue = WdlBoolean(value) val result = true testContinueOnReturnCode(Option(wdlValue)) should be(result) - ContinueOnReturnCodeValidation.default.validateOptionalExpression(Option(wdlValue)) should be(result) + ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalExpression(Option(wdlValue)) should be(result) val valid = - ContinueOnReturnCodeValidation.default.validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> wdlValue)) + ContinueOnReturnCodeValidation.default(optionalConfig).validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> wdlValue)) valid.isValid should be(result) valid.toEither.right.get should be(ContinueOnReturnCodeFlag(value)) } @@ -77,9 +80,9 @@ class BackendWorkflowInitializationActorSpec extends TestKitSuite("BackendWorkfl val wdlValue = WdlString(value.toString) val result = true testContinueOnReturnCode(Option(wdlValue)) should be(result) - ContinueOnReturnCodeValidation.default.validateOptionalExpression(Option(wdlValue)) should be(result) + ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalExpression(Option(wdlValue)) should be(result) val valid = - ContinueOnReturnCodeValidation.default.validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> wdlValue)) + ContinueOnReturnCodeValidation.default(optionalConfig).validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> wdlValue)) valid.isValid should be(result) valid.toEither.right.get should be(ContinueOnReturnCodeFlag(value)) } @@ -88,7 +91,7 @@ class BackendWorkflowInitializationActorSpec extends TestKitSuite("BackendWorkfl val wdlValue = WdlExpression.fromString(value.toString) val result = true testContinueOnReturnCode(Option(wdlValue)) should be(result) - ContinueOnReturnCodeValidation.default.validateOptionalExpression(Option(wdlValue)) should be(result) + ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalExpression(Option(wdlValue)) should be(result) // NOTE: expressions are never valid to validate } @@ -96,9 +99,9 @@ class BackendWorkflowInitializationActorSpec extends TestKitSuite("BackendWorkfl val wdlValue = WdlInteger(value) val result = true testContinueOnReturnCode(Option(wdlValue)) should be(result) - ContinueOnReturnCodeValidation.default.validateOptionalExpression(Option(wdlValue)) should be(result) + ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalExpression(Option(wdlValue)) should be(result) val valid = - ContinueOnReturnCodeValidation.default.validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> wdlValue)) + ContinueOnReturnCodeValidation.default(optionalConfig).validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> wdlValue)) valid.isValid should be(result) valid.toEither.right.get should be(ContinueOnReturnCodeSet(Set(value))) } @@ -107,9 +110,9 @@ class BackendWorkflowInitializationActorSpec extends TestKitSuite("BackendWorkfl val wdlValue = WdlString(value.toString) val result = true testContinueOnReturnCode(Option(wdlValue)) should be(result) - ContinueOnReturnCodeValidation.default.validateOptionalExpression(Option(wdlValue)) should be(result) + ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalExpression(Option(wdlValue)) should be(result) val valid = - ContinueOnReturnCodeValidation.default.validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> wdlValue)) + ContinueOnReturnCodeValidation.default(optionalConfig).validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> wdlValue)) valid.isValid should be(result) valid.toEither.right.get should be(ContinueOnReturnCodeSet(Set(value))) } @@ -118,7 +121,7 @@ class BackendWorkflowInitializationActorSpec extends TestKitSuite("BackendWorkfl val wdlValue = WdlExpression.fromString(value.toString) val result = true testContinueOnReturnCode(Option(wdlValue)) should be(result) - ContinueOnReturnCodeValidation.default.validateOptionalExpression(Option(wdlValue)) should be(result) + ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalExpression(Option(wdlValue)) should be(result) // NOTE: expressions are never valid to validate } @@ -126,9 +129,9 @@ class BackendWorkflowInitializationActorSpec extends TestKitSuite("BackendWorkfl val wdlValue = WdlArray(WdlArrayType(WdlIntegerType), Seq(WdlInteger(value))) val result = true testContinueOnReturnCode(Option(wdlValue)) should be(result) - ContinueOnReturnCodeValidation.default.validateOptionalExpression(Option(wdlValue)) should be(result) + ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalExpression(Option(wdlValue)) should be(result) val valid = - ContinueOnReturnCodeValidation.default.validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> wdlValue)) + ContinueOnReturnCodeValidation.default(optionalConfig).validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> wdlValue)) valid.isValid should be(result) valid.toEither.right.get should be(ContinueOnReturnCodeSet(Set(value))) } @@ -137,9 +140,9 @@ class BackendWorkflowInitializationActorSpec extends TestKitSuite("BackendWorkfl val wdlValue = WdlArray(WdlArrayType(WdlStringType), Seq(WdlString(value.toString))) val result = true testContinueOnReturnCode(Option(wdlValue)) should be(result) - ContinueOnReturnCodeValidation.default.validateOptionalExpression(Option(wdlValue)) should be(result) + ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalExpression(Option(wdlValue)) should be(result) val valid = - ContinueOnReturnCodeValidation.default.validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> wdlValue)) + ContinueOnReturnCodeValidation.default(optionalConfig).validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> wdlValue)) valid.isValid should be(result) valid.toEither.right.get should be(ContinueOnReturnCodeSet(Set(value))) } @@ -148,7 +151,7 @@ class BackendWorkflowInitializationActorSpec extends TestKitSuite("BackendWorkfl val wdlValue = WdlArray(WdlArrayType(WdlExpressionType), Seq(WdlExpression.fromString(value.toString))) val result = false testContinueOnReturnCode(Option(wdlValue)) should be(result) - ContinueOnReturnCodeValidation.default.validateOptionalExpression(Option(wdlValue)) should be(result) + ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalExpression(Option(wdlValue)) should be(result) // NOTE: expressions are never valid to validate } @@ -156,16 +159,16 @@ class BackendWorkflowInitializationActorSpec extends TestKitSuite("BackendWorkfl val wdlValue = WdlExpression.fromString(expression) val result = true testContinueOnReturnCode(Option(wdlValue)) should be(result) - ContinueOnReturnCodeValidation.default.validateOptionalExpression(Option(wdlValue)) should be(result) + ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalExpression(Option(wdlValue)) should be(result) // NOTE: expressions are never valid to validate } forAll(invalidWdlValueRows) { wdlValue => val result = false testContinueOnReturnCode(Option(wdlValue)) should be(result) - ContinueOnReturnCodeValidation.default.validateOptionalExpression(Option(wdlValue)) should be(result) + ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalExpression(Option(wdlValue)) should be(result) val valid = - ContinueOnReturnCodeValidation.default.validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> wdlValue)) + ContinueOnReturnCodeValidation.default(optionalConfig).validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> wdlValue)) valid.isValid should be(result) valid.toEither.left.get.toList should contain theSameElementsAs List( "Expecting continueOnReturnCode runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]" @@ -177,6 +180,7 @@ class BackendWorkflowInitializationActorSpec extends TestKitSuite("BackendWorkfl } class TestPredicateBackendWorkflowInitializationActor extends BackendWorkflowInitializationActor { + override val serviceRegistryActor: ActorRef = context.system.deadLetters override def calls: Set[TaskCall] = throw new NotImplementedError("calls") @@ -194,8 +198,7 @@ class TestPredicateBackendWorkflowInitializationActor extends BackendWorkflowIni override protected def workflowDescriptor: BackendWorkflowDescriptor = throw new NotImplementedError("workflowDescriptor") - override protected def configurationDescriptor: BackendConfigurationDescriptor = - throw new NotImplementedError("configurationDescriptor") + override protected def configurationDescriptor: BackendConfigurationDescriptor = BackendConfigurationDescriptor(TestConfig.sampleBackendRuntimeConfig, ConfigFactory.empty()) override def continueOnReturnCodePredicate(valueRequired: Boolean) (wdlExpressionMaybe: Option[WdlValue]): Boolean = { diff --git a/backend/src/test/scala/cromwell/backend/TestConfig.scala b/backend/src/test/scala/cromwell/backend/TestConfig.scala new file mode 100644 index 000000000..42050dbe3 --- /dev/null +++ b/backend/src/test/scala/cromwell/backend/TestConfig.scala @@ -0,0 +1,44 @@ +package cromwell.backend + +import com.typesafe.config.ConfigFactory + +object TestConfig { + + lazy val sampleBackendRuntimeConfigString = + s""" + |default-runtime-attributes { + | failOnStderr: false + | continueOnReturnCode: 0 + | memory: "2 GB" + |} + |""".stripMargin + + lazy val allBackendRuntimeAttrsString = + s""" + |default-runtime-attributes { + | cpu: 1 + | failOnStderr: false + | continueOnReturnCode: 0 + | memory: "1 GB" + | bootDiskSizeGb: 10 + | disks: "local-disk 10 SSD" + | noAddress: false + | preemptible: 0 + | zones: ["us-central1-a", "us-central1-b"] + |} + """.stripMargin + + lazy val sampleBackendRuntimeConfig = ConfigFactory.parseString(sampleBackendRuntimeConfigString) + + lazy val allRuntimeAttrsConfig = ConfigFactory.parseString(allBackendRuntimeAttrsString).getConfig("default-runtime-attributes") + + lazy val optionalRuntimeConfig = sampleBackendRuntimeConfig.getConfig("default-runtime-attributes") + + lazy val globalConfig = ConfigFactory.load() + + lazy val emptyConfig = ConfigFactory.empty() + + lazy val emptyBackendConfigDescriptor = BackendConfigurationDescriptor(emptyConfig, globalConfig) + + lazy val backendRuntimeConfigDescriptor = BackendConfigurationDescriptor(sampleBackendRuntimeConfig, emptyConfig) +} diff --git a/backend/src/test/scala/cromwell/backend/io/JobPathsSpec.scala b/backend/src/test/scala/cromwell/backend/io/JobPathsSpec.scala index 0a8cb5936..2b38535ad 100644 --- a/backend/src/test/scala/cromwell/backend/io/JobPathsSpec.scala +++ b/backend/src/test/scala/cromwell/backend/io/JobPathsSpec.scala @@ -1,7 +1,7 @@ package cromwell.backend.io import com.typesafe.config.ConfigFactory -import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptorKey, BackendSpec} +import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptorKey, BackendSpec, TestConfig} import cromwell.core.path.DefaultPathBuilder import org.scalatest.{FlatSpec, Matchers} import wdl4s.TaskCall @@ -23,9 +23,8 @@ class JobPathsSpec extends FlatSpec with Matchers with BackendSpec { | } """.stripMargin - val globalConfig = ConfigFactory.load() val backendConfig = ConfigFactory.parseString(configString) - val defaultBackendConfigDescriptor = BackendConfigurationDescriptor(backendConfig, globalConfig) + val defaultBackendConfigDescriptor = BackendConfigurationDescriptor(backendConfig, TestConfig.globalConfig) "JobPaths" should "provide correct paths for a job" in { diff --git a/backend/src/test/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala b/backend/src/test/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala index b116aae72..e4e18bbfb 100644 --- a/backend/src/test/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala +++ b/backend/src/test/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala @@ -1,6 +1,6 @@ package cromwell.backend.standard -import cromwell.backend.RuntimeAttributeDefinition +import cromwell.backend.{RuntimeAttributeDefinition, TestConfig} import cromwell.backend.validation.RuntimeAttributesKeys._ import cromwell.backend.validation._ import cromwell.core.WorkflowOptions @@ -153,6 +153,8 @@ class StandardValidatedRuntimeAttributesBuilderSpec extends WordSpecLike with Ma val defaultLogger: Logger = LoggerFactory.getLogger(classOf[StandardValidatedRuntimeAttributesBuilderSpec]) val emptyWorkflowOptions: WorkflowOptions = WorkflowOptions.fromMap(Map.empty).get + val mockBackendRuntimeConfig = Option(TestConfig.optionalRuntimeConfig) + private def assertRuntimeAttributesSuccessfulCreation(runtimeAttributes: Map[String, WdlValue], expectedRuntimeAttributes: Map[String, Any], includeDockerSupport: Boolean = true, @@ -160,9 +162,9 @@ class StandardValidatedRuntimeAttributesBuilderSpec extends WordSpecLike with Ma logger: Logger = defaultLogger): Unit = { val builder = if (includeDockerSupport) { - StandardValidatedRuntimeAttributesBuilder.default.withValidation(DockerValidation.optional) + StandardValidatedRuntimeAttributesBuilder.default(mockBackendRuntimeConfig).withValidation(DockerValidation.optional) } else { - StandardValidatedRuntimeAttributesBuilder.default + StandardValidatedRuntimeAttributesBuilder.default(mockBackendRuntimeConfig) } val runtimeAttributeDefinitions = builder.definitions.toSet val addDefaultsToAttributes = RuntimeAttributeDefinition.addDefaultsToAttributes(runtimeAttributeDefinitions, workflowOptions) _ @@ -189,9 +191,9 @@ class StandardValidatedRuntimeAttributesBuilderSpec extends WordSpecLike with Ma logger: Logger = defaultLogger): Unit = { val thrown = the[RuntimeException] thrownBy { val builder = if (supportsDocker) { - StandardValidatedRuntimeAttributesBuilder.default.withValidation(DockerValidation.optional) + StandardValidatedRuntimeAttributesBuilder.default(mockBackendRuntimeConfig).withValidation(DockerValidation.optional) } else { - StandardValidatedRuntimeAttributesBuilder.default + StandardValidatedRuntimeAttributesBuilder.default(mockBackendRuntimeConfig) } val runtimeAttributeDefinitions = builder.definitions.toSet val addDefaultsToAttributes = RuntimeAttributeDefinition.addDefaultsToAttributes(runtimeAttributeDefinitions, workflowOptions) _ diff --git a/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala b/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala index 22180bb77..c6de0b819 100644 --- a/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala +++ b/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala @@ -2,12 +2,16 @@ package cromwell.backend.validation import cats.data.Validated.{Invalid, Valid} import cats.syntax.validated._ +import com.typesafe.config.{Config, ConfigFactory} +import cromwell.backend.TestConfig import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpecLike} import wdl4s.types.{WdlArrayType, WdlIntegerType, WdlStringType} import wdl4s.values.{WdlArray, WdlBoolean, WdlInteger, WdlString} class RuntimeAttributesValidationSpec extends WordSpecLike with Matchers with BeforeAndAfterAll { + val mockBackendRuntimeConfig = TestConfig.allRuntimeAttrsConfig + "RuntimeAttributesValidation" should { "return success when tries to validate a valid Docker entry" in { val dockerValue = Some(WdlString("someImage")) @@ -278,5 +282,83 @@ class RuntimeAttributesValidationSpec extends WordSpecLike with Matchers with Be case Invalid(e) => assert(e.head == "Failed to get cpu mandatory key from runtime attributes") } } + + "return default values as WdlValues when they can be coerced into expected WdlTypes" in { + val optionalConfig = Option(TestConfig.allRuntimeAttrsConfig) + + val defaultVals = Map( + "cpu" -> CpuValidation.configDefaultWdlValue(optionalConfig).get, + "failOnStderr" -> FailOnStderrValidation.configDefaultWdlValue(optionalConfig).get, + "continueOnReturnCode" -> ContinueOnReturnCodeValidation.configDefaultWdlValue(optionalConfig).get + ) + + val expectedDefaultVals = Map( + "cpu" -> WdlInteger(1), + "failOnStderr" -> WdlBoolean(false), + "continueOnReturnCode" -> WdlInteger(0) + ) + + defaultVals shouldBe expectedDefaultVals + } + + "return default values as BadDefaultAttribute when they can't be coerced to expected WdlTypes" in { + val optionalInvalidAttrsConfig = Option(ConfigFactory.parseString( + """ + |cpu = 1.4 + |failOnStderr = "notReal" + |continueOnReturnCode = 0 + """.stripMargin)) + + val defaultVals = Map( + "cpu" -> CpuValidation.configDefaultWdlValue(optionalInvalidAttrsConfig).get, + "failOnStderr" -> FailOnStderrValidation.configDefaultWdlValue(optionalInvalidAttrsConfig).get, + "continueOnReturnCode" -> ContinueOnReturnCodeValidation.configDefaultWdlValue(optionalInvalidAttrsConfig).get + ) + + val expectedDefaultVals = Map( + "cpu" -> BadDefaultAttribute(WdlString("1.4")), + "failOnStderr" -> BadDefaultAttribute(WdlString("notReal")), + "continueOnReturnCode" -> WdlInteger(0) + ) + + defaultVals shouldBe expectedDefaultVals + } + + "should parse memory successfully" in { + val backendConfigTemplate: String = + s""" + | default-runtime-attributes { + | memory: "2 GB" + | } + |""".stripMargin + + val backendConfig: Config = ConfigFactory.parseString(backendConfigTemplate).getConfig("default-runtime-attributes") + + val memoryVal = MemoryValidation.configDefaultString(Some(backendConfig)) + MemoryValidation.withDefaultMemory(memoryVal.get).runtimeAttributeDefinition.factoryDefault shouldBe Some((WdlInteger(2000000000))) + } + + "shouldn't throw up if the value for a default-runtime-attribute key cannot be coerced into an expected WdlType" in { + val backendConfigTemplate: String = + s""" + | default-runtime-attributes { + | memory: "blahblah" + | } + |""".stripMargin + + val backendConfig: Config = ConfigFactory.parseString(backendConfigTemplate).getConfig("default-runtime-attributes") + + val memoryVal = MemoryValidation.configDefaultString(Some(backendConfig)) + MemoryValidation.withDefaultMemory(memoryVal.get).runtimeAttributeDefinition.factoryDefault shouldBe Some(BadDefaultAttribute(WdlString("blahblah"))) + } + + "should be able to coerce a list of return codes into an WdlArray" in { + val optinalBackendConfig = Option(ConfigFactory.parseString( + s""" + |continueOnReturnCode = [0,1,2] + |""".stripMargin)) + + ContinueOnReturnCodeValidation.configDefaultWdlValue(optinalBackendConfig).get shouldBe WdlArray(WdlArrayType(WdlIntegerType), Array(WdlInteger(0), WdlInteger(1), WdlInteger(2))) + } } } diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf index 787303136..fd1321aeb 100644 --- a/core/src/main/resources/reference.conf +++ b/core/src/main/resources/reference.conf @@ -235,6 +235,11 @@ backend { } } } + + default-runtime-attributes { + failOnStderr: false + continueOnReturnCode: 0 + } } } @@ -244,6 +249,13 @@ backend { # root = "cromwell-executions" # dockerRoot = "/cromwell-executions" # endpoint = "http://127.0.0.1:9000/v1/jobs" + # default-runtime-attributes { + # cpu: 1 + # failOnStderr: false + # continueOnReturnCode: 0 + # memory: "2 GB" + # disk: "2 GB" + # } # } #} @@ -426,8 +438,6 @@ backend { # # Pipelines and manipulate auth JSONs. # auth = "application-default" # - # # Specifies the zone(s) to use for JES jobs unless overridden by a task's runtime attributes - # default-zones = ["us-central1-b"] # # // alternative service account to use on the launched compute instance # // NOTE: If combined with service account authorization, both that serivce account and this service account @@ -444,6 +454,19 @@ backend { # auth = "application-default" # } # } + # + # default-runtime-attributes { + # cpu: 1 + # failOnStderr: false + # continueOnReturnCode: 0 + # memory: "2 GB" + # bootDiskSizeGb: 10 + # # Allowed to be a String, or a list of Strings + # disks: "local-disk 10 SSD" + # noAddress: false + # preemptible: 0 + # zones: ["us-central1-a", "us-central1-b"] + # } # } #} diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorInitializationActor.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorInitializationActor.scala index 1212876b6..eb298c558 100644 --- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorInitializationActor.scala +++ b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorInitializationActor.scala @@ -3,7 +3,7 @@ package cromwell.backend.impl.htcondor import akka.actor.{ActorRef, Props} import cromwell.backend.impl.htcondor.HtCondorInitializationActor._ import cromwell.backend.impl.htcondor.HtCondorRuntimeAttributes._ -import cromwell.backend.validation.RuntimeAttributesDefault +import cromwell.backend.validation.{ContinueOnReturnCodeValidation, RuntimeAttributesDefault} import cromwell.backend.validation.RuntimeAttributesKeys._ import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendWorkflowDescriptor, BackendWorkflowInitializationActor} import cromwell.core.WorkflowOptions @@ -66,4 +66,10 @@ class HtCondorInitializationActor(override val workflowDescriptor: BackendWorkfl override protected def coerceDefaultRuntimeAttributes(options: WorkflowOptions): Try[Map[String, WdlValue]] = { RuntimeAttributesDefault.workflowOptionsDefault(options, HtCondorRuntimeAttributes.coercionMap) } + + override def continueOnReturnCodePredicate(valueRequired: Boolean)(wdlExpressionMaybe: Option[WdlValue]): Boolean = { + val continueOnReturnCodeDefaultValue = HtCondorRuntimeAttributes.staticDefaults.get(ContinueOnReturnCodeKey).get + ContinueOnReturnCodeValidation.instance.withDefault(continueOnReturnCodeDefaultValue).validateOptionalExpression(wdlExpressionMaybe) + } + } diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributes.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributes.scala index c96fd2f0a..4f72c01a6 100644 --- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributes.scala +++ b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributes.scala @@ -28,7 +28,7 @@ object HtCondorRuntimeAttributes { val DiskKey = "disk" val NativeSpecsKey = "nativeSpecs" - val staticDefaults = Map( + val staticDefaults: Map[String, WdlValue] = Map( FailOnStderrKey -> WdlBoolean(FailOnStderrDefaultValue), ContinueOnReturnCodeKey -> WdlInteger(ContinueOnRcDefaultValue), CpuKey -> WdlInteger(CpuDefaultValue), diff --git a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorInitializationActorSpec.scala b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorInitializationActorSpec.scala index 29d19d117..ba569802f 100644 --- a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorInitializationActorSpec.scala +++ b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorInitializationActorSpec.scala @@ -2,7 +2,7 @@ package cromwell.backend.impl.htcondor import akka.testkit.{EventFilter, ImplicitSender, TestDuration} import cromwell.backend.BackendWorkflowInitializationActor.Initialize -import cromwell.backend.{BackendConfigurationDescriptor, BackendSpec, BackendWorkflowDescriptor} +import cromwell.backend.{BackendConfigurationDescriptor, BackendSpec, BackendWorkflowDescriptor, TestConfig} import cromwell.core.TestKitSuite import org.scalatest.{Matchers, WordSpecLike} import wdl4s.TaskCall @@ -44,7 +44,7 @@ class HtCondorInitializationActorSpec extends TestKitSuite("HtCondorInitializati EventFilter.warning(message = s"Key/s [proc] is/are not supported by HtCondorBackend. Unsupported attributes will not be part of jobs executions.", occurrences = 1) intercept { val workflowDescriptor = buildWorkflowDescriptor(HelloWorld, runtime = """runtime { proc: 1 }""") val backend = getHtCondorBackend(workflowDescriptor, workflowDescriptor.workflow.taskCalls, - emptyBackendConfig) + TestConfig.emptyBackendConfigDescriptor) backend ! Initialize } } diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala index 5b85dc05e..5ce291a31 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAttributes.scala @@ -2,7 +2,6 @@ package cromwell.backend.impl.jes import java.net.{URI, URL} -import cats.data.NonEmptyList import cats.data.Validated._ import cats.syntax.cartesian._ import cats.syntax.validated._ @@ -16,7 +15,7 @@ import lenthall.validation.ErrorOr._ import lenthall.validation.Validation._ import net.ceedubs.ficus.Ficus._ import net.ceedubs.ficus.readers.{StringReader, ValueReader} -import org.slf4j.LoggerFactory +import org.slf4j.{Logger, LoggerFactory} import scala.collection.JavaConversions._ @@ -26,8 +25,7 @@ case class JesAttributes(project: String, executionBucket: String, endpointUrl: URL, maxPollingInterval: Int, - qps: Int Refined Positive, - defaultZones: NonEmptyList[String]) + qps: Int Refined Positive) object JesAttributes { lazy val Logger = LoggerFactory.getLogger("JesAttributes") @@ -47,8 +45,11 @@ object JesAttributes { "genomics.auth", "genomics.endpoint-url", "filesystems.gcs.auth", - "genomics-api-queries-per-100-seconds", - "genomics.default-zones" + "genomics-api-queries-per-100-seconds" + ) + + private val deprecatedJesKeys: Map[String, String] = Map( + "genomics.default-zones" -> "default-runtime-attributes.zones" ) private val context = "Jes" @@ -59,6 +60,13 @@ object JesAttributes { val configKeys = backendConfig.entrySet().toSet map { entry: java.util.Map.Entry[String, ConfigValue] => entry.getKey } warnNotRecognized(configKeys, jesKeys, context, Logger) + def warnDeprecated(keys: Set[String], deprecated: Map[String, String], context: String, logger: Logger) = { + val deprecatedKeys = keys.intersect(deprecated.keySet) + deprecatedKeys foreach { key => logger.warn(s"Found deprecated configuration key $key, replaced with ${deprecated.get(key)}") } + } + + warnDeprecated(configKeys, deprecatedJesKeys, context, Logger) + val project: ErrorOr[String] = validate { backendConfig.as[String]("project") } val executionBucket: ErrorOr[String] = validate { backendConfig.as[String]("root") } val endpointUrl: ErrorOr[URL] = validate { backendConfig.as[URL]("genomics.endpoint-url") } @@ -67,13 +75,13 @@ object JesAttributes { val genomicsAuthName: ErrorOr[String] = validate { backendConfig.as[String]("genomics.auth") } val gcsFilesystemAuthName: ErrorOr[String] = validate { backendConfig.as[String]("filesystems.gcs.auth") } val qpsValidation = validateQps(backendConfig) - val defaultZones = defaultZonesFromConfig(backendConfig) - (project |@| executionBucket |@| endpointUrl |@| genomicsAuthName |@| gcsFilesystemAuthName |@| defaultZones |@| qpsValidation) map { - (_, _, _, _, _, _, _) - } flatMap { case (p, b, u, genomicsName, gcsName, d, qps) => + + (project |@| executionBucket |@| endpointUrl |@| genomicsAuthName |@| gcsFilesystemAuthName |@| qpsValidation) map { + (_, _, _, _, _, _) + } flatMap { case (p, b, u, genomicsName, gcsName, qps) => (googleConfig.auth(genomicsName) |@| googleConfig.auth(gcsName)) map { case (genomicsAuth, gcsAuth) => - JesAttributes(p, computeServiceAccount, JesAuths(genomicsAuth, gcsAuth), b, u, maxPollingInterval, qps, d) + JesAttributes(p, computeServiceAccount, JesAuths(genomicsAuth, gcsAuth), b, u, maxPollingInterval, qps) } } match { case Valid(r) => r @@ -85,14 +93,6 @@ object JesAttributes { } } - def defaultZonesFromConfig(config: Config): ErrorOr[NonEmptyList[String]] = { - val zones = config.as[Option[List[String]]]("genomics.default-zones").getOrElse(List("us-central1-b")) - zones match { - case x :: xs => NonEmptyList(x, xs).validNel - case _ => "genomics.default-zones was set but no values were provided".invalidNel - } - } - def validateQps(config: Config): ErrorOr[Int Refined Positive] = { import eu.timepit.refined._ diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesConfiguration.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesConfiguration.scala index a5a3fa01c..636c0dad1 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesConfiguration.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesConfiguration.scala @@ -10,6 +10,7 @@ class JesConfiguration(val configurationDescriptor: BackendConfigurationDescript val googleConfig = GoogleConfiguration(configurationDescriptor.globalConfig) val root = configurationDescriptor.backendConfig.getString("root") + val runtimeConfig = configurationDescriptor.backendRuntimeConfig val jesAttributes = JesAttributes(googleConfig, configurationDescriptor.backendConfig) val jesAuths = jesAttributes.auths val jesComputeServiceAccount = jesAttributes.computeServiceAccount @@ -18,5 +19,4 @@ class JesConfiguration(val configurationDescriptor: BackendConfigurationDescript val dockerCredentials = DockerConfiguration.build(configurationDescriptor.backendConfig).dockerCredentials map JesDockerCredentials.apply val needAuthFileUpload = jesAuths.gcs.requiresAuthFile || dockerCredentials.isDefined val qps = jesAttributes.qps - val defaultZones = jesAttributes.defaultZones } diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobCachingActorHelper.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobCachingActorHelper.scala index 496eb6015..acdca18e2 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobCachingActorHelper.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobCachingActorHelper.scala @@ -23,7 +23,8 @@ trait JesJobCachingActorHelper extends StandardCachingActorHelper { lazy val jesCallPaths: JesJobPaths = jobPaths.asInstanceOf[JesJobPaths] - lazy val runtimeAttributes = JesRuntimeAttributes(validatedRuntimeAttributes) + lazy val runtimeAttributes = JesRuntimeAttributes(validatedRuntimeAttributes, jesConfiguration.runtimeConfig) + lazy val workingDisk: JesAttachedDisk = runtimeAttributes.disks.find(_.name == JesWorkingDisk.Name).get lazy val callRootPath: Path = jesCallPaths.callExecutionRoot diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesRuntimeAttributes.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesRuntimeAttributes.scala index f64716d9b..311d9f4b7 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesRuntimeAttributes.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesRuntimeAttributes.scala @@ -1,9 +1,9 @@ package cromwell.backend.impl.jes -import cats.data.NonEmptyList +import cats.syntax.validated._ import cats.data.Validated._ import cats.syntax.cartesian._ -import cats.syntax.validated._ +import com.typesafe.config.Config import cromwell.backend.MemorySize import cromwell.backend.impl.jes.io.{JesAttachedDisk, JesWorkingDisk} import cromwell.backend.standard.StandardValidatedRuntimeAttributesBuilder @@ -12,6 +12,7 @@ import lenthall.validation.ErrorOr._ import wdl4s.types._ import wdl4s.values._ + case class JesRuntimeAttributes(cpu: Int, zones: Vector[String], preemptible: Int, @@ -24,72 +25,80 @@ case class JesRuntimeAttributes(cpu: Int, noAddress: Boolean) object JesRuntimeAttributes { - private val MemoryDefaultValue = "2 GB" val ZonesKey = "zones" + private val ZonesDefaultValue = WdlString("us-central1-b") val PreemptibleKey = "preemptible" - private val PreemptibleDefaultValue = 0 + private val preemptibleValidationInstance = new IntRuntimeAttributesValidation(PreemptibleKey) + private val PreemptibleDefaultValue = WdlInteger(0) val BootDiskSizeKey = "bootDiskSizeGb" - private val BootDiskSizeDefaultValue = 10 + private val bootDiskValidationInstance = new IntRuntimeAttributesValidation(BootDiskSizeKey) + private val BootDiskDefaultValue = WdlInteger(10) val NoAddressKey = "noAddress" - private val NoAddressDefaultValue = false + private val noAddressValidationInstance = new BooleanRuntimeAttributesValidation(NoAddressKey) + private val NoAddressDefaultValue = WdlBoolean(false) val DisksKey = "disks" - private val DisksDefaultValue = s"${JesWorkingDisk.Name} 10 SSD" + private val DisksDefaultValue = WdlString(s"${JesWorkingDisk.Name} 10 SSD") + + private val MemoryDefaultValue = "2 GB" + + private def cpuValidation(runtimeConfig: Option[Config]): RuntimeAttributesValidation[Int] = CpuValidation.instance + .withDefault(CpuValidation.configDefaultWdlValue(runtimeConfig) getOrElse CpuValidation.default) - private val cpuValidation: RuntimeAttributesValidation[Int] = CpuValidation.default + private def failOnStderrValidation(runtimeConfig: Option[Config]) = FailOnStderrValidation.default(runtimeConfig) - private val disksValidation: RuntimeAttributesValidation[Seq[JesAttachedDisk]] = - DisksValidation.withDefault(WdlString(DisksDefaultValue)) + private def continueOnReturnCodeValidation(runtimeConfig: Option[Config]) = ContinueOnReturnCodeValidation.default(runtimeConfig) - private def zonesValidation(defaultZones: NonEmptyList[String]): RuntimeAttributesValidation[Vector[String]] = - ZonesValidation.withDefault(WdlString(defaultZones.toList.mkString(" "))) + private def disksValidation(runtimeConfig: Option[Config]): RuntimeAttributesValidation[Seq[JesAttachedDisk]] = DisksValidation + .withDefault(DisksValidation.configDefaultWdlValue(runtimeConfig) getOrElse DisksDefaultValue) - private val preemptibleValidation: RuntimeAttributesValidation[Int] = - new IntRuntimeAttributesValidation(JesRuntimeAttributes.PreemptibleKey) - .withDefault(WdlInteger(PreemptibleDefaultValue)) + private def zonesValidation(runtimeConfig: Option[Config]): RuntimeAttributesValidation[Vector[String]] = ZonesValidation + .withDefault(ZonesValidation.configDefaultWdlValue(runtimeConfig) getOrElse ZonesDefaultValue) - private val memoryValidation: RuntimeAttributesValidation[MemorySize] = - MemoryValidation.withDefaultMemory(MemorySize.parse(MemoryDefaultValue).get) + private def preemptibleValidation(runtimeConfig: Option[Config]): RuntimeAttributesValidation[Int] = preemptibleValidationInstance + .withDefault(preemptibleValidationInstance.configDefaultWdlValue(runtimeConfig) getOrElse PreemptibleDefaultValue) - private val bootDiskSizeValidation: RuntimeAttributesValidation[Int] = - new IntRuntimeAttributesValidation(JesRuntimeAttributes.BootDiskSizeKey) - .withDefault(WdlInteger(BootDiskSizeDefaultValue)) + private def memoryValidation(runtimeConfig: Option[Config]): RuntimeAttributesValidation[MemorySize] = { + MemoryValidation.withDefaultMemory(MemoryValidation.configDefaultString(runtimeConfig) getOrElse MemoryDefaultValue) + } - private val noAddressValidation: RuntimeAttributesValidation[Boolean] = - new BooleanRuntimeAttributesValidation(JesRuntimeAttributes.NoAddressKey) - .withDefault(WdlBoolean(NoAddressDefaultValue)) + private def bootDiskSizeValidation(runtimeConfig: Option[Config]): RuntimeAttributesValidation[Int] = bootDiskValidationInstance + .withDefault(bootDiskValidationInstance.configDefaultWdlValue(runtimeConfig) getOrElse BootDiskDefaultValue) + + private def noAddressValidation(runtimeConfig: Option[Config]): RuntimeAttributesValidation[Boolean] = noAddressValidationInstance + .withDefault(noAddressValidationInstance.configDefaultWdlValue(runtimeConfig) getOrElse NoAddressDefaultValue) private val dockerValidation: RuntimeAttributesValidation[String] = DockerValidation.instance - def runtimeAttributesBuilder(jesConfiguration: JesConfiguration): StandardValidatedRuntimeAttributesBuilder = - StandardValidatedRuntimeAttributesBuilder.default.withValidation( - cpuValidation, - disksValidation, - zonesValidation(jesConfiguration.defaultZones), - preemptibleValidation, - memoryValidation, - bootDiskSizeValidation, - noAddressValidation, + def runtimeAttributesBuilder(jesConfiguration: JesConfiguration): StandardValidatedRuntimeAttributesBuilder = { + val runtimeConfig = jesConfiguration.runtimeConfig + StandardValidatedRuntimeAttributesBuilder.default(runtimeConfig).withValidation( + cpuValidation(runtimeConfig), + disksValidation(runtimeConfig), + zonesValidation(runtimeConfig), + preemptibleValidation(runtimeConfig), + memoryValidation(runtimeConfig), + bootDiskSizeValidation(runtimeConfig), + noAddressValidation(runtimeConfig), dockerValidation ) + } - def apply(validatedRuntimeAttributes: ValidatedRuntimeAttributes): JesRuntimeAttributes = { - val cpu: Int = RuntimeAttributesValidation.extract(cpuValidation, validatedRuntimeAttributes) + def apply(validatedRuntimeAttributes: ValidatedRuntimeAttributes, runtimeAttrsConfig: Option[Config]): JesRuntimeAttributes = { + val cpu: Int = RuntimeAttributesValidation.extract(cpuValidation(runtimeAttrsConfig), validatedRuntimeAttributes) val zones: Vector[String] = RuntimeAttributesValidation.extract(ZonesValidation, validatedRuntimeAttributes) - val preemptible: Int = RuntimeAttributesValidation.extract(preemptibleValidation, validatedRuntimeAttributes) - val bootDiskSize: Int = RuntimeAttributesValidation.extract(bootDiskSizeValidation, validatedRuntimeAttributes) - val memory: MemorySize = RuntimeAttributesValidation.extract(memoryValidation, validatedRuntimeAttributes) - val disks: Seq[JesAttachedDisk] = RuntimeAttributesValidation.extract(disksValidation, validatedRuntimeAttributes) + val preemptible: Int = RuntimeAttributesValidation.extract(preemptibleValidation(runtimeAttrsConfig), validatedRuntimeAttributes) + val bootDiskSize: Int = RuntimeAttributesValidation.extract(bootDiskSizeValidation(runtimeAttrsConfig), validatedRuntimeAttributes) + val memory: MemorySize = RuntimeAttributesValidation.extract(memoryValidation(runtimeAttrsConfig), validatedRuntimeAttributes) + val disks: Seq[JesAttachedDisk] = RuntimeAttributesValidation.extract(disksValidation(runtimeAttrsConfig), validatedRuntimeAttributes) val docker: String = RuntimeAttributesValidation.extract(dockerValidation, validatedRuntimeAttributes) - val failOnStderr: Boolean = - RuntimeAttributesValidation.extract(FailOnStderrValidation.default, validatedRuntimeAttributes) - val continueOnReturnCode: ContinueOnReturnCode = - RuntimeAttributesValidation.extract(ContinueOnReturnCodeValidation.default, validatedRuntimeAttributes) - val noAddress: Boolean = RuntimeAttributesValidation.extract(noAddressValidation, validatedRuntimeAttributes) + val failOnStderr: Boolean = RuntimeAttributesValidation.extract(failOnStderrValidation(runtimeAttrsConfig), validatedRuntimeAttributes) + val continueOnReturnCode: ContinueOnReturnCode = RuntimeAttributesValidation.extract(continueOnReturnCodeValidation(runtimeAttrsConfig), validatedRuntimeAttributes) + val noAddress: Boolean = RuntimeAttributesValidation.extract(noAddressValidation(runtimeAttrsConfig), validatedRuntimeAttributes) new JesRuntimeAttributes( cpu, diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAttributesSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAttributesSpec.scala index 5d9a5724e..1145cddbc 100644 --- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAttributesSpec.scala +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesAttributesSpec.scala @@ -14,8 +14,11 @@ class JesAttributesSpec extends FlatSpec with Matchers { behavior of "JesAttributes" + val googleConfig = GoogleConfiguration(JesGlobalConfig) + val runtimeConfig = ConfigFactory.load() + it should "parse correct JES config" taggedAs IntegrationTest in { - val googleConfig = GoogleConfiguration(JesGlobalConfig) + val backendConfig = ConfigFactory.parseString(configString()) val jesAttributes = JesAttributes(googleConfig, backendConfig) @@ -24,11 +27,9 @@ class JesAttributesSpec extends FlatSpec with Matchers { jesAttributes.executionBucket should be("gs://myBucket") jesAttributes.maxPollingInterval should be(600) jesAttributes.computeServiceAccount should be("default") - jesAttributes.defaultZones.toList should be (List("us-central1-a")) } it should "parse correct preemptible config" taggedAs IntegrationTest in { - val googleConfig = GoogleConfiguration(JesGlobalConfig) val backendConfig = ConfigFactory.parseString(configString(preemptible = "preemptible = 3")) val jesAttributes = JesAttributes(googleConfig, backendConfig) @@ -36,11 +37,9 @@ class JesAttributesSpec extends FlatSpec with Matchers { jesAttributes.project should be("myProject") jesAttributes.executionBucket should be("gs://myBucket") jesAttributes.maxPollingInterval should be(600) - jesAttributes.defaultZones.toList should be (List("us-central1-a")) } it should "parse compute service account" taggedAs IntegrationTest in { - val googleConfig = GoogleConfiguration(JesGlobalConfig) val backendConfig = ConfigFactory.parseString(configString(genomics = """compute-service-account = "testing" """)) val jesAttributes = JesAttributes(googleConfig, backendConfig) @@ -54,13 +53,10 @@ class JesAttributesSpec extends FlatSpec with Matchers { |{ | genomics { | endpoint-url = "myEndpoint" - | default-zones = [] | } |} """.stripMargin) - val googleConfig = GoogleConfiguration(JesGlobalConfig) - val exception = intercept[IllegalArgumentException with MessageAggregation] { JesAttributes(googleConfig, nakedConfig) } @@ -70,7 +66,6 @@ class JesAttributesSpec extends FlatSpec with Matchers { errorsList should contain("No configuration setting found for key 'genomics.auth'") errorsList should contain("No configuration setting found for key 'filesystems'") errorsList should contain("URI is not absolute") - errorsList should contain("genomics.default-zones was set but no values were provided") } def configString(preemptible: String = "", genomics: String = ""): String = @@ -86,7 +81,6 @@ class JesAttributesSpec extends FlatSpec with Matchers { | auth = "application-default" | $genomics | endpoint-url = "http://myEndpoint" - | default-zones = ["us-central1-a"] | } | | filesystems = { diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesConfigurationSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesConfigurationSpec.scala index 23b029086..2ead86d6c 100644 --- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesConfigurationSpec.scala +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesConfigurationSpec.scala @@ -42,6 +42,7 @@ class JesConfigurationSpec extends FlatSpec with Matchers with TableDrivenProper | } | ] |} + | """.stripMargin) val backendConfig = ConfigFactory.parseString( @@ -62,7 +63,18 @@ class JesConfigurationSpec extends FlatSpec with Matchers with TableDrivenProper | auth = "application-default" | // Endpoint for APIs, no reason to change this unless directed by Google. | endpoint-url = "https://genomics.googleapis.com/" - | default-zones = ["us-central1-a", "us-central1-b"] + | } + | + | default-runtime-attributes { + | failOnStderr: false + | continueOnReturnCode: 0 + | cpu: 1 + | memory: "2 GB" + | bootDiskSizeGb: 10 + | disks: "local-disk 10 SSD" + | noAddress: false + | preemptible: 3 + | zones:["us-central1-a", "us-central1-b"] | } | | dockerhub { @@ -76,6 +88,7 @@ class JesConfigurationSpec extends FlatSpec with Matchers with TableDrivenProper | auth = "application-default" | } | } + | """.stripMargin) it should "fail to instantiate if any required configuration is missing" in { @@ -103,10 +116,6 @@ class JesConfigurationSpec extends FlatSpec with Matchers with TableDrivenProper new JesConfiguration(BackendConfigurationDescriptor(backendConfig, globalConfig)).root shouldBe "gs://my-cromwell-workflows-bucket" } - it should "have the correct default zones" in { - new JesConfiguration(BackendConfigurationDescriptor(backendConfig, globalConfig)).defaultZones.toList shouldBe List("us-central1-a", "us-central1-b") - } - it should "have correct docker" in { val dockerConf = new JesConfiguration(BackendConfigurationDescriptor(backendConfig, globalConfig)).dockerCredentials dockerConf shouldBe defined diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala index f702c3860..6432ce3c1 100644 --- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesInitializationActorSpec.scala @@ -67,7 +67,7 @@ class JesInitializationActorSpec extends TestKitSuite("JesInitializationActorSpe | } | ] |} - | """.stripMargin) + |""".stripMargin) val backendConfigTemplate: String = """ @@ -89,6 +89,19 @@ class JesInitializationActorSpec extends TestKitSuite("JesInitializationActorSpe | endpoint-url = "https://genomics.googleapis.com/" | } | + | default-runtime-attributes { + | cpu: 1 + | failOnStderr: false + | # Allowed to be a boolean, or a list of Ints, or an Int + | continueOnReturnCode: 0 + | memory: "2 GB" + | bootDiskSizeGb: 10 + | # Allowed to be a String, or a list of Strings + | disks: "local-disk 10 SSD" + | noAddress: false + | preemptible: 0 + | zones: ["us-central1-a", "us-central1-b"] + | } | filesystems { | gcs { | // A reference to a potentially different auth for manipulating files via engine functions. @@ -119,6 +132,20 @@ class JesInitializationActorSpec extends TestKitSuite("JesInitializationActorSpe | endpoint-url = "https://genomics.googleapis.com/" | } | + | default-runtime-attributes { + | cpu: 1 + | failOnStderr: false + | # Allowed to be a boolean, or a list of Ints, or an Int + | continueOnReturnCode: 0 + | memory: "2 GB" + | bootDiskSizeGb: 10 + | # Allowed to be a String, or a list of Strings + | disks: "local-disk 10 SSD" + | noAddress: false + | preemptible: 0 + | zones: ["us-central1-a", "us-central1-b"] + | } + | | filesystems { | gcs { | // A reference to a potentially different auth for manipulating files via engine functions. diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesRuntimeAttributesSpec.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesRuntimeAttributesSpec.scala index 23ab89ed1..18ae4f432 100644 --- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesRuntimeAttributesSpec.scala +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesRuntimeAttributesSpec.scala @@ -2,7 +2,7 @@ package cromwell.backend.impl.jes import cats.data.NonEmptyList import cromwell.backend.impl.jes.io.{DiskType, JesAttachedDisk, JesWorkingDisk} -import cromwell.backend.validation.ContinueOnReturnCodeSet +import cromwell.backend.validation.{ContinueOnReturnCodeFlag, ContinueOnReturnCodeSet} import cromwell.backend.{MemorySize, RuntimeAttributeDefinition} import cromwell.core.WorkflowOptions import org.scalatest.{Matchers, WordSpecLike} @@ -22,7 +22,7 @@ class JesRuntimeAttributesSpec extends WordSpecLike with Matchers with Mockito { } val expectedDefaults = new JesRuntimeAttributes(1, Vector("us-central1-b", "us-central1-a"), 0, 10, - MemorySize(2, MemoryUnit.GB), Seq(JesWorkingDisk(DiskType.SSD, 10)), "ubuntu:latest", false, + MemorySize(2, MemoryUnit.GB), Vector(JesWorkingDisk(DiskType.SSD, 10)), "ubuntu:latest", false, ContinueOnReturnCodeSet(Set(0)), false) "JesRuntimeAttributes" should { @@ -32,6 +32,12 @@ class JesRuntimeAttributesSpec extends WordSpecLike with Matchers with Mockito { assertJesRuntimeAttributesFailedCreation(runtimeAttributes, "Can't find an attribute value for key docker") } + "use hardcoded defaults if not declared in task, workflow options, or config (except for docker)" in { + val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest")) + val expectedRuntimeAttributes = expectedDefaults + assertJesRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes, jesConfiguration = noDefaultsJesConfiguration) + } + "validate a valid Docker entry" in { val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest")) val expectedRuntimeAttributes = expectedDefaults @@ -54,12 +60,18 @@ class JesRuntimeAttributesSpec extends WordSpecLike with Matchers with Mockito { assertJesRuntimeAttributesFailedCreation(runtimeAttributes, "Expecting failOnStderr runtime attribute to be a Boolean or a String with values of 'true' or 'false'") } - "validate a valid continueOnReturnCode entry" in { + "validate a valid continueOnReturnCode integer entry" in { val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest"), "continueOnReturnCode" -> WdlInteger(1)) val expectedRuntimeAttributes = expectedDefaults.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1))) assertJesRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) } + "validate a valid continueOnReturnCode boolean entry" in { + val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest"), "continueOnReturnCode" -> WdlBoolean(false)) + val expectedRuntimeAttributes = expectedDefaults.copy(continueOnReturnCode = ContinueOnReturnCodeFlag(false)) + assertJesRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) + } + "validate a valid continueOnReturnCode array entry" in { val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest"), "continueOnReturnCode" -> WdlArray(WdlArrayType(WdlIntegerType), Array(WdlInteger(1), WdlInteger(2)))) val expectedRuntimeAttributes = expectedDefaults.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1, 2))) @@ -184,17 +196,54 @@ class JesRuntimeAttributesSpec extends WordSpecLike with Matchers with Mockito { "Expecting noAddress runtime attribute to be a Boolean") } - "use reasonable default values" in { + "override config default attributes with default attributes declared in workflow options" in { val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest")) - val expectedRuntimeAttributes = expectedDefaults - assertJesRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) + + val workflowOptionsJson = + """{ + | "default_runtime_attributes": { "cpu": 2 } + |} + """.stripMargin.parseJson.asInstanceOf[JsObject] + + val workflowOptions = WorkflowOptions.fromJsonObject(workflowOptionsJson).get + val expectedRuntimeAttributes = expectedDefaults.copy(cpu = 2) + assertJesRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes, workflowOptions) + } + + "override config default runtime attributes with task runtime attributes" in { + val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest"), "cpu" -> WdlInteger(4)) + + val workflowOptionsJson = + """{ + | "default_runtime_attributes": { "cpu": 2 } + |} + """.stripMargin.parseJson.asInstanceOf[JsObject] + + val workflowOptions = WorkflowOptions.fromJsonObject(workflowOptionsJson).get + val expectedRuntimeAttributes = expectedDefaults.copy(cpu = 4) + assertJesRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes, workflowOptions) + } + + "override invalid config default attributes with task runtime attributes" in { + val runtimeAttributes = Map("docker" -> WdlString("ubuntu:latest"), "cpu" -> WdlInteger(4)) + + val workflowOptionsJson = + """{ + | "default_runtime_attributes": { "cpu": 2.2 } + |} + """.stripMargin.parseJson.asInstanceOf[JsObject] + + val workflowOptions = WorkflowOptions.fromJsonObject(workflowOptionsJson).get + val expectedRuntimeAttributes = expectedDefaults.copy(cpu = 4) + assertJesRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes, workflowOptions) } } private def assertJesRuntimeAttributesSuccessfulCreation(runtimeAttributes: Map[String, WdlValue], expectedRuntimeAttributes: JesRuntimeAttributes, workflowOptions: WorkflowOptions = emptyWorkflowOptions, - defaultZones: NonEmptyList[String] = defaultZones): Unit = { + defaultZones: NonEmptyList[String] = defaultZones, + jesConfiguration: JesConfiguration = jesConfiguration): Unit = { try { val actualRuntimeAttributes = toJesRuntimeAttributes(runtimeAttributes, workflowOptions, jesConfiguration) assert(actualRuntimeAttributes == expectedRuntimeAttributes) @@ -204,7 +253,9 @@ class JesRuntimeAttributesSpec extends WordSpecLike with Matchers with Mockito { () } - private def assertJesRuntimeAttributesFailedCreation(runtimeAttributes: Map[String, WdlValue], exMsg: String, workflowOptions: WorkflowOptions = emptyWorkflowOptions): Unit = { + private def assertJesRuntimeAttributesFailedCreation(runtimeAttributes: Map[String, WdlValue], + exMsg: String, + workflowOptions: WorkflowOptions = emptyWorkflowOptions): Unit = { try { toJesRuntimeAttributes(runtimeAttributes, workflowOptions, jesConfiguration) fail(s"A RuntimeException was expected with message: $exMsg") @@ -221,16 +272,13 @@ class JesRuntimeAttributesSpec extends WordSpecLike with Matchers with Mockito { val defaultedAttributes = RuntimeAttributeDefinition.addDefaultsToAttributes( staticRuntimeAttributeDefinitions, workflowOptions)(runtimeAttributes) val validatedRuntimeAttributes = runtimeAttributesBuilder.build(defaultedAttributes, NOPLogger.NOP_LOGGER) - JesRuntimeAttributes(validatedRuntimeAttributes) + JesRuntimeAttributes(validatedRuntimeAttributes, jesConfiguration.runtimeConfig) } private val emptyWorkflowOptions = WorkflowOptions.fromMap(Map.empty).get private val defaultZones = NonEmptyList.of("us-central1-b", "us-central1-a") - private val jesConfiguration = { - val config = mock[JesConfiguration] - config.defaultZones returns defaultZones - config - } + private val jesConfiguration = new JesConfiguration(JesTestConfig.JesBackendConfigurationDescriptor) + private val noDefaultsJesConfiguration = new JesConfiguration(JesTestConfig.NoDefaultsConfigurationDescriptor) private val staticRuntimeAttributeDefinitions: Set[RuntimeAttributeDefinition] = JesRuntimeAttributes.runtimeAttributesBuilder(jesConfiguration).definitions.toSet } diff --git a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesTestConfig.scala b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesTestConfig.scala index 25f061387..3423a8b71 100644 --- a/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesTestConfig.scala +++ b/supportedBackends/jes/src/test/scala/cromwell/backend/impl/jes/JesTestConfig.scala @@ -4,6 +4,7 @@ import com.typesafe.config.ConfigFactory import cromwell.backend.BackendConfigurationDescriptor object JesTestConfig { + private val JesBackendConfigString = """ |project = "my-cromwell-workflows" @@ -19,6 +20,37 @@ object JesTestConfig { | auth = "application-default" | } |} + | + |default-runtime-attributes { + | cpu: 1 + | failOnStderr: false + | continueOnReturnCode: 0 + | docker: "ubuntu:latest" + | memory: "2 GB" + | bootDiskSizeGb: 10 + | disks: "local-disk 10 SSD" + | noAddress: false + | preemptible: 0 + | zones:["us-central1-b", "us-central1-a"] + |} + | + |""".stripMargin + + private val NoDefaultsConfigString = + """ + |project = "my-cromwell-workflows" + |root = "gs://my-cromwell-workflows-bucket" + | + |genomics { + | auth = "application-default" + | endpoint-url = "https://genomics.googleapis.com/" + |} + | + |filesystems { + | gcs { + | auth = "application-default" + | } + |} |""".stripMargin private val JesGlobalConfigString = @@ -44,9 +76,12 @@ object JesTestConfig { | } | } |} + | |""".stripMargin val JesBackendConfig = ConfigFactory.parseString(JesBackendConfigString) val JesGlobalConfig = ConfigFactory.parseString(JesGlobalConfigString) + val JesBackendNoDefaultConfig = ConfigFactory.parseString(NoDefaultsConfigString) val JesBackendConfigurationDescriptor = BackendConfigurationDescriptor(JesBackendConfig, JesGlobalConfig) + val NoDefaultsConfigurationDescriptor = BackendConfigurationDescriptor(JesBackendNoDefaultConfig, JesGlobalConfig) } diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/DeclarationValidation.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/DeclarationValidation.scala index 587042d5e..c13279d6d 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/DeclarationValidation.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/DeclarationValidation.scala @@ -25,7 +25,7 @@ object DeclarationValidation { // Docker and CPU are special keys understood by cromwell. case name if name == DockerValidation.instance.key => new DeclarationValidation(declaration, DockerValidation.instance) - case name if name == CpuValidation.instance.key => new DeclarationValidation(declaration, CpuValidation.default) + case name if name == CpuValidation.instance.key => new DeclarationValidation(declaration, CpuValidation.instance) // See MemoryDeclarationValidation for more info case name if MemoryDeclarationValidation.isMemoryDeclaration(name) => new MemoryDeclarationValidation(declaration) diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemInitializationActorSpec.scala b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemInitializationActorSpec.scala index ffc623361..2c037b963 100644 --- a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemInitializationActorSpec.scala +++ b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemInitializationActorSpec.scala @@ -2,10 +2,11 @@ package cromwell.backend.sfs import akka.actor.Props import akka.testkit.{EventFilter, ImplicitSender, TestDuration} +import com.typesafe.config.ConfigFactory import cromwell.backend.BackendSpec._ import cromwell.backend.BackendWorkflowInitializationActor.Initialize import cromwell.backend.standard.DefaultInitializationActorParams -import cromwell.backend.{BackendConfigurationDescriptor, BackendWorkflowDescriptor} +import cromwell.backend.{BackendConfigurationDescriptor, BackendWorkflowDescriptor, TestConfig} import cromwell.core.TestKitSuite import cromwell.core.logging.LoggingTest._ import org.scalatest.{Matchers, WordSpecLike} @@ -47,7 +48,7 @@ class SharedFileSystemInitializationActorSpec extends TestKitSuite("SharedFileSy "log a warning message when there are unsupported runtime attributes" in { within(Timeout) { val workflowDescriptor = buildWorkflowDescriptor(HelloWorld, runtime = """runtime { unsupported: 1 }""") - val conf = emptyBackendConfig + val conf = BackendConfigurationDescriptor(TestConfig.sampleBackendRuntimeConfig, ConfigFactory.empty()) val backend = getActorRef(workflowDescriptor, workflowDescriptor.workflow.taskCalls, conf) val pattern = "Key/s [unsupported] is/are not supported by backend. " + "Unsupported attributes will not be part of job executions." diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala index 85bbc68e4..bb2e8211b 100644 --- a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala +++ b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala @@ -30,7 +30,7 @@ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSyst behavior of "SharedFileSystemJobExecutionActor" lazy val runtimeAttributeDefinitions: Set[RuntimeAttributeDefinition] = - StandardValidatedRuntimeAttributesBuilder.default.definitions.toSet + StandardValidatedRuntimeAttributesBuilder.default(Some(TestConfig.optionalRuntimeConfig)).definitions.toSet def executeSpec(docker: Boolean): Any = { val expectedOutputs: CallOutputs = Map( @@ -39,7 +39,7 @@ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSyst val expectedResponse = JobSucceededResponse(mock[BackendJobDescriptorKey], Some(0), expectedOutputs, None, Seq.empty) val runtime = if (docker) """runtime { docker: "ubuntu:latest" }""" else "" val workflowDescriptor = buildWorkflowDescriptor(HelloWorld, runtime = runtime) - val workflow = TestWorkflow(workflowDescriptor, emptyBackendConfig, expectedResponse) + val workflow = TestWorkflow(workflowDescriptor, TestConfig.backendRuntimeConfigDescriptor, expectedResponse) val backend = createBackend(jobDescriptorFromSingleCallWorkflow(workflow.workflowDescriptor, Map.empty, WorkflowOptions.empty, runtimeAttributeDefinitions), workflow.config) testWorkflow(workflow, backend) } @@ -55,27 +55,29 @@ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSyst it should "send back an execution failure if the task fails" in { val expectedResponse = JobFailedNonRetryableResponse(mock[BackendJobDescriptorKey], WrongReturnCode("wf_goodbye.goodbye:NA:1", 1, None), Option(1)) - val workflow = TestWorkflow(buildWorkflowDescriptor(GoodbyeWorld), emptyBackendConfig, expectedResponse) + val workflow = TestWorkflow(buildWorkflowDescriptor(GoodbyeWorld), TestConfig.backendRuntimeConfigDescriptor, expectedResponse) val backend = createBackend(jobDescriptorFromSingleCallWorkflow(workflow.workflowDescriptor, Map.empty, WorkflowOptions.empty, runtimeAttributeDefinitions), workflow.config) testWorkflow(workflow, backend) } def localizationSpec(docker: Boolean): Assertion = { - def templateConf(localizers: String) = BackendConfigurationDescriptor( - ConfigFactory.parseString( - s"""|{ - | root = "local-cromwell-executions" - | filesystems { - | local { - | localization = [ - | $localizers - | ] - | } - | } - |} - |""".stripMargin), - ConfigFactory.parseString("{}") - ) + def templateConf(localizers: String) = BackendConfigurationDescriptor(ConfigFactory.parseString( + s"""|{ + | root = "local-cromwell-executions" + | filesystems { + | local { + | localization = [ + | $localizers + | ] + | } + | } + | default-runtime-attributes { + | cpu: 1 + | failOnStderr: false + | continueOnReturnCode: 0 + | } + |} + |""".stripMargin), ConfigFactory.parseString("{}")) val hardConf = templateConf("hard-link") val symConf = templateConf("soft-link") @@ -142,7 +144,7 @@ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSyst it should "abort a job and kill a process" in { val workflowDescriptor = buildWorkflowDescriptor(Sleep10) val jobDescriptor: BackendJobDescriptor = jobDescriptorFromSingleCallWorkflow(workflowDescriptor, Map.empty, WorkflowOptions.empty, runtimeAttributeDefinitions) - val backendRef = createBackendRef(jobDescriptor, emptyBackendConfig) + val backendRef = createBackendRef(jobDescriptor, TestConfig.backendRuntimeConfigDescriptor) val backend = backendRef.underlyingActor val execute = backend.execute @@ -156,7 +158,7 @@ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSyst def recoverSpec(completed: Boolean, writeReturnCode: Boolean = true): Assertion = { val workflowDescriptor = buildWorkflowDescriptor(HelloWorld) val jobDescriptor: BackendJobDescriptor = jobDescriptorFromSingleCallWorkflow(workflowDescriptor, Map.empty, WorkflowOptions.empty, runtimeAttributeDefinitions) - val backendRef = createBackendRef(jobDescriptor, emptyBackendConfig) + val backendRef = createBackendRef(jobDescriptor, TestConfig.backendRuntimeConfigDescriptor) val backend = backendRef.underlyingActor val jobPaths = new JobPathsWithDocker(jobDescriptor.key, workflowDescriptor, ConfigFactory.empty) @@ -235,7 +237,7 @@ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSyst val jobDescriptor: BackendJobDescriptor = BackendJobDescriptor(workflowDescriptor, BackendJobDescriptorKey(call, Option(shard), 1), runtimeAttributes, fqnMapToDeclarationMap(symbolMaps), CallCachingEligible, Map.empty) - val backend = createBackend(jobDescriptor, emptyBackendConfig) + val backend = createBackend(jobDescriptor, TestConfig.backendRuntimeConfigDescriptor) val response = JobSucceededResponse(mock[BackendJobDescriptorKey], Some(0), Map("out" -> JobOutput(WdlInteger(shard))), None, Seq.empty) executeJobAndAssertOutputs(backend, response) @@ -249,8 +251,8 @@ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSyst } val workflowDescriptor = buildWorkflowDescriptor(OutputProcess, inputs) val jobDescriptor: BackendJobDescriptor = jobDescriptorFromSingleCallWorkflow(workflowDescriptor, inputs, WorkflowOptions.empty, runtimeAttributeDefinitions) - val backend = createBackend(jobDescriptor, emptyBackendConfig) - val jobPaths = new JobPathsWithDocker(jobDescriptor.key, workflowDescriptor, emptyBackendConfig.backendConfig) + val backend = createBackend(jobDescriptor, TestConfig.backendRuntimeConfigDescriptor) + val jobPaths = new JobPathsWithDocker(jobDescriptor.key, workflowDescriptor, TestConfig.backendRuntimeConfigDescriptor.backendConfig) val expectedA = WdlFile(jobPaths.callExecutionRoot.resolve("a").toAbsolutePath.pathAsString) val expectedB = WdlFile(jobPaths.callExecutionRoot.resolve("dir").toAbsolutePath.resolve("b").pathAsString) val expectedOutputs = Map( @@ -266,7 +268,7 @@ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSyst it should "fail post processing if an output file is not found" in { val expectedResponse = JobFailedNonRetryableResponse(mock[BackendJobDescriptorKey], AggregatedException("Could not process output, file not found:", Seq.empty), Option(0)) - val workflow = TestWorkflow(buildWorkflowDescriptor(MissingOutputProcess), emptyBackendConfig, expectedResponse) + val workflow = TestWorkflow(buildWorkflowDescriptor(MissingOutputProcess), TestConfig.backendRuntimeConfigDescriptor, expectedResponse) val backend = createBackend(jobDescriptorFromSingleCallWorkflow(workflow.workflowDescriptor, Map.empty, WorkflowOptions.empty, runtimeAttributeDefinitions), workflow.config) testWorkflow(workflow, backend) } diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/TestLocalAsyncJobExecutionActor.scala b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/TestLocalAsyncJobExecutionActor.scala index ecd91a227..c0a5ef6b7 100644 --- a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/TestLocalAsyncJobExecutionActor.scala +++ b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/TestLocalAsyncJobExecutionActor.scala @@ -37,7 +37,7 @@ object TestLocalAsyncJobExecutionActor { val ioActor = system.actorOf(SimpleIoActor.props) val workflowPaths = new WorkflowPathsWithDocker(jobDescriptor.workflowDescriptor, configurationDescriptor.backendConfig) val initializationData = new StandardInitializationData(workflowPaths, - StandardValidatedRuntimeAttributesBuilder.default.withValidation(DockerValidation.optional), + StandardValidatedRuntimeAttributesBuilder.default(configurationDescriptor.backendRuntimeConfig).withValidation(DockerValidation.optional), classOf[SharedFileSystemExpressionFunctions]) val asyncClass = classOf[TestLocalAsyncJobExecutionActor] diff --git a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkInitializationActorSpec.scala b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkInitializationActorSpec.scala index de58b7c06..be1c9ba9c 100644 --- a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkInitializationActorSpec.scala +++ b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkInitializationActorSpec.scala @@ -3,7 +3,7 @@ package cromwell.backend.impl.spark import akka.testkit.{EventFilter, ImplicitSender, TestDuration} import cromwell.backend.BackendSpec._ import cromwell.backend.BackendWorkflowInitializationActor.Initialize -import cromwell.backend.{BackendConfigurationDescriptor, BackendWorkflowDescriptor} +import cromwell.backend.{BackendConfigurationDescriptor, BackendWorkflowDescriptor, TestConfig} import cromwell.core.TestKitSuite import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpecLike} import wdl4s._ @@ -42,7 +42,7 @@ class SparkInitializationActorSpec extends TestKitSuite("SparkInitializationAc within(Timeout) { EventFilter.warning(message = s"Key/s [memory] is/are not supported by SparkBackend. Unsupported attributes will not be part of jobs executions.", occurrences = 1) intercept { val workflowDescriptor = buildWorkflowDescriptor(HelloWorld, runtime = """runtime { memory: 1 %s: "%s"}""".format("appMainClass", "test")) - val backend = getSparkBackend(workflowDescriptor, workflowDescriptor.workflow.taskCalls, emptyBackendConfig) + val backend = getSparkBackend(workflowDescriptor, workflowDescriptor.workflow.taskCalls, TestConfig.emptyBackendConfigDescriptor) backend ! Initialize } } diff --git a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesConfiguration.scala b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesConfiguration.scala index 65f4deec6..9a499dde5 100644 --- a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesConfiguration.scala +++ b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesConfiguration.scala @@ -4,4 +4,5 @@ import cromwell.backend.BackendConfigurationDescriptor class TesConfiguration(val configurationDescriptor: BackendConfigurationDescriptor) { val endpointURL = configurationDescriptor.backendConfig.getString("endpoint") + val runtimeConfig = configurationDescriptor.backendRuntimeConfig } diff --git a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesInitializationActor.scala b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesInitializationActor.scala index 773522f43..7da53c5c3 100644 --- a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesInitializationActor.scala +++ b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesInitializationActor.scala @@ -27,7 +27,7 @@ class TesInitializationActor(params: TesInitializationActorParams) new TesWorkflowPaths(workflowDescriptor, tesConfiguration.configurationDescriptor.backendConfig, pathBuilders) override lazy val runtimeAttributesBuilder: StandardValidatedRuntimeAttributesBuilder = - TesRuntimeAttributes.runtimeAttributesBuilder + TesRuntimeAttributes.runtimeAttributesBuilder(tesConfiguration.runtimeConfig) override def beforeAll(): Future[Option[BackendInitializationData]] = { Future.fromTry(Try { diff --git a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesJobCachingActorHelper.scala b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesJobCachingActorHelper.scala index 0f12d802f..58828915f 100644 --- a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesJobCachingActorHelper.scala +++ b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesJobCachingActorHelper.scala @@ -18,5 +18,5 @@ trait TesJobCachingActorHelper extends StandardCachingActorHelper { lazy val tesConfiguration: TesConfiguration = initializationData.tesConfiguration - lazy val runtimeAttributes = TesRuntimeAttributes(validatedRuntimeAttributes) + lazy val runtimeAttributes = TesRuntimeAttributes(validatedRuntimeAttributes, tesConfiguration.runtimeConfig) } diff --git a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesRuntimeAttributes.scala b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesRuntimeAttributes.scala index 06069c12b..b159fee47 100644 --- a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesRuntimeAttributes.scala +++ b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesRuntimeAttributes.scala @@ -1,6 +1,7 @@ package cromwell.backend.impl.tes import cats.syntax.validated._ +import com.typesafe.config.Config import cromwell.backend.MemorySize import cromwell.backend.standard.StandardValidatedRuntimeAttributesBuilder import cromwell.backend.validation._ @@ -8,6 +9,8 @@ import lenthall.validation.ErrorOr.ErrorOr import wdl4s.parser.MemoryUnit import wdl4s.values.{WdlInteger, WdlString, WdlValue} +import scala.util.{Failure, Success} + case class TesRuntimeAttributes(continueOnReturnCode: ContinueOnReturnCode, dockerImage: String, dockerWorkingDir: Option[String], @@ -25,37 +28,43 @@ object TesRuntimeAttributes { val DiskSizeKey = "disk" private val DiskSizeDefaultValue = "2 GB" - private val cpuValidation: RuntimeAttributesValidation[Int] = CpuValidation.default + private def cpuValidation(runtimeConfig: Option[Config]): RuntimeAttributesValidation[Int] = CpuValidation.instance + .withDefault(CpuValidation.configDefaultWdlValue(runtimeConfig) getOrElse CpuValidation.default) + + private def failOnStderrValidation(runtimeConfig: Option[Config]) = FailOnStderrValidation.default(runtimeConfig) + + private def continueOnReturnCodeValidation(runtimeConfig: Option[Config]) = ContinueOnReturnCodeValidation.default(runtimeConfig) - private val diskSizeValidation: RuntimeAttributesValidation[MemorySize] = - DiskSizeValidation.withDefaultDiskSize(MemorySize.parse(DiskSizeDefaultValue).get) + private def diskSizeValidation(runtimeConfig: Option[Config]): RuntimeAttributesValidation[MemorySize] = DiskSizeValidation + .withDefaultDiskSize(DiskSizeValidation.configDefaultString(runtimeConfig) getOrElse DiskSizeDefaultValue) + + private def memoryValidation(runtimeConfig: Option[Config]): RuntimeAttributesValidation[MemorySize] = { + MemoryValidation.withDefaultMemory(MemoryValidation.configDefaultString(runtimeConfig) getOrElse MemoryDefaultValue) + } private val dockerValidation: RuntimeAttributesValidation[String] = DockerValidation.instance private val dockerWorkingDirValidation: OptionalRuntimeAttributesValidation[String] = DockerWorkingDirValidation.optional - private val memoryValidation: RuntimeAttributesValidation[MemorySize] = - MemoryValidation.withDefaultMemory(MemorySize.parse(MemoryDefaultValue).get) - - def runtimeAttributesBuilder: StandardValidatedRuntimeAttributesBuilder = - StandardValidatedRuntimeAttributesBuilder.default.withValidation( - cpuValidation, - memoryValidation, - diskSizeValidation, + def runtimeAttributesBuilder(backendRuntimeConfig: Option[Config]): StandardValidatedRuntimeAttributesBuilder = + StandardValidatedRuntimeAttributesBuilder.default(backendRuntimeConfig).withValidation( + cpuValidation(backendRuntimeConfig), + memoryValidation(backendRuntimeConfig), + diskSizeValidation(backendRuntimeConfig), dockerValidation, dockerWorkingDirValidation ) - def apply(validatedRuntimeAttributes: ValidatedRuntimeAttributes): TesRuntimeAttributes = { + def apply(validatedRuntimeAttributes: ValidatedRuntimeAttributes, backendRuntimeConfig: Option[Config]): TesRuntimeAttributes = { val docker: String = RuntimeAttributesValidation.extract(dockerValidation, validatedRuntimeAttributes) val dockerWorkingDir: Option[String] = RuntimeAttributesValidation.extractOption(dockerWorkingDirValidation.key, validatedRuntimeAttributes) - val cpu: Int = RuntimeAttributesValidation.extract(cpuValidation, validatedRuntimeAttributes) - val memory: MemorySize = RuntimeAttributesValidation.extract(memoryValidation, validatedRuntimeAttributes) - val disk: MemorySize = RuntimeAttributesValidation.extract(diskSizeValidation, validatedRuntimeAttributes) + val cpu: Int = RuntimeAttributesValidation.extract(cpuValidation(backendRuntimeConfig), validatedRuntimeAttributes) + val memory: MemorySize = RuntimeAttributesValidation.extract(memoryValidation(backendRuntimeConfig), validatedRuntimeAttributes) + val disk: MemorySize = RuntimeAttributesValidation.extract(diskSizeValidation(backendRuntimeConfig), validatedRuntimeAttributes) val failOnStderr: Boolean = - RuntimeAttributesValidation.extract(FailOnStderrValidation.default, validatedRuntimeAttributes) + RuntimeAttributesValidation.extract(failOnStderrValidation(backendRuntimeConfig), validatedRuntimeAttributes) val continueOnReturnCode: ContinueOnReturnCode = - RuntimeAttributesValidation.extract(ContinueOnReturnCodeValidation.default, validatedRuntimeAttributes) + RuntimeAttributesValidation.extract(continueOnReturnCodeValidation(backendRuntimeConfig), validatedRuntimeAttributes) new TesRuntimeAttributes( continueOnReturnCode, @@ -84,9 +93,13 @@ class DockerWorkingDirValidation extends StringRuntimeAttributesValidation(TesRu object DiskSizeValidation { lazy val instance: RuntimeAttributesValidation[MemorySize] = new DiskSizeValidation lazy val optional: OptionalRuntimeAttributesValidation[MemorySize] = instance.optional - - def withDefaultDiskSize(memorySize: MemorySize): RuntimeAttributesValidation[MemorySize] = - instance.withDefault(WdlInteger(memorySize.bytes.toInt)) + def configDefaultString(runtimeConfig: Option[Config]): Option[String] = instance.configDefaultValue(runtimeConfig) + def withDefaultDiskSize(memorySize: String): RuntimeAttributesValidation[MemorySize] = { + MemorySize.parse(memorySize) match { + case Success(memory) => instance.withDefault(WdlInteger(memory.bytes.toInt)) + case Failure(_) => instance.withDefault(BadDefaultAttribute(WdlString(memorySize.toString))) + } + } private val wrongAmountFormat = s"Expecting ${TesRuntimeAttributes.DiskSizeKey} runtime attribute value greater than 0 but got %s" diff --git a/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesInitializationActorSpec.scala b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesInitializationActorSpec.scala index 0f033bbb1..f75c11561 100644 --- a/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesInitializationActorSpec.scala +++ b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesInitializationActorSpec.scala @@ -50,6 +50,17 @@ class TesInitializationActorSpec extends TestKitSuite("TesInitializationActorSpe |// Polling for completion backs-off gradually for slower-running jobs. |// This is the maximum polling interval (in seconds): |maximum-polling-interval = 600 + | + |default-runtime-attributes { + | cpu: 1 + | failOnStderr: false + | continueOnReturnCode: 0 + | memory: "2 GB" + | disk: "2 GB" + | # The keys below have been commented out as they are optional runtime attributes. + | # dockerWorkingDir + | # docker + |} |""".stripMargin diff --git a/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesJobPathsSpec.scala b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesJobPathsSpec.scala index a5c1e6eeb..1d2ed7db6 100644 --- a/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesJobPathsSpec.scala +++ b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesJobPathsSpec.scala @@ -1,40 +1,18 @@ package cromwell.backend.impl.tes import better.files._ -import com.typesafe.config.ConfigFactory -import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptorKey, BackendSpec} +import cromwell.backend.{BackendJobDescriptorKey, BackendSpec} import org.scalatest.{FlatSpec, Matchers} import wdl4s.TaskCall class TesJobPathsSpec extends FlatSpec with Matchers with BackendSpec { - val configString = - """ - |root = "local-cromwell-executions" - |dockerRoot = "/cromwell-executions" - | - |filesystems { - | local { - | localization: [ - | "hard-link", "soft-link", "copy" - | ] - | } - | gcs { - | auth = "application-default" - | } - |} - |""".stripMargin - - val globalConfig = ConfigFactory.load() - val backendConfig = ConfigFactory.parseString(configString) - val defaultBackendConfigDescriptor = BackendConfigurationDescriptor(backendConfig, globalConfig) - "JobPaths" should "provide correct paths for a job" in { val wd = buildWorkflowDescriptor(TestWorkflows.HelloWorld) val call: TaskCall = wd.workflow.taskCalls.head val jobKey = BackendJobDescriptorKey(call, None, 1) - val jobPaths = new TesJobPaths(jobKey, wd, backendConfig) + val jobPaths = new TesJobPaths(jobKey, wd, TesTestConfig.backendConfig) val id = wd.id jobPaths.callRoot.toString shouldBe File(s"local-cromwell-executions/wf_hello/$id/call-hello").pathAsString @@ -56,17 +34,17 @@ class TesJobPathsSpec extends FlatSpec with Matchers with BackendSpec { File(s"/cromwell-executions/wf_hello/$id/call-hello/execution").pathAsString val jobKeySharded = BackendJobDescriptorKey(call, Option(0), 1) - val jobPathsSharded = new TesJobPaths(jobKeySharded, wd, backendConfig) + val jobPathsSharded = new TesJobPaths(jobKeySharded, wd, TesTestConfig.backendConfig) jobPathsSharded.callExecutionRoot.toString shouldBe File(s"local-cromwell-executions/wf_hello/$id/call-hello/shard-0/execution").pathAsString val jobKeyAttempt = BackendJobDescriptorKey(call, None, 2) - val jobPathsAttempt = new TesJobPaths(jobKeyAttempt, wd, backendConfig) + val jobPathsAttempt = new TesJobPaths(jobKeyAttempt, wd, TesTestConfig.backendConfig) jobPathsAttempt.callExecutionRoot.toString shouldBe File(s"local-cromwell-executions/wf_hello/$id/call-hello/attempt-2/execution").pathAsString val jobKeyShardedAttempt = BackendJobDescriptorKey(call, Option(0), 2) - val jobPathsShardedAttempt = new TesJobPaths(jobKeyShardedAttempt, wd, backendConfig) + val jobPathsShardedAttempt = new TesJobPaths(jobKeyShardedAttempt, wd, TesTestConfig.backendConfig) jobPathsShardedAttempt.callExecutionRoot.toString shouldBe File(s"local-cromwell-executions/wf_hello/$id/call-hello/shard-0/attempt-2/execution").pathAsString } diff --git a/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesRuntimeAttributesSpec.scala b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesRuntimeAttributesSpec.scala index 5fb711925..ee21eef66 100644 --- a/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesRuntimeAttributesSpec.scala +++ b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesRuntimeAttributesSpec.scala @@ -1,7 +1,7 @@ package cromwell.backend.impl.tes import cromwell.backend.validation.ContinueOnReturnCodeSet -import cromwell.backend.{MemorySize, RuntimeAttributeDefinition} +import cromwell.backend.{BackendConfigurationDescriptor, MemorySize, RuntimeAttributeDefinition, TestConfig} import cromwell.core.WorkflowOptions import org.scalatest.{Matchers, WordSpecLike} import org.slf4j.helpers.NOPLogger @@ -137,12 +137,15 @@ class TesRuntimeAttributesSpec extends WordSpecLike with Matchers { ) } + private val mockConfigurationDescriptor = BackendConfigurationDescriptor(TesTestConfig.backendConfig, TestConfig.globalConfig) + private val mockTesConfiguration = new TesConfiguration(mockConfigurationDescriptor) + private def assertSuccess(runtimeAttributes: Map[String, WdlValue], expectedRuntimeAttributes: TesRuntimeAttributes, workflowOptions: WorkflowOptions = emptyWorkflowOptions): Unit = { try { - val actualRuntimeAttributes = toTesRuntimeAttributes(runtimeAttributes, workflowOptions) + val actualRuntimeAttributes = toTesRuntimeAttributes(runtimeAttributes, workflowOptions, mockTesConfiguration) assert(actualRuntimeAttributes == expectedRuntimeAttributes) } catch { case ex: RuntimeException => fail(s"Exception was not expected but received: ${ex.getMessage}") @@ -154,7 +157,7 @@ class TesRuntimeAttributesSpec extends WordSpecLike with Matchers { exMsg: String, workflowOptions: WorkflowOptions = emptyWorkflowOptions): Unit = { try { - toTesRuntimeAttributes(runtimeAttributes, workflowOptions) + toTesRuntimeAttributes(runtimeAttributes, workflowOptions, mockTesConfiguration) fail("A RuntimeException was expected.") } catch { case ex: RuntimeException => assert(ex.getMessage.contains(exMsg)) @@ -164,15 +167,17 @@ class TesRuntimeAttributesSpec extends WordSpecLike with Matchers { private val emptyWorkflowOptions = WorkflowOptions.fromMap(Map.empty).get private val staticRuntimeAttributeDefinitions: Set[RuntimeAttributeDefinition] = - TesRuntimeAttributes.runtimeAttributesBuilder.definitions.toSet + TesRuntimeAttributes.runtimeAttributesBuilder(mockTesConfiguration.runtimeConfig).definitions.toSet private def toTesRuntimeAttributes(runtimeAttributes: Map[String, WdlValue], - workflowOptions: WorkflowOptions): TesRuntimeAttributes = { - val runtimeAttributesBuilder = TesRuntimeAttributes.runtimeAttributesBuilder + workflowOptions: WorkflowOptions, + tesConfiguration: TesConfiguration): TesRuntimeAttributes = { + val runtimeAttributesBuilder = TesRuntimeAttributes.runtimeAttributesBuilder(tesConfiguration.runtimeConfig) val defaultedAttributes = RuntimeAttributeDefinition.addDefaultsToAttributes( staticRuntimeAttributeDefinitions, workflowOptions)(runtimeAttributes) val validatedRuntimeAttributes = runtimeAttributesBuilder.build(defaultedAttributes, NOPLogger.NOP_LOGGER) - TesRuntimeAttributes(validatedRuntimeAttributes) + TesRuntimeAttributes(validatedRuntimeAttributes, tesConfiguration.runtimeConfig + ) } -} \ No newline at end of file +} diff --git a/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesTestConfig.scala b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesTestConfig.scala new file mode 100644 index 000000000..eb3085706 --- /dev/null +++ b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesTestConfig.scala @@ -0,0 +1,27 @@ +package cromwell.backend.impl.tes + +import com.typesafe.config.ConfigFactory + +object TesTestConfig { + + private val backendConfigString = + """ + |root = "local-cromwell-executions" + |dockerRoot = "/cromwell-executions" + |endpoint = "http://127.0.0.1:9000/v1/jobs" + | + |default-runtime-attributes { + | cpu: 1 + | failOnStderr: false + | continueOnReturnCode: 0 + | memory: "2 GB" + | disk: "2 GB" + | # The keys below have been commented out as they are optional runtime attributes. + | # dockerWorkingDir + | # docker + |} + |""".stripMargin + + val backendConfig = ConfigFactory.parseString(backendConfigString) +} + diff --git a/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesWorkflowPathsSpec.scala b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesWorkflowPathsSpec.scala index 591b65193..8ace9b40c 100644 --- a/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesWorkflowPathsSpec.scala +++ b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesWorkflowPathsSpec.scala @@ -1,36 +1,16 @@ package cromwell.backend.impl.tes import better.files._ -import com.typesafe.config.ConfigFactory import cromwell.backend.{BackendJobBreadCrumb, BackendSpec, BackendWorkflowDescriptor} import cromwell.core.{JobKey, WorkflowId} import org.scalatest.{FlatSpec, Matchers} import wdl4s.{Call, Workflow} class TesWorkflowPathsSpec extends FlatSpec with Matchers with BackendSpec { - val configString = - """ - |root = "local-cromwell-executions" - |dockerRoot = "cromwell-executions" - | - |filesystems { - | local { - | localization: [ - | "hard-link", "soft-link", "copy" - | ] - | } - | gcs { - | auth = "application-default" - | } - |} - |""".stripMargin - - val globalConfig = ConfigFactory.load() - val backendConfig = ConfigFactory.parseString(configString) "WorkflowPaths" should "provide correct paths for a workflow" in { val wd = buildWorkflowDescriptor(TestWorkflows.HelloWorld) - val workflowPaths = new TesWorkflowPaths(wd, backendConfig) + val workflowPaths = new TesWorkflowPaths(wd, TesTestConfig.backendConfig) val id = wd.id workflowPaths.workflowRoot.toString shouldBe File(s"local-cromwell-executions/wf_hello/$id").pathAsString @@ -68,7 +48,7 @@ class TesWorkflowPathsSpec extends FlatSpec with Matchers with BackendSpec { subWd.breadCrumbs returns List(BackendJobBreadCrumb(rootWorkflow, rootWorkflowId, jobKey)) subWd.id returns subWorkflowId - val workflowPaths = new TesWorkflowPaths(subWd, backendConfig) + val workflowPaths = new TesWorkflowPaths(subWd, TesTestConfig.backendConfig) workflowPaths.workflowRoot.toString shouldBe File(s"local-cromwell-executions/rootWorkflow/$rootWorkflowId/call-call1/shard-1/attempt-2/subWorkflow/$subWorkflowId").pathAsString workflowPaths.dockerWorkflowRoot.toString shouldBe s"/cromwell-executions/rootWorkflow/$rootWorkflowId/call-call1/shard-1/attempt-2/subWorkflow/$subWorkflowId" } From 9aff9f2957d303a4789801d6a482777faf47d48f Mon Sep 17 00:00:00 2001 From: Thib Date: Fri, 14 Apr 2017 17:58:25 -0400 Subject: [PATCH 006/134] Enable local docker lookup (#2139) * Added a docker cli flow. Changed the default behavior to use the cli (aka local) flow, with an option to add or switch to the remote registry flow. * Add enable flag for docker hash lookup * Disable docker lookup if backend doesn't support docker * changelog * get ready for merge --- CHANGELOG.md | 10 ++ README.md | 30 +++-- backend/CHANGELOG.MD | 5 - core/src/main/resources/reference.conf | 18 ++- .../cromwell/core/BackendDockerConfiguration.scala | 31 +++++ .../scala/cromwell/core/DockerConfiguration.scala | 45 +++++++ .../scala/cromwell/core/DockerCredentials.scala | 40 ------ .../cromwell/core/actor/RobustClientHelper.scala | 4 +- .../docker/DockerClientHelper.scala | 6 +- .../{core/callcaching => }/docker/DockerFlow.scala | 2 +- .../callcaching => }/docker/DockerHashActor.scala | 4 +- .../docker/DockerHashRequest.scala | 2 +- .../callcaching => }/docker/DockerHashResult.scala | 2 +- .../docker/DockerImageIdentifier.scala | 2 +- .../cromwell/docker/local/DockerCliClient.scala | 105 +++++++++++++++ .../cromwell/docker/local/DockerCliFlow.scala | 142 +++++++++++++++++++++ .../scala/cromwell/docker/local/DockerCliKey.scala | 14 ++ .../registryv2/DockerRegistryV2AbstractFlow.scala | 12 +- .../docker/registryv2/flows/FlowUtils.scala | 2 +- .../registryv2/flows/HttpFlowWithRetry.scala | 6 +- .../registryv2/flows/dockerhub/DockerHubFlow.scala | 12 +- .../registryv2/flows/gcr/GcrAbstractFlow.scala | 8 +- .../docker/registryv2/flows/gcr/GcrEuFlow.scala | 4 +- .../docker/registryv2/flows/gcr/GcrFlow.scala | 4 +- .../docker/registryv2/flows/gcr/GcrUsFlow.scala | 4 +- .../docker/registryv2/flows/gcr/GoogleFlow.scala | 8 +- .../cromwell/docker/DockerEmptyFlowSpec.scala | 45 +++++++ .../scala/cromwell/docker/DockerFlowSpec.scala | 36 ++++++ .../docker/DockerHashActorSpec.scala | 40 +----- .../callcaching => }/docker/DockerHashMocks.scala | 6 +- .../docker/DockerImageIdentifierSpec.scala | 2 +- .../docker/local/DockerCliClientSpec.scala | 54 ++++++++ .../cromwell/docker/local/DockerCliFlowSpec.scala | 55 ++++++++ .../docker/local/DockerCliTimeoutFlowSpec.scala | 81 ++++++++++++ .../registryv2/flows/HttpFlowWithRetrySpec.scala | 4 +- .../preparation/JobPreparationActor.scala | 17 ++- .../scala/cromwell/server/CromwellRootActor.scala | 31 +++-- .../test/scala/cromwell/CromwellTestKitSpec.scala | 4 +- .../preparation/JobPreparationActorSpec.scala | 4 +- .../preparation/JobPreparationTestHelper.scala | 1 + src/bin/travis/resources/local_centaur.conf | 37 ++++++ .../backend/impl/jes/JesConfiguration.scala | 4 +- 42 files changed, 777 insertions(+), 166 deletions(-) delete mode 100644 backend/CHANGELOG.MD create mode 100644 core/src/main/scala/cromwell/core/BackendDockerConfiguration.scala create mode 100644 core/src/main/scala/cromwell/core/DockerConfiguration.scala delete mode 100644 core/src/main/scala/cromwell/core/DockerCredentials.scala rename dockerHashing/src/main/scala/cromwell/{core/callcaching => }/docker/DockerClientHelper.scala (84%) rename dockerHashing/src/main/scala/cromwell/{core/callcaching => }/docker/DockerFlow.scala (91%) rename dockerHashing/src/main/scala/cromwell/{core/callcaching => }/docker/DockerHashActor.scala (98%) rename dockerHashing/src/main/scala/cromwell/{core/callcaching => }/docker/DockerHashRequest.scala (73%) rename dockerHashing/src/main/scala/cromwell/{core/callcaching => }/docker/DockerHashResult.scala (93%) rename dockerHashing/src/main/scala/cromwell/{core/callcaching => }/docker/DockerImageIdentifier.scala (99%) create mode 100644 dockerHashing/src/main/scala/cromwell/docker/local/DockerCliClient.scala create mode 100644 dockerHashing/src/main/scala/cromwell/docker/local/DockerCliFlow.scala create mode 100644 dockerHashing/src/main/scala/cromwell/docker/local/DockerCliKey.scala rename dockerHashing/src/main/scala/cromwell/{core/callcaching => }/docker/registryv2/DockerRegistryV2AbstractFlow.scala (95%) rename dockerHashing/src/main/scala/cromwell/{core/callcaching => }/docker/registryv2/flows/FlowUtils.scala (95%) rename dockerHashing/src/main/scala/cromwell/{core/callcaching => }/docker/registryv2/flows/HttpFlowWithRetry.scala (97%) rename dockerHashing/src/main/scala/cromwell/{core/callcaching => }/docker/registryv2/flows/dockerhub/DockerHubFlow.scala (77%) rename dockerHashing/src/main/scala/cromwell/{core/callcaching => }/docker/registryv2/flows/gcr/GcrAbstractFlow.scala (80%) rename dockerHashing/src/main/scala/cromwell/{core/callcaching => }/docker/registryv2/flows/gcr/GcrEuFlow.scala (65%) rename dockerHashing/src/main/scala/cromwell/{core/callcaching => }/docker/registryv2/flows/gcr/GcrFlow.scala (65%) rename dockerHashing/src/main/scala/cromwell/{core/callcaching => }/docker/registryv2/flows/gcr/GcrUsFlow.scala (65%) rename dockerHashing/src/main/scala/cromwell/{core/callcaching => }/docker/registryv2/flows/gcr/GoogleFlow.scala (86%) create mode 100644 dockerHashing/src/test/scala/cromwell/docker/DockerEmptyFlowSpec.scala create mode 100644 dockerHashing/src/test/scala/cromwell/docker/DockerFlowSpec.scala rename dockerHashing/src/test/scala/cromwell/{core/callcaching => }/docker/DockerHashActorSpec.scala (69%) rename dockerHashing/src/test/scala/cromwell/{core/callcaching => }/docker/DockerHashMocks.scala (89%) rename dockerHashing/src/test/scala/cromwell/{core/callcaching => }/docker/DockerImageIdentifierSpec.scala (98%) create mode 100644 dockerHashing/src/test/scala/cromwell/docker/local/DockerCliClientSpec.scala create mode 100644 dockerHashing/src/test/scala/cromwell/docker/local/DockerCliFlowSpec.scala create mode 100644 dockerHashing/src/test/scala/cromwell/docker/local/DockerCliTimeoutFlowSpec.scala rename dockerHashing/src/test/scala/cromwell/{core/callcaching => }/docker/registryv2/flows/HttpFlowWithRetrySpec.scala (98%) diff --git a/CHANGELOG.md b/CHANGELOG.md index e28de5e73..9e9efc661 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## 27 +### Breaking Changes * The update to Slick 3.2 requires a database stanza to [switch](http://slick.lightbend.com/doc/3.2.0/upgrade.html#profiles-vs-drivers) from using `driver` to `profile`. @@ -20,6 +21,15 @@ database { } ``` +### Local Docker Lookup + +* The Docker section of the configuration has been slightly reworked +An option to specify how a Docker hash should be looked up has been added. Two methods are available. + "local" will try to look for the image on the machine where cromwell is running. If it can't be found, Cromwell will try to `pull` the image and use the hash from the retrieved image. + "remote" will try to look up the image hash directly on the remote repository where the image is located (Docker Hub and GCR are supported) +Note that the "local" option will require docker to be installed on the machine running cromwell, in order for it to call the docker CLI. + + ## 26 ### Breaking Changes diff --git a/README.md b/README.md index 358c454b7..f272cef09 100644 --- a/README.md +++ b/README.md @@ -1902,14 +1902,28 @@ When Cromwell finds a job ready to be run, it will first look at its docker runt If call caching writing is turned on, Cromwell will still write the job in the cache database, using: * the hash if the lookup succeeded * the floating tag otherwise. - -Docker registry and access levels supported by Cromwell for docker hash lookup: - -| | DockerHub || GCR || -|:-----:|:---------:|:-------:|:------:|:-------:| -| | Public | Private | Public | Private | -| JES | X | X | X | X | -| Other | X | | X | | + +### Docker Lookup + +Cromwell provides 2 methods to lookup a docker hash from a docker tag: + +* Local + In this mode, cromwell will first attempt to find the image on the local machine where it's running using the docker CLI. If the image is present, then its hash will be used. + If it's not present, cromwell will execute a `docker pull` to try and retrieve it. If this succeeds, the newly retrieved hash will be used. Otherwise the lookup will be considered failed. + Note that cromwell runs the `docker` CLI the same way a human would. This means two things: + * The machine Cromwell is running on needs to have docker installed and a docker daemon running. + * Whichever credentials (and only those) are available on that machine will be available to pull the image. + +* Remote + In this mode, cromwell will attempt to retrieve the hash by contacting the remote docker registry where the image is stored. This currently supports Docker Hub and GCR. + + Docker registry and access levels supported by Cromwell for docker hash lookup in "remote" mode: + + | | DockerHub || GCR || + |:-----:|:---------:|:-------:|:------:|:-------:| + | | Public | Private | Public | Private | + | JES | X | X | X | X | + | Other | X | | X | | ## Local Filesystem Options When running a job on the Config (Shared Filesystem) backend, Cromwell provides some additional options in the backend's config section: diff --git a/backend/CHANGELOG.MD b/backend/CHANGELOG.MD deleted file mode 100644 index 6e6cfdff0..000000000 --- a/backend/CHANGELOG.MD +++ /dev/null @@ -1,5 +0,0 @@ -# Cromwell-Backend Change Log - -1.0: -===== - diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf index fd1321aeb..c8b718e57 100644 --- a/core/src/main/resources/reference.conf +++ b/core/src/main/resources/reference.conf @@ -149,12 +149,18 @@ google { } docker { - // Set this to match your available quota against the Google Container Engine API - gcr-api-queries-per-100-seconds = 1000 - // Time in minutes before an entry expires from the docker hashes cache and needs to be fetched again - cache-entry-ttl = "20 minutes" - // Maximum number of elements to be kept in the cache. If the limit is reached, old elements will be removed from the cache - cache-size = 200 + hash-lookup { + // Set this to match your available quota against the Google Container Engine API + gcr-api-queries-per-100-seconds = 1000 + // Time in minutes before an entry expires from the docker hashes cache and needs to be fetched again + cache-entry-ttl = "20 minutes" + // Maximum number of elements to be kept in the cache. If the limit is reached, old elements will be removed from the cache + cache-size = 200 + // How should docker hashes be looked up. Possible values are "local" and "remote" + // "local": Lookup hashes on the local docker daemon using the cli + // "remote": Lookup hashes on docker hub and gcr + method = "remote" + } } engine { diff --git a/core/src/main/scala/cromwell/core/BackendDockerConfiguration.scala b/core/src/main/scala/cromwell/core/BackendDockerConfiguration.scala new file mode 100644 index 000000000..5957bd50c --- /dev/null +++ b/core/src/main/scala/cromwell/core/BackendDockerConfiguration.scala @@ -0,0 +1,31 @@ +package cromwell.core + +import com.typesafe.config.Config +import cromwell.core.ConfigUtil._ + +/** + * Encapsulate docker credential information. + */ +case class DockerCredentials(account: String, token: String) + +case class BackendDockerConfiguration(dockerCredentials: Option[DockerCredentials]) + +/** + * Singleton encapsulating a DockerConf instance. + */ +object BackendDockerConfiguration { + + private val dockerKeys = Set("account", "token") + + def build(config: Config) = { + import net.ceedubs.ficus.Ficus._ + val dockerConf: Option[DockerCredentials] = for { + dockerConf <- config.as[Option[Config]]("dockerhub") + _ = dockerConf.warnNotRecognized(dockerKeys, "dockerhub") + account <- dockerConf.validateString("account").toOption + token <- dockerConf.validateString("token").toOption + } yield DockerCredentials(account, token) + + new BackendDockerConfiguration(dockerConf) + } +} diff --git a/core/src/main/scala/cromwell/core/DockerConfiguration.scala b/core/src/main/scala/cromwell/core/DockerConfiguration.scala new file mode 100644 index 000000000..ca0f872fc --- /dev/null +++ b/core/src/main/scala/cromwell/core/DockerConfiguration.scala @@ -0,0 +1,45 @@ +package cromwell.core + +import com.typesafe.config.ConfigFactory + +import scala.concurrent.duration.FiniteDuration +import cats.data.Validated._ +import cats.syntax.cartesian._ +import lenthall.exception.AggregatedMessageException +import net.ceedubs.ficus.Ficus._ +import lenthall.validation.Validation._ + +object DockerConfiguration { + private lazy val dockerConfig = ConfigFactory.load().getConfig("docker") + private lazy val dockerHashLookupConfig = dockerConfig.getConfig("hash-lookup") + + lazy val instance: DockerConfiguration = { + val gcrApiQueriesPer100Seconds = validate { dockerHashLookupConfig.as[Int]("gcr-api-queries-per-100-seconds") } + val cacheEntryTtl = validate { dockerHashLookupConfig.as[FiniteDuration]("cache-entry-ttl") } + val cacheSize = validate { dockerHashLookupConfig.as[Long]("cache-size") } + val method = validate { dockerHashLookupConfig.as[String]("method") } map { + case "local" => DockerLocalLookup + case "remote" => DockerRemoteLookup + case other => throw new IllegalArgumentException(s"Unrecognized docker hash lookup method: $other") + } + + val dockerConfiguration = (gcrApiQueriesPer100Seconds |@| cacheEntryTtl |@| cacheSize |@| method) map DockerConfiguration.apply + + dockerConfiguration match { + case Valid(conf) => conf + case Invalid(errors) => throw AggregatedMessageException("Invalid docker configuration", errors.toList) + } + } +} + +case class DockerConfiguration( + gcrApiQueriesPer100Seconds: Int, + cacheEntryTtl: FiniteDuration, + cacheSize: Long, + method: DockerHashLookupMethod + ) + +sealed trait DockerHashLookupMethod + +case object DockerLocalLookup extends DockerHashLookupMethod +case object DockerRemoteLookup extends DockerHashLookupMethod diff --git a/core/src/main/scala/cromwell/core/DockerCredentials.scala b/core/src/main/scala/cromwell/core/DockerCredentials.scala deleted file mode 100644 index cfe7be01f..000000000 --- a/core/src/main/scala/cromwell/core/DockerCredentials.scala +++ /dev/null @@ -1,40 +0,0 @@ -package cromwell.core - -import com.typesafe.config.Config -import cromwell.core.ConfigUtil._ - -/** - * Encapsulate docker credential information. - */ -case class DockerCredentials(account: String, token: String) - -case class DockerHubConfiguration(namespace: String, v1Registry: String, v2Registry: String) - -case class DockerConfiguration(dockerCredentials: Option[DockerCredentials], dockerHubConf: DockerHubConfiguration) - -/** - * Singleton encapsulating a DockerConf instance. - */ -object DockerConfiguration { - - private val dockerKeys = Set("account", "token") - - def build(config: Config) = { - import net.ceedubs.ficus.Ficus._ - val dockerConf: Option[DockerCredentials] = for { - dockerConf <- config.as[Option[Config]]("dockerhub") - _ = dockerConf.warnNotRecognized(dockerKeys, "dockerhub") - account <- dockerConf.validateString("account").toOption - token <- dockerConf.validateString("token").toOption - } yield DockerCredentials(account, token) - - val dockerHubConf = { - DockerHubConfiguration( - namespace = config.as[Option[String]]("docker.hub.namespace").getOrElse("docker.io"), - v1Registry = config.as[Option[String]]("docker.hub.v1Registry").getOrElse("index.docker.io"), - v2Registry = config.as[Option[String]]("docker.hub.v2Registry").getOrElse("registry-1.docker.io") - ) - } - new DockerConfiguration(dockerConf, dockerHubConf) - } -} diff --git a/core/src/main/scala/cromwell/core/actor/RobustClientHelper.scala b/core/src/main/scala/cromwell/core/actor/RobustClientHelper.scala index 579c6a6a7..ff413ee78 100644 --- a/core/src/main/scala/cromwell/core/actor/RobustClientHelper.scala +++ b/core/src/main/scala/cromwell/core/actor/RobustClientHelper.scala @@ -24,7 +24,7 @@ trait RobustClientHelper { this: Actor with ActorLogging => protected def backpressureTimeout: FiniteDuration = 10 seconds protected def backpressureRandomizerFactor: Double = 0.5D - private [core] def robustReceive: Receive = { + def robustReceive: Receive = { case BackPressure(request) => val snd = sender() newTimer(request, snd, generateBackpressureTime) @@ -37,7 +37,7 @@ trait RobustClientHelper { this: Actor with ActorLogging => context.system.scheduler.scheduleOnce(in, to, msg)(robustActorHelperEc, self) } - private [core] def robustSend(msg: Any, to: ActorRef, timeout: FiniteDuration = DefaultRequestLostTimeout): Unit = { + def robustSend(msg: Any, to: ActorRef, timeout: FiniteDuration = DefaultRequestLostTimeout): Unit = { to ! msg addTimeout(msg, to, timeout) } diff --git a/dockerHashing/src/main/scala/cromwell/core/callcaching/docker/DockerClientHelper.scala b/dockerHashing/src/main/scala/cromwell/docker/DockerClientHelper.scala similarity index 84% rename from dockerHashing/src/main/scala/cromwell/core/callcaching/docker/DockerClientHelper.scala rename to dockerHashing/src/main/scala/cromwell/docker/DockerClientHelper.scala index 6865f167e..9e97ca931 100644 --- a/dockerHashing/src/main/scala/cromwell/core/callcaching/docker/DockerClientHelper.scala +++ b/dockerHashing/src/main/scala/cromwell/docker/DockerClientHelper.scala @@ -1,8 +1,8 @@ -package cromwell.core.callcaching.docker +package cromwell.docker import akka.actor.{Actor, ActorLogging, ActorRef} import cromwell.core.actor.RobustClientHelper -import cromwell.core.callcaching.docker.DockerHashActor.DockerHashResponse +import cromwell.docker.DockerHashActor.DockerHashResponse import scala.concurrent.duration.FiniteDuration @@ -10,7 +10,7 @@ trait DockerClientHelper extends RobustClientHelper { this: Actor with ActorLogg protected def dockerHashingActor: ActorRef - private [core] def dockerResponseReceive: Receive = { + private [docker] def dockerResponseReceive: Receive = { case dockerResponse: DockerHashResponse if hasTimeout(dockerResponse.request) => cancelTimeout(dockerResponse.request) receive.apply(dockerResponse) diff --git a/dockerHashing/src/main/scala/cromwell/core/callcaching/docker/DockerFlow.scala b/dockerHashing/src/main/scala/cromwell/docker/DockerFlow.scala similarity index 91% rename from dockerHashing/src/main/scala/cromwell/core/callcaching/docker/DockerFlow.scala rename to dockerHashing/src/main/scala/cromwell/docker/DockerFlow.scala index cc5dc8bf0..e553cb8ee 100644 --- a/dockerHashing/src/main/scala/cromwell/core/callcaching/docker/DockerFlow.scala +++ b/dockerHashing/src/main/scala/cromwell/docker/DockerFlow.scala @@ -1,4 +1,4 @@ -package cromwell.core.callcaching.docker +package cromwell.docker import akka.NotUsed import akka.stream.{FlowShape, Graph} diff --git a/dockerHashing/src/main/scala/cromwell/core/callcaching/docker/DockerHashActor.scala b/dockerHashing/src/main/scala/cromwell/docker/DockerHashActor.scala similarity index 98% rename from dockerHashing/src/main/scala/cromwell/core/callcaching/docker/DockerHashActor.scala rename to dockerHashing/src/main/scala/cromwell/docker/DockerHashActor.scala index 1035e6a0a..9a28877d5 100644 --- a/dockerHashing/src/main/scala/cromwell/core/callcaching/docker/DockerHashActor.scala +++ b/dockerHashing/src/main/scala/cromwell/docker/DockerHashActor.scala @@ -1,4 +1,4 @@ -package cromwell.core.callcaching.docker +package cromwell.docker import akka.actor.{Actor, ActorLogging, ActorRef, Props} import akka.stream._ @@ -7,7 +7,7 @@ import com.google.common.cache.CacheBuilder import cromwell.core.Dispatcher import cromwell.core.actor.StreamActorHelper import cromwell.core.actor.StreamIntegration.StreamContext -import cromwell.core.callcaching.docker.DockerHashActor._ +import cromwell.docker.DockerHashActor._ import org.slf4j.LoggerFactory import scala.concurrent.duration.FiniteDuration diff --git a/dockerHashing/src/main/scala/cromwell/core/callcaching/docker/DockerHashRequest.scala b/dockerHashing/src/main/scala/cromwell/docker/DockerHashRequest.scala similarity index 73% rename from dockerHashing/src/main/scala/cromwell/core/callcaching/docker/DockerHashRequest.scala rename to dockerHashing/src/main/scala/cromwell/docker/DockerHashRequest.scala index 6c7206d25..c6956318f 100644 --- a/dockerHashing/src/main/scala/cromwell/core/callcaching/docker/DockerHashRequest.scala +++ b/dockerHashing/src/main/scala/cromwell/docker/DockerHashRequest.scala @@ -1,3 +1,3 @@ -package cromwell.core.callcaching.docker +package cromwell.docker case class DockerHashRequest(dockerImageID: DockerImageIdentifierWithoutHash, credentials: List[Any] = List.empty) diff --git a/dockerHashing/src/main/scala/cromwell/core/callcaching/docker/DockerHashResult.scala b/dockerHashing/src/main/scala/cromwell/docker/DockerHashResult.scala similarity index 93% rename from dockerHashing/src/main/scala/cromwell/core/callcaching/docker/DockerHashResult.scala rename to dockerHashing/src/main/scala/cromwell/docker/DockerHashResult.scala index fbfd18de9..740d8e16c 100644 --- a/dockerHashing/src/main/scala/cromwell/core/callcaching/docker/DockerHashResult.scala +++ b/dockerHashing/src/main/scala/cromwell/docker/DockerHashResult.scala @@ -1,4 +1,4 @@ -package cromwell.core.callcaching.docker +package cromwell.docker object DockerHashResult { // See https://docs.docker.com/registry/spec/api/#/content-digests diff --git a/dockerHashing/src/main/scala/cromwell/core/callcaching/docker/DockerImageIdentifier.scala b/dockerHashing/src/main/scala/cromwell/docker/DockerImageIdentifier.scala similarity index 99% rename from dockerHashing/src/main/scala/cromwell/core/callcaching/docker/DockerImageIdentifier.scala rename to dockerHashing/src/main/scala/cromwell/docker/DockerImageIdentifier.scala index 9e40ab15d..e0accc8da 100644 --- a/dockerHashing/src/main/scala/cromwell/core/callcaching/docker/DockerImageIdentifier.scala +++ b/dockerHashing/src/main/scala/cromwell/docker/DockerImageIdentifier.scala @@ -1,4 +1,4 @@ -package cromwell.core.callcaching.docker +package cromwell.docker import scala.util.{Failure, Success, Try} diff --git a/dockerHashing/src/main/scala/cromwell/docker/local/DockerCliClient.scala b/dockerHashing/src/main/scala/cromwell/docker/local/DockerCliClient.scala new file mode 100644 index 000000000..9f3372422 --- /dev/null +++ b/dockerHashing/src/main/scala/cromwell/docker/local/DockerCliClient.scala @@ -0,0 +1,105 @@ +package cromwell.docker.local + +import scala.util.Try + +/** + * Wrapper around the docker cli. + * https://docs.docker.com/engine/reference/commandline/docker/ + * + * An alternative to using REST calls directly to the docker engine that has several versions, for example: + * https://docs.docker.com/engine/api/v1.27/ + */ +trait DockerCliClient { + /** + * Looks up a docker hash. + * + * @param dockerCliKey The docker hash to lookup. + * @return The hash if found, None if not found, and Failure if an error occurs. + */ + def lookupHash(dockerCliKey: DockerCliKey): Try[Option[String]] = { + /* + The stdout contains the tab separated repository/tag/digest for __all__ local images. + Would be great to just get a single hash using the key... unfortunately + https://github.com/docker/docker/issues/29901 + */ + forRun("docker", "images", "--digests", "--format", """{{printf "%s\t%s\t%s" .Repository .Tag .Digest}}""") { + _.flatMap(parseHashLine).find(_.key == dockerCliKey).map(_.digest) + } + } + + /** + * Pulls a docker image. + * @param dockerCliKey The docker hash to lookup. + * @return Failure if an error occurs. + */ + def pull(dockerCliKey: DockerCliKey): Try[Unit] = { + forRun("docker", "pull", dockerCliKey.fullName) { _ => () } + } + + /** + * Tries to run the command, then feeds the stdout to `f`. If the exit code is non-zero, returns a `Failure` with + * a `RuntimeException` containing the stderr. + * + * @param cmd Command line to run. + * @param f The function to run on the stdout contents. + * @tparam A Return type. + * @return An attempt to run A. + */ + private def forRun[A](cmd: String*)(f: Seq[String] => A): Try[A] = { + Try { + val dockerCliResult = run(cmd) + if (dockerCliResult.exitCode == 0) { + f(dockerCliResult.stdout) + } else { + throw new RuntimeException( + s"""|Error running: ${cmd.mkString(" ")} + |Exit code: ${dockerCliResult.exitCode} + |${dockerCliResult.stderr.mkString("\n")} + |""".stripMargin) + } + } + } + + /** + * Run a command and return the result. Overridable for testing. + * + * @param cmd The command to run. + * @return The results of the run wrapped in a DockerCliResult. + */ + private[local] def run(cmd: Seq[String]): DockerCliResult = { + import sys.process._ + var stdout = List.empty[String] + var stderr = List.empty[String] + val exitCode = cmd.!(ProcessLogger(line => stdout :+= line, line => stderr :+= line)) + DockerCliResult(exitCode, stdout, stderr) + } + + /** + * Parses a line for `lookupHash`, returning None for lines that contain the string "" for any of the columns. + * @param hashLine The line output by the stdout of the `lookupHash` command. + * @return An optional `DockerCliHash`, if the all the columns are found. + */ + private[local] def parseHashLine(hashLine: String): Option[DockerCliHash] = { + val none = "" + val tokens = hashLine.split("\t").lift + for { + repository <- tokens(0) if repository != none + tag <- tokens(1) if tag != none + digest <- tokens(2) if digest != none + } yield DockerCliHash(DockerCliKey(repository, tag), digest) + } +} + +object DockerCliClient extends DockerCliClient + +/** + * Used by `lookupHash` to return a mapping of keys to digests. + * See note in `lookupHash` regarding why we even have to loop over all images. + */ +case class DockerCliHash(key: DockerCliKey, digest: String) + +/** + * Utility that encapsulates a command's exit code, stdout, and stderr. + * Also used by tests to simulate cli responses. + */ +case class DockerCliResult(exitCode: Int, stdout: Seq[String], stderr: Seq[String]) diff --git a/dockerHashing/src/main/scala/cromwell/docker/local/DockerCliFlow.scala b/dockerHashing/src/main/scala/cromwell/docker/local/DockerCliFlow.scala new file mode 100644 index 000000000..760c04077 --- /dev/null +++ b/dockerHashing/src/main/scala/cromwell/docker/local/DockerCliFlow.scala @@ -0,0 +1,142 @@ +package cromwell.docker.local + +import java.util.concurrent.TimeoutException + +import akka.actor.Scheduler +import akka.stream.scaladsl.{Flow, GraphDSL, Merge, Partition} +import akka.stream.{ActorMaterializer, FlowShape} +import cromwell.docker.DockerHashActor._ +import cromwell.docker.{DockerFlow, DockerHashActor, DockerHashResult, DockerImageIdentifierWithoutHash} + +import scala.concurrent.duration._ +import scala.concurrent.{ExecutionContext, Future} +import scala.util.{Failure, Success} + +/** + * A docker flow using the CLI to return docker hashes. + */ +class DockerCliFlow(implicit ec: ExecutionContext, materializer: ActorMaterializer, scheduler: Scheduler) + extends DockerFlow { + + // If the docker cli hangs it would be difficult to debug. So timeout the first request after a short duration. + // https://github.com/docker/docker/issues/18279 + // https://github.com/docker/docker/issues/12606 + lazy val firstLookupTimeout = 5.seconds + + override def accepts(dockerImageIdentifierWithoutHash: DockerImageIdentifierWithoutHash): Boolean = true + + override def buildFlow() = GraphDSL.create() { implicit builder => + import GraphDSL.Implicits._ + + // Try a lookup. + val firstLookup = builder.add( + Flow + .fromFunction(DockerCliFlow.lookupHashOrTimeout(firstLookupTimeout)) + .mapAsync(1)(identity) + ) + + // Check if the hash was not found + val partition = builder.add(Partition[(DockerHashResponse, DockerHashContext)](2, { + case (_: DockerHashNotFound, _) => 0 // If we didn't find the docker hash, go to out(0) + case _ => 1 // Otherwise, go to out(1) + })) + + // If the hash wasn't found, `docker pull` then try a lookup a second time. If the docker pull fails for any + // reason, the error is ignored, allowing the second lookup to return not found. + val dockerPull = builder.add(Flow.fromFunction(Function.tupled(DockerCliFlow.pull _))) + val secondLookup = builder.add(Flow.fromFunction(DockerCliFlow.lookupHash)) + + // Use either the first or second docker lookup. + val merge = builder.add(Merge[(DockerHashResponse, DockerHashContext)](2)) + + // @formatter:off + firstLookup.out ~> partition + partition.out(0) ~> dockerPull ~> secondLookup ~> merge.in(0) + partition.out(1) ~> merge.in(1) + // @formatter:on + + FlowShape(firstLookup.in, merge.out) + } +} + +object DockerCliFlow { + /** + * Lookup the hash for the image referenced in the context. + * + * @param context The image to lookup. + * @return The docker hash response plus the context of our flow. + */ + private def lookupHash(context: DockerHashContext): (DockerHashResponse, DockerHashContext) = { + val dockerCliKey = cliKeyFromImageId(context) + DockerHashActor.logger.debug("Looking up hash of {}", dockerCliKey.fullName) + val result = DockerCliClient.lookupHash(dockerCliKey) match { + case Success(None) => DockerHashNotFound(context.request) + case Success(Some(hash)) => DockerHashResponseSuccess(DockerHashResult(hash), context.request) + case Failure(throwable) => DockerHashFailedResponse(throwable, context.request) + } + // give the compiler a hint on the debug() override we're trying to use. + DockerHashActor.logger.debug("Hash result of {} was {}", dockerCliKey.fullName, result.asInstanceOf[Any]) + (result, context) + } + + /** + * Lookup the hash for the image referenced in the context within the timeout, or DockerHashFailedResponse + * containing a TimeoutException. + * + * @param timeout How long to wait for the exception. + * @param context The image to lookup. + * @param scheduler Schedules the timout exception. + * @return The docker hash response plus the context of our flow. + */ + private def lookupHashOrTimeout(timeout: FiniteDuration) + (context: DockerHashContext) + (implicit ec: ExecutionContext, + scheduler: Scheduler): Future[(DockerHashResponse, DockerHashContext)] = { + val normal = Future(lookupHash(context)) + val delayed = akka.pattern.after(timeout, scheduler) { + val dockerCliKey = cliKeyFromImageId(context) + val exception = new TimeoutException( + s"""|Timeout while looking up hash of ${dockerCliKey.fullName}. + |Ensure that docker is running correctly. + |""".stripMargin) + val response = DockerHashFailedResponse(exception, context.request) + Future.successful((response, context)) + } + Future.firstCompletedOf(Seq(normal, delayed)) + } + + /** + * Pull the docker image referenced in context. + * + * @param notUsed Output of `lookupHash`, passed in only to help creating the graph easier. + * @param context The image to pull. + * @return The context of our flow. + */ + private def pull(notUsed: DockerHashResponse, context: DockerHashContext): DockerHashContext = { + val dockerCliKey = cliKeyFromImageId(context) + DockerHashActor.logger.info(s"Attempting to pull {}", dockerCliKey.fullName) + val result = DockerCliClient.pull(dockerCliKey) + result match { + case Success(_) => DockerHashActor.logger.info("Pulled {}", dockerCliKey.fullName) + case Failure(throwable) => DockerHashActor.logger.error("Docker pull failed", throwable) + } + context + } + + /** Utility for converting the flow image id to the format output by the docker cli. */ + private def cliKeyFromImageId(context: DockerHashContext): DockerCliKey = { + val imageId = context.dockerImageID + (imageId.host, imageId.repository) match { + case (None, "library") => + // For docker hub images (host == None), and don't include "library". + val repository = imageId.image + val tag = imageId.reference + DockerCliKey(repository, tag) + case _ => + // For all other images, include the host and repository. + val repository = s"${imageId.hostAsString}${imageId.repository}/${imageId.image}" + val tag = imageId.reference + DockerCliKey(repository, tag) + } + } +} diff --git a/dockerHashing/src/main/scala/cromwell/docker/local/DockerCliKey.scala b/dockerHashing/src/main/scala/cromwell/docker/local/DockerCliKey.scala new file mode 100644 index 000000000..a074a968b --- /dev/null +++ b/dockerHashing/src/main/scala/cromwell/docker/local/DockerCliKey.scala @@ -0,0 +1,14 @@ +package cromwell.docker.local + +/** + * Used by the docker cli for lookups or pulls. + * + * The format of `repository` is based on the output of `docker images`. It is also valid as input for the cli + * commands. + * + * @param repository The repository as listed by `docker images`. Includes the image host and name, but not the tag. + * @param tag The docker image tag. + */ +case class DockerCliKey(repository: String, tag: String) { + val fullName = s"$repository:$tag" +} diff --git a/dockerHashing/src/main/scala/cromwell/core/callcaching/docker/registryv2/DockerRegistryV2AbstractFlow.scala b/dockerHashing/src/main/scala/cromwell/docker/registryv2/DockerRegistryV2AbstractFlow.scala similarity index 95% rename from dockerHashing/src/main/scala/cromwell/core/callcaching/docker/registryv2/DockerRegistryV2AbstractFlow.scala rename to dockerHashing/src/main/scala/cromwell/docker/registryv2/DockerRegistryV2AbstractFlow.scala index 48229dd0a..9429d74f9 100644 --- a/dockerHashing/src/main/scala/cromwell/core/callcaching/docker/registryv2/DockerRegistryV2AbstractFlow.scala +++ b/dockerHashing/src/main/scala/cromwell/docker/registryv2/DockerRegistryV2AbstractFlow.scala @@ -1,4 +1,4 @@ -package cromwell.core.callcaching.docker.registryv2 +package cromwell.docker.registryv2 import akka.NotUsed import akka.actor.Scheduler @@ -9,11 +9,11 @@ import akka.http.scaladsl.model.headers.{Authorization, OAuth2BearerToken} import akka.http.scaladsl.unmarshalling.Unmarshal import akka.stream._ import akka.stream.scaladsl.{Flow, GraphDSL, Merge} -import cromwell.core.callcaching.docker.DockerHashActor._ -import cromwell.core.callcaching.docker.registryv2.DockerRegistryV2AbstractFlow._ -import cromwell.core.callcaching.docker.registryv2.flows.HttpFlowWithRetry.ContextWithRequest -import cromwell.core.callcaching.docker.registryv2.flows.{FlowUtils, HttpFlowWithRetry} -import cromwell.core.callcaching.docker.{DockerFlow, DockerHashResult, DockerImageIdentifierWithoutHash} +import cromwell.docker.DockerHashActor._ +import cromwell.docker.registryv2.DockerRegistryV2AbstractFlow._ +import cromwell.docker.registryv2.flows.HttpFlowWithRetry.ContextWithRequest +import cromwell.docker.registryv2.flows.{FlowUtils, HttpFlowWithRetry} +import cromwell.docker.{DockerFlow, DockerHashResult, DockerImageIdentifierWithoutHash} import spray.json._ import scala.concurrent.{ExecutionContext, Future} diff --git a/dockerHashing/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/FlowUtils.scala b/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/FlowUtils.scala similarity index 95% rename from dockerHashing/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/FlowUtils.scala rename to dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/FlowUtils.scala index 715d7a724..b98f45949 100644 --- a/dockerHashing/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/FlowUtils.scala +++ b/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/FlowUtils.scala @@ -1,4 +1,4 @@ -package cromwell.core.callcaching.docker.registryv2.flows +package cromwell.docker.registryv2.flows import akka.stream.FanOutShape2 import akka.stream.scaladsl.{GraphDSL, Partition} diff --git a/dockerHashing/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/HttpFlowWithRetry.scala b/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/HttpFlowWithRetry.scala similarity index 97% rename from dockerHashing/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/HttpFlowWithRetry.scala rename to dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/HttpFlowWithRetry.scala index 8bb6c78d2..d4dcba918 100644 --- a/dockerHashing/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/HttpFlowWithRetry.scala +++ b/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/HttpFlowWithRetry.scala @@ -1,4 +1,4 @@ -package cromwell.core.callcaching.docker.registryv2.flows +package cromwell.docker.registryv2.flows import akka.NotUsed import akka.actor.Scheduler @@ -6,8 +6,8 @@ import akka.http.scaladsl.model.{HttpRequest, HttpResponse, StatusCodes} import akka.stream.FanOutShape2 import akka.stream.javadsl.MergePreferred import akka.stream.scaladsl.{Flow, GraphDSL, Partition} -import cromwell.core.callcaching.docker.registryv2.flows.FlowUtils._ -import cromwell.core.callcaching.docker.registryv2.flows.HttpFlowWithRetry._ +import cromwell.docker.registryv2.flows.FlowUtils._ +import cromwell.docker.registryv2.flows.HttpFlowWithRetry._ import cromwell.core.retry.{Backoff, SimpleExponentialBackoff} import scala.concurrent.duration._ diff --git a/dockerHashing/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/dockerhub/DockerHubFlow.scala b/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/dockerhub/DockerHubFlow.scala similarity index 77% rename from dockerHashing/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/dockerhub/DockerHubFlow.scala rename to dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/dockerhub/DockerHubFlow.scala index b1e505b8c..6691dbce4 100644 --- a/dockerHashing/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/dockerhub/DockerHubFlow.scala +++ b/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/dockerhub/DockerHubFlow.scala @@ -1,15 +1,15 @@ -package cromwell.core.callcaching.docker.registryv2.flows.dockerhub +package cromwell.docker.registryv2.flows.dockerhub import akka.actor.Scheduler import akka.http.scaladsl.model.headers.{Authorization, BasicHttpCredentials} import akka.http.scaladsl.model.{HttpMethod, HttpMethods} import akka.stream.ActorMaterializer import cromwell.core.DockerCredentials -import cromwell.core.callcaching.docker.DockerHashActor.DockerHashContext -import cromwell.core.callcaching.docker.DockerImageIdentifierWithoutHash -import cromwell.core.callcaching.docker.registryv2.flows.dockerhub.DockerHubFlow._ -import cromwell.core.callcaching.docker.registryv2.DockerRegistryV2AbstractFlow -import cromwell.core.callcaching.docker.registryv2.DockerRegistryV2AbstractFlow.HttpDockerFlow +import cromwell.docker.DockerHashActor.DockerHashContext +import cromwell.docker.DockerImageIdentifierWithoutHash +import cromwell.docker.registryv2.flows.dockerhub.DockerHubFlow._ +import cromwell.docker.registryv2.DockerRegistryV2AbstractFlow +import cromwell.docker.registryv2.DockerRegistryV2AbstractFlow.HttpDockerFlow import scala.concurrent.ExecutionContext diff --git a/dockerHashing/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/gcr/GcrAbstractFlow.scala b/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/gcr/GcrAbstractFlow.scala similarity index 80% rename from dockerHashing/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/gcr/GcrAbstractFlow.scala rename to dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/gcr/GcrAbstractFlow.scala index 17398d6d8..eb851235e 100644 --- a/dockerHashing/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/gcr/GcrAbstractFlow.scala +++ b/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/gcr/GcrAbstractFlow.scala @@ -1,12 +1,12 @@ -package cromwell.core.callcaching.docker.registryv2.flows.gcr +package cromwell.docker.registryv2.flows.gcr import akka.actor.Scheduler import akka.http.scaladsl.model.headers.{Authorization, OAuth2BearerToken} import akka.stream.ActorMaterializer import com.google.auth.oauth2.OAuth2Credentials -import cromwell.core.callcaching.docker.DockerHashActor.DockerHashContext -import cromwell.core.callcaching.docker.registryv2.DockerRegistryV2AbstractFlow -import cromwell.core.callcaching.docker.registryv2.DockerRegistryV2AbstractFlow.HttpDockerFlow +import cromwell.docker.DockerHashActor.DockerHashContext +import cromwell.docker.registryv2.DockerRegistryV2AbstractFlow +import cromwell.docker.registryv2.DockerRegistryV2AbstractFlow.HttpDockerFlow import scala.concurrent.ExecutionContext import scala.concurrent.duration._ diff --git a/dockerHashing/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/gcr/GcrEuFlow.scala b/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/gcr/GcrEuFlow.scala similarity index 65% rename from dockerHashing/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/gcr/GcrEuFlow.scala rename to dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/gcr/GcrEuFlow.scala index 422afd866..f51c1c0a5 100644 --- a/dockerHashing/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/gcr/GcrEuFlow.scala +++ b/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/gcr/GcrEuFlow.scala @@ -1,8 +1,8 @@ -package cromwell.core.callcaching.docker.registryv2.flows.gcr +package cromwell.docker.registryv2.flows.gcr import akka.actor.Scheduler import akka.stream.ActorMaterializer -import cromwell.core.callcaching.docker.registryv2.DockerRegistryV2AbstractFlow.HttpDockerFlow +import cromwell.docker.registryv2.DockerRegistryV2AbstractFlow.HttpDockerFlow import scala.concurrent.ExecutionContext diff --git a/dockerHashing/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/gcr/GcrFlow.scala b/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/gcr/GcrFlow.scala similarity index 65% rename from dockerHashing/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/gcr/GcrFlow.scala rename to dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/gcr/GcrFlow.scala index c65f661dc..1d4ba17c9 100644 --- a/dockerHashing/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/gcr/GcrFlow.scala +++ b/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/gcr/GcrFlow.scala @@ -1,8 +1,8 @@ -package cromwell.core.callcaching.docker.registryv2.flows.gcr +package cromwell.docker.registryv2.flows.gcr import akka.actor.Scheduler import akka.stream.ActorMaterializer -import cromwell.core.callcaching.docker.registryv2.DockerRegistryV2AbstractFlow.HttpDockerFlow +import cromwell.docker.registryv2.DockerRegistryV2AbstractFlow.HttpDockerFlow import scala.concurrent.ExecutionContext diff --git a/dockerHashing/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/gcr/GcrUsFlow.scala b/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/gcr/GcrUsFlow.scala similarity index 65% rename from dockerHashing/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/gcr/GcrUsFlow.scala rename to dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/gcr/GcrUsFlow.scala index c4253b6bc..0194913fa 100644 --- a/dockerHashing/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/gcr/GcrUsFlow.scala +++ b/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/gcr/GcrUsFlow.scala @@ -1,8 +1,8 @@ -package cromwell.core.callcaching.docker.registryv2.flows.gcr +package cromwell.docker.registryv2.flows.gcr import akka.actor.Scheduler import akka.stream.ActorMaterializer -import cromwell.core.callcaching.docker.registryv2.DockerRegistryV2AbstractFlow.HttpDockerFlow +import cromwell.docker.registryv2.DockerRegistryV2AbstractFlow.HttpDockerFlow import scala.concurrent.ExecutionContext diff --git a/dockerHashing/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/gcr/GoogleFlow.scala b/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/gcr/GoogleFlow.scala similarity index 86% rename from dockerHashing/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/gcr/GoogleFlow.scala rename to dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/gcr/GoogleFlow.scala index c85ecef8b..94c00a549 100644 --- a/dockerHashing/src/main/scala/cromwell/core/callcaching/docker/registryv2/flows/gcr/GoogleFlow.scala +++ b/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/gcr/GoogleFlow.scala @@ -1,11 +1,11 @@ -package cromwell.core.callcaching.docker.registryv2.flows.gcr +package cromwell.docker.registryv2.flows.gcr import akka.actor.Scheduler import akka.stream.scaladsl.{Flow, GraphDSL, Merge, Partition} import akka.stream.{ActorMaterializer, FlowShape, ThrottleMode} -import cromwell.core.callcaching.docker.DockerHashActor.{DockerHashContext, DockerHashResponse, DockerHashUnknownRegistry} -import cromwell.core.callcaching.docker.registryv2.DockerRegistryV2AbstractFlow.HttpDockerFlow -import cromwell.core.callcaching.docker.{DockerFlow, DockerImageIdentifierWithoutHash} +import cromwell.docker.DockerHashActor.{DockerHashContext, DockerHashResponse, DockerHashUnknownRegistry} +import cromwell.docker.registryv2.DockerRegistryV2AbstractFlow.HttpDockerFlow +import cromwell.docker.{DockerFlow, DockerImageIdentifierWithoutHash} import scala.concurrent.ExecutionContext import scala.concurrent.duration._ diff --git a/dockerHashing/src/test/scala/cromwell/docker/DockerEmptyFlowSpec.scala b/dockerHashing/src/test/scala/cromwell/docker/DockerEmptyFlowSpec.scala new file mode 100644 index 000000000..621dffc37 --- /dev/null +++ b/dockerHashing/src/test/scala/cromwell/docker/DockerEmptyFlowSpec.scala @@ -0,0 +1,45 @@ +package cromwell.docker + +import cromwell.docker.DockerHashActor.DockerHashUnknownRegistry +import org.scalatest.{FlatSpecLike, Matchers} + +import scala.concurrent.duration._ + +class DockerEmptyFlowSpec extends DockerFlowSpec("DockerEmptyFlowSpec") with FlatSpecLike with Matchers { + behavior of "An empty docker flow" + + override protected def registryFlows: Seq[DockerFlow] = Seq() + + it should "send an unrecognized host message back for a public docker hash" in { + dockerActor ! makeRequest("ubuntu:latest") + + expectMsgClass(5.seconds, classOf[DockerHashUnknownRegistry]) + } + + it should "send an unrecognized host message back for a public docker on gcr" in { + dockerActor ! makeRequest("gcr.io/google-containers/alpine-with-bash:1.0") + + expectMsgClass(5.seconds, classOf[DockerHashUnknownRegistry]) + } + + it should "send an unrecognized host message back if the image does not exist" in { + val notFound = makeRequest("ubuntu:nonexistingtag") + dockerActor ! notFound + + expectMsgClass(5.seconds, classOf[DockerHashUnknownRegistry]) + } + + it should "send an unrecognized host message back if the user doesn't have permission on this image" in { + val unauthorized = makeRequest("tjeandet/sinatra:v1") + dockerActor ! unauthorized + + expectMsgClass(5.seconds, classOf[DockerHashUnknownRegistry]) + } + + it should "send an unrecognized host message back for an unrecognized host" in { + val unauthorized = makeRequest("unknown.io/image:v1") + dockerActor ! unauthorized + + expectMsgClass(5.seconds, classOf[DockerHashUnknownRegistry]) + } +} diff --git a/dockerHashing/src/test/scala/cromwell/docker/DockerFlowSpec.scala b/dockerHashing/src/test/scala/cromwell/docker/DockerFlowSpec.scala new file mode 100644 index 000000000..afe9cd556 --- /dev/null +++ b/dockerHashing/src/test/scala/cromwell/docker/DockerFlowSpec.scala @@ -0,0 +1,36 @@ +package cromwell.docker + +import akka.http.scaladsl.Http +import akka.stream.ActorMaterializer +import akka.testkit.ImplicitSender +import cromwell.core.TestKitSuite +import cromwell.docker.DockerHashActor._ +import cromwell.docker.registryv2.flows.HttpFlowWithRetry.ContextWithRequest + +import scala.concurrent.duration._ + +abstract class DockerFlowSpec(actorSystemName: String) extends TestKitSuite(actorSystemName) with ImplicitSender { + implicit val materializer = ActorMaterializer() + implicit val ex = system.dispatcher + implicit val scheduler = system.scheduler + + lazy val httpPool = Http().superPool[ContextWithRequest[DockerHashContext]]() + + protected def registryFlows: Seq[DockerFlow] + + // Disable cache by setting a cache size of 0 - A separate test tests the cache + lazy val dockerActor = system.actorOf(DockerHashActor.props(registryFlows, 1000, 20.minutes, 0)(materializer)) + + def dockerImage(string: String) = DockerImageIdentifier.fromString(string).get.asInstanceOf[DockerImageIdentifierWithoutHash] + + def makeRequest(string: String) = { + DockerHashRequest(dockerImage(string)) + } + + override protected def afterAll() = { + system.stop(dockerActor) + Http().shutdownAllConnectionPools() + materializer.shutdown() + super.afterAll() + } +} diff --git a/dockerHashing/src/test/scala/cromwell/core/callcaching/docker/DockerHashActorSpec.scala b/dockerHashing/src/test/scala/cromwell/docker/DockerHashActorSpec.scala similarity index 69% rename from dockerHashing/src/test/scala/cromwell/core/callcaching/docker/DockerHashActorSpec.scala rename to dockerHashing/src/test/scala/cromwell/docker/DockerHashActorSpec.scala index 076f727de..121103bf9 100644 --- a/dockerHashing/src/test/scala/cromwell/core/callcaching/docker/DockerHashActorSpec.scala +++ b/dockerHashing/src/test/scala/cromwell/docker/DockerHashActorSpec.scala @@ -1,45 +1,19 @@ -package cromwell.core.callcaching.docker +package cromwell.docker -import akka.http.scaladsl.Http -import akka.stream.ActorMaterializer -import akka.testkit.ImplicitSender import cromwell.core.Tags.IntegrationTest -import cromwell.core.TestKitSuite -import cromwell.core.callcaching.docker.DockerHashActor._ -import cromwell.core.callcaching.docker.registryv2.flows.HttpFlowWithRetry.ContextWithRequest -import cromwell.core.callcaching.docker.registryv2.flows.dockerhub.DockerHubFlow -import cromwell.core.callcaching.docker.registryv2.flows.gcr.GoogleFlow +import cromwell.docker.DockerHashActor._ +import cromwell.docker.registryv2.flows.dockerhub.DockerHubFlow +import cromwell.docker.registryv2.flows.gcr.GoogleFlow import org.scalatest.{FlatSpecLike, Matchers} import scala.concurrent.duration._ import scala.language.postfixOps -class DockerHashActorSpec extends TestKitSuite with FlatSpecLike with Matchers with ImplicitSender { - +class DockerHashActorSpec extends DockerFlowSpec("DockerHashActorSpec") with FlatSpecLike with Matchers { behavior of "DockerRegistryActor" - - implicit val materializer = ActorMaterializer() - implicit val ex = system.dispatcher - implicit val scheduler = system.scheduler - - val httpPool = Http().superPool[ContextWithRequest[DockerHashContext]]() - val registryFlows = Seq(new DockerHubFlow(httpPool), new GoogleFlow(httpPool, 50000)) - // Disable cache by setting a cache size of 0 - A separate test tests the cache - val dockerActor = system.actorOf(DockerHashActor.props(registryFlows, 1000, 20 minutes, 0)(materializer)) - - private def dockerImage(string: String) = DockerImageIdentifier.fromString(string).get.asInstanceOf[DockerImageIdentifierWithoutHash] - - private def makeRequest(string: String) = { - DockerHashRequest(dockerImage(string)) - } - override protected def afterAll() = { - system.stop(dockerActor) - Http().shutdownAllConnectionPools() - materializer.shutdown() - super.afterAll() - } - + override protected lazy val registryFlows = Seq(new DockerHubFlow(httpPool), new GoogleFlow(httpPool, 50000)) + it should "retrieve a public docker hash" taggedAs IntegrationTest in { dockerActor ! makeRequest("ubuntu:latest") diff --git a/dockerHashing/src/test/scala/cromwell/core/callcaching/docker/DockerHashMocks.scala b/dockerHashing/src/test/scala/cromwell/docker/DockerHashMocks.scala similarity index 89% rename from dockerHashing/src/test/scala/cromwell/core/callcaching/docker/DockerHashMocks.scala rename to dockerHashing/src/test/scala/cromwell/docker/DockerHashMocks.scala index 3f1fd754a..39d262730 100644 --- a/dockerHashing/src/test/scala/cromwell/core/callcaching/docker/DockerHashMocks.scala +++ b/dockerHashing/src/test/scala/cromwell/docker/DockerHashMocks.scala @@ -1,9 +1,9 @@ -package cromwell.core.callcaching.docker +package cromwell.docker import akka.http.scaladsl.model._ import akka.stream.scaladsl.Flow -import cromwell.core.callcaching.docker.DockerHashActor.{DockerHashContext, DockerHashResponse} -import cromwell.core.callcaching.docker.registryv2.flows.HttpFlowWithRetry.ContextWithRequest +import cromwell.docker.DockerHashActor.{DockerHashContext, DockerHashResponse} +import cromwell.docker.registryv2.flows.HttpFlowWithRetry.ContextWithRequest import scala.util.Try diff --git a/dockerHashing/src/test/scala/cromwell/core/callcaching/docker/DockerImageIdentifierSpec.scala b/dockerHashing/src/test/scala/cromwell/docker/DockerImageIdentifierSpec.scala similarity index 98% rename from dockerHashing/src/test/scala/cromwell/core/callcaching/docker/DockerImageIdentifierSpec.scala rename to dockerHashing/src/test/scala/cromwell/docker/DockerImageIdentifierSpec.scala index 9d0fb6218..dfce2d596 100644 --- a/dockerHashing/src/test/scala/cromwell/core/callcaching/docker/DockerImageIdentifierSpec.scala +++ b/dockerHashing/src/test/scala/cromwell/docker/DockerImageIdentifierSpec.scala @@ -1,4 +1,4 @@ -package cromwell.core.callcaching.docker +package cromwell.docker import org.scalatest.prop.TableDrivenPropertyChecks import org.scalatest.{FlatSpec, Matchers} diff --git a/dockerHashing/src/test/scala/cromwell/docker/local/DockerCliClientSpec.scala b/dockerHashing/src/test/scala/cromwell/docker/local/DockerCliClientSpec.scala new file mode 100644 index 000000000..a9ce1b9fa --- /dev/null +++ b/dockerHashing/src/test/scala/cromwell/docker/local/DockerCliClientSpec.scala @@ -0,0 +1,54 @@ +package cromwell.docker.local + +import org.scalatest.prop.TableDrivenPropertyChecks +import org.scalatest.{FlatSpecLike, Matchers} + +import scala.util.{Failure, Success} + +class DockerCliClientSpec extends FlatSpecLike with Matchers with TableDrivenPropertyChecks { + behavior of "DockerCliClient" + + private val lookupSuccessStdout = Seq( + "\t\t", + "fauxbuntu\tlatest\tsha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "fauxbuntu\tmytag\tsha256:00001111222233334444555566667777888899990000aaaabbbbccccddddeeee") + + private val lookupSuccessHashes = Table( + ("dockerCliKey", "hashValue"), + (DockerCliKey("fauxbuntu", "latest"), "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"), + (DockerCliKey("fauxbuntu", "mytag"), "00001111222233334444555566667777888899990000aaaabbbbccccddddeeee")) + + forAll(lookupSuccessHashes) { (dockerCliKey, hashValue) => + it should s"successfully lookup simulated hash for ${dockerCliKey.fullName}" in { + val client = new TestDockerCliClient(DockerCliResult(0, lookupSuccessStdout, Nil)) + val response = client.lookupHash(dockerCliKey) + response should be(Success(Option(s"sha256:$hashValue"))) + } + } + + val lookupFailStderr = Seq("Error response from daemon: Bad response from Docker engine") + + it should "return not found for simulated hash fauxbuntu:unknowntag" in { + val client = new TestDockerCliClient(DockerCliResult(0, lookupSuccessStdout, Nil)) + val dockerCliKey = DockerCliKey("fauxbuntu", "unknowntag") + val response = client.lookupHash(dockerCliKey) + response should be(Success(None)) + } + + it should "return expected error messages" in { + val client = new TestDockerCliClient(DockerCliResult(1, Nil, lookupFailStderr)) + val dockerCliKey = DockerCliKey("fauxbuntu", "dockernotrunning") + val response = client.lookupHash(dockerCliKey) + response should be(a[Failure[_]]) + val exception = response.asInstanceOf[Failure[_]].exception + exception.getMessage should be( + """|Error running: docker images --digests --format {{printf "%s\t%s\t%s" .Repository .Tag .Digest}} + |Exit code: 1 + |Error response from daemon: Bad response from Docker engine + |""".stripMargin) + } +} + +class TestDockerCliClient(dockerCliResult: DockerCliResult) extends DockerCliClient { + override private[local] def run(cmd: Seq[String]) = dockerCliResult +} diff --git a/dockerHashing/src/test/scala/cromwell/docker/local/DockerCliFlowSpec.scala b/dockerHashing/src/test/scala/cromwell/docker/local/DockerCliFlowSpec.scala new file mode 100644 index 000000000..913d326bb --- /dev/null +++ b/dockerHashing/src/test/scala/cromwell/docker/local/DockerCliFlowSpec.scala @@ -0,0 +1,55 @@ +package cromwell.docker.local + +import cromwell.core.Tags.IntegrationTest +import cromwell.docker.DockerHashActor.{DockerHashNotFound, DockerHashResponseSuccess} +import cromwell.docker.{DockerFlow, DockerFlowSpec, DockerHashResult} +import org.scalatest.{FlatSpecLike, Matchers} + +import scala.concurrent.duration._ + +class DockerCliFlowSpec extends DockerFlowSpec("DockerCliFlowSpec") with FlatSpecLike with Matchers { + behavior of "DockerCliFlow" + + override protected def registryFlows: Seq[DockerFlow] = Seq(new DockerCliFlow) + + it should "retrieve a public docker hash" taggedAs IntegrationTest in { + dockerActor ! makeRequest("ubuntu:latest") + + expectMsgPF(30.seconds) { + case DockerHashResponseSuccess(DockerHashResult(alg, hash), _) => + alg shouldBe "sha256" + hash should not be empty + } + } + + it should "retrieve a public docker hash on gcr" taggedAs IntegrationTest in { + dockerActor ! makeRequest("gcr.io/google-containers/alpine-with-bash:1.0") + + expectMsgPF(30.seconds) { + case DockerHashResponseSuccess(DockerHashResult(alg, hash), _) => + alg shouldBe "sha256" + hash should not be empty + } + } + + it should "send image not found message back if the image does not exist" taggedAs IntegrationTest in { + val notFound = makeRequest("ubuntu:nonexistingtag") + dockerActor ! notFound + + expectMsgClass(5.seconds, classOf[DockerHashNotFound]) + } + + it should "send image not found if the user doesn't have permission on this image" taggedAs IntegrationTest in { + val unauthorized = makeRequest("tjeandet/sinatra:v1") + dockerActor ! unauthorized + + expectMsgClass(5.seconds, classOf[DockerHashNotFound]) + } + + it should "send image not found for an unrecognized host" taggedAs IntegrationTest in { + val unauthorized = makeRequest("unknown.io/image:v1") + dockerActor ! unauthorized + + expectMsgClass(5.seconds, classOf[DockerHashNotFound]) + } +} diff --git a/dockerHashing/src/test/scala/cromwell/docker/local/DockerCliTimeoutFlowSpec.scala b/dockerHashing/src/test/scala/cromwell/docker/local/DockerCliTimeoutFlowSpec.scala new file mode 100644 index 000000000..383b06966 --- /dev/null +++ b/dockerHashing/src/test/scala/cromwell/docker/local/DockerCliTimeoutFlowSpec.scala @@ -0,0 +1,81 @@ +package cromwell.docker.local + +import cromwell.core.Tags.IntegrationTest +import cromwell.docker.DockerHashActor.DockerHashFailedResponse +import cromwell.docker.{DockerFlow, DockerFlowSpec} +import org.scalatest.{FlatSpecLike, Matchers} + +import scala.concurrent.TimeoutException +import scala.concurrent.duration._ + +class DockerCliTimeoutFlowSpec extends DockerFlowSpec("DockerCliTimeoutFlowSpec") with FlatSpecLike with Matchers { + behavior of "A DockerCliFlow that times out" + + override protected def registryFlows: Seq[DockerFlow] = Seq(new DockerCliFlow { + override lazy val firstLookupTimeout = 0.seconds + }) + + it should "timeout retrieving a public docker hash" taggedAs IntegrationTest in { + dockerActor ! makeRequest("ubuntu:latest") + + expectMsgPF(5.seconds) { + case DockerHashFailedResponse(exception: TimeoutException, _) => + exception.getMessage should be( + """|Timeout while looking up hash of ubuntu:latest. + |Ensure that docker is running correctly. + |""".stripMargin) + } + } + + it should "timeout retrieving a public docker hash on gcr" taggedAs IntegrationTest in { + dockerActor ! makeRequest("gcr.io/google-containers/alpine-with-bash:1.0") + + expectMsgPF(5.seconds) { + case DockerHashFailedResponse(exception: TimeoutException, _) => + exception.getMessage should be( + """|Timeout while looking up hash of gcr.io/google-containers/alpine-with-bash:1.0. + |Ensure that docker is running correctly. + |""".stripMargin) + } + + } + + it should "timeout retrieving an image that does not exist" taggedAs IntegrationTest in { + val notFound = makeRequest("ubuntu:nonexistingtag") + dockerActor ! notFound + + expectMsgPF(5.seconds) { + case DockerHashFailedResponse(exception: TimeoutException, _) => + exception.getMessage should be( + """|Timeout while looking up hash of ubuntu:nonexistingtag. + |Ensure that docker is running correctly. + |""".stripMargin) + } + } + + it should "timeout retrieving an image if the user doesn't have permission" taggedAs IntegrationTest in { + val unauthorized = makeRequest("tjeandet/sinatra:v1") + dockerActor ! unauthorized + + expectMsgPF(5.seconds) { + case DockerHashFailedResponse(exception: TimeoutException, _) => + exception.getMessage should be( + """|Timeout while looking up hash of tjeandet/sinatra:v1. + |Ensure that docker is running correctly. + |""".stripMargin) + } + } + + it should "timeout retrieving an image for an unrecognized host" taggedAs IntegrationTest in { + val unauthorized = makeRequest("unknown.io/image:v1") + dockerActor ! unauthorized + + expectMsgPF(5.seconds) { + case DockerHashFailedResponse(exception: TimeoutException, _) => + exception.getMessage should be( + """|Timeout while looking up hash of unknown.io/library/image:v1. + |Ensure that docker is running correctly. + |""".stripMargin) + } + } +} diff --git a/dockerHashing/src/test/scala/cromwell/core/callcaching/docker/registryv2/flows/HttpFlowWithRetrySpec.scala b/dockerHashing/src/test/scala/cromwell/docker/registryv2/flows/HttpFlowWithRetrySpec.scala similarity index 98% rename from dockerHashing/src/test/scala/cromwell/core/callcaching/docker/registryv2/flows/HttpFlowWithRetrySpec.scala rename to dockerHashing/src/test/scala/cromwell/docker/registryv2/flows/HttpFlowWithRetrySpec.scala index e1b21ff79..6807de46c 100644 --- a/dockerHashing/src/test/scala/cromwell/core/callcaching/docker/registryv2/flows/HttpFlowWithRetrySpec.scala +++ b/dockerHashing/src/test/scala/cromwell/docker/registryv2/flows/HttpFlowWithRetrySpec.scala @@ -1,4 +1,4 @@ -package cromwell.core.callcaching.docker.registryv2.flows +package cromwell.docker.registryv2.flows import akka.NotUsed import akka.http.scaladsl.model.{HttpRequest, HttpResponse, StatusCodes} @@ -6,7 +6,7 @@ import akka.stream._ import akka.stream.scaladsl.{GraphDSL, RunnableGraph, Sink, Source} import akka.testkit.{ImplicitSender, TestProbe} import cromwell.core.TestKitSuite -import cromwell.core.callcaching.docker.{HttpMock, MockHttpResponse} +import cromwell.docker.{HttpMock, MockHttpResponse} import org.scalatest.{FlatSpecLike, Matchers} import scala.concurrent.duration._ diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/preparation/JobPreparationActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/preparation/JobPreparationActor.scala index cb290ca1a..ff4595cd1 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/preparation/JobPreparationActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/preparation/JobPreparationActor.scala @@ -2,12 +2,12 @@ package cromwell.engine.workflow.lifecycle.execution.preparation import akka.actor.{ActorRef, FSM, Props} import cromwell.backend._ -import cromwell.backend.validation.RuntimeAttributesKeys +import cromwell.backend.validation.{DockerValidation, RuntimeAttributesKeys} import cromwell.core.Dispatcher.EngineDispatcher import cromwell.core.callcaching._ -import cromwell.core.callcaching.docker.DockerHashActor.{DockerHashFailureResponse, DockerHashResponseSuccess} -import cromwell.core.callcaching.docker._ import cromwell.core.logging.WorkflowLogging +import cromwell.docker.DockerHashActor.{DockerHashFailureResponse, DockerHashResponseSuccess} +import cromwell.docker._ import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActorData import cromwell.engine.workflow.lifecycle.execution.preparation.CallPreparation._ import cromwell.engine.workflow.lifecycle.execution.preparation.JobPreparationActor.{DockerNoResponseTimeout, _} @@ -41,6 +41,8 @@ class JobPreparationActor(executionData: WorkflowExecutionActorData, private lazy val workflowDescriptor = executionData.workflowDescriptor private[preparation] lazy val expressionLanguageFunctions = factory.expressionLanguageFunctions(workflowDescriptor.backendDescriptor, jobKey, initializationData) private[preparation] lazy val dockerHashCredentials = factory.dockerHashCredentials(initializationData) + private[preparation] lazy val runtimeAttributeDefinitions = factory.runtimeAttributeDefinitions(initializationData) + private[preparation] lazy val hasDockerDefinition = runtimeAttributeDefinitions.exists(_.name == DockerValidation.instance.key) startWith(Idle, JobPreparationActorNoData) @@ -111,9 +113,10 @@ class JobPreparationActor(executionData: WorkflowExecutionActorData, } def handleDockerValue(value: String) = DockerImageIdentifier.fromString(value) match { - case Success(dockerImageId: DockerImageIdentifierWithoutHash) => sendDockerRequest(dockerImageId) - case Success(dockerImageIdWithHash: DockerImageIdentifierWithHash) => - // If the docker value already has a hash - we're ok for call caching + case Success(dockerImageId: DockerImageIdentifierWithoutHash) if hasDockerDefinition => sendDockerRequest(dockerImageId) + case Success(_) => + // If the docker value already has a hash, or the backend doesn't support docker - + // no need to lookup and we're ok for call caching val response = prepareBackendDescriptor(inputs, attributes, CallCachingEligible, kvStoreLookupResults.unscoped) sendResponseAndStop(response) case Failure(failure) => sendFailureAndStop(failure) @@ -167,7 +170,7 @@ class JobPreparationActor(executionData: WorkflowExecutionActorData, private [preparation] def prepareRuntimeAttributes(inputEvaluation: Map[Declaration, WdlValue]): Try[Map[LocallyQualifiedName, WdlValue]] = { import RuntimeAttributeDefinition.{addDefaultsToAttributes, evaluateRuntimeAttributes} - val curriedAddDefaultsToAttributes = addDefaultsToAttributes(factory.runtimeAttributeDefinitions(initializationData), workflowDescriptor.backendDescriptor.workflowOptions) _ + val curriedAddDefaultsToAttributes = addDefaultsToAttributes(runtimeAttributeDefinitions, workflowDescriptor.backendDescriptor.workflowOptions) _ for { unevaluatedRuntimeAttributes <- Try(jobKey.call.task.runtimeAttributes) diff --git a/engine/src/main/scala/cromwell/server/CromwellRootActor.scala b/engine/src/main/scala/cromwell/server/CromwellRootActor.scala index 3263e5d01..10a531e0c 100644 --- a/engine/src/main/scala/cromwell/server/CromwellRootActor.scala +++ b/engine/src/main/scala/cromwell/server/CromwellRootActor.scala @@ -7,13 +7,14 @@ import akka.http.scaladsl.Http import akka.routing.RoundRobinPool import akka.stream.ActorMaterializer import com.typesafe.config.ConfigFactory -import cromwell.core.Dispatcher +import cromwell.core.{Dispatcher, DockerConfiguration, DockerLocalLookup, DockerRemoteLookup} import cromwell.core.actor.StreamActorHelper.ActorRestartException -import cromwell.core.callcaching.docker.DockerHashActor -import cromwell.core.callcaching.docker.DockerHashActor.DockerHashContext -import cromwell.core.callcaching.docker.registryv2.flows.HttpFlowWithRetry.ContextWithRequest -import cromwell.core.callcaching.docker.registryv2.flows.dockerhub.DockerHubFlow -import cromwell.core.callcaching.docker.registryv2.flows.gcr.GoogleFlow +import cromwell.docker.DockerHashActor +import cromwell.docker.DockerHashActor.DockerHashContext +import cromwell.docker.local.DockerCliFlow +import cromwell.docker.registryv2.flows.HttpFlowWithRetry.ContextWithRequest +import cromwell.docker.registryv2.flows.dockerhub.DockerHubFlow +import cromwell.docker.registryv2.flows.gcr.GoogleFlow import cromwell.core.io.Throttle import cromwell.engine.backend.{BackendSingletonCollection, CromwellBackends} import cromwell.engine.io.IoActor @@ -82,17 +83,19 @@ import scala.language.postfixOps // Docker Actor lazy val ioEc = context.system.dispatchers.lookup(Dispatcher.IoDispatcher) - lazy val gcrQueriesPer100Sec = config.getAs[Int]("docker.gcr-api-queries-per-100-seconds") getOrElse 1000 - lazy val dockerCacheEntryTTL = config.as[Option[FiniteDuration]]("docker.cache-entry-ttl").getOrElse(DefaultCacheTTL) - lazy val dockerCacheSize = config.getAs[Long]("docker.cache-size") getOrElse 200L + lazy val dockerConf = DockerConfiguration.instance // Sets the number of requests that the docker actor will accept before it starts backpressuring (modulo the number of in flight requests) lazy val dockerActorQueueSize = 500 lazy val dockerHttpPool = Http().superPool[ContextWithRequest[DockerHashContext]]() - lazy val googleFlow = new GoogleFlow(dockerHttpPool, gcrQueriesPer100Sec)(ioEc, materializer, system.scheduler) + lazy val googleFlow = new GoogleFlow(dockerHttpPool, dockerConf.gcrApiQueriesPer100Seconds)(ioEc, materializer, system.scheduler) lazy val dockerHubFlow = new DockerHubFlow(dockerHttpPool)(ioEc, materializer, system.scheduler) - lazy val dockerFlows = Seq(dockerHubFlow, googleFlow) - lazy val dockerHashActor = context.actorOf(DockerHashActor.props(dockerFlows, dockerActorQueueSize, dockerCacheEntryTTL, dockerCacheSize)(materializer).withDispatcher(Dispatcher.IoDispatcher)) + lazy val dockerCliFlow = new DockerCliFlow()(ioEc, materializer, system.scheduler) + lazy val dockerFlows = dockerConf.method match { + case DockerLocalLookup => Seq(dockerCliFlow) + case DockerRemoteLookup => Seq(dockerHubFlow, googleFlow) + } + lazy val dockerHashActor = context.actorOf(DockerHashActor.props(dockerFlows, dockerActorQueueSize, dockerConf.cacheEntryTtl, dockerConf.cacheSize)(materializer).withDispatcher(Dispatcher.IoDispatcher)) lazy val backendSingletons = CromwellBackends.instance.get.backendLifecycleActorFactories map { case (name, factory) => name -> (factory.backendSingletonActorProps map context.actorOf) @@ -118,8 +121,8 @@ import scala.language.postfixOps * of Cromwell by passing a Throwable to the guardian. */ override val supervisorStrategy = OneForOneStrategy() { - case actorInitializationException: ActorInitializationException => Escalate - case restart: ActorRestartException => Restart + case _: ActorInitializationException => Escalate + case _: ActorRestartException => Restart case t => super.supervisorStrategy.decider.applyOrElse(t, (_: Any) => Escalate) } } diff --git a/engine/src/test/scala/cromwell/CromwellTestKitSpec.scala b/engine/src/test/scala/cromwell/CromwellTestKitSpec.scala index d97e5038e..81ad65fcb 100644 --- a/engine/src/test/scala/cromwell/CromwellTestKitSpec.scala +++ b/engine/src/test/scala/cromwell/CromwellTestKitSpec.scala @@ -11,8 +11,8 @@ import com.typesafe.config.{Config, ConfigFactory} import cromwell.CromwellTestKitSpec._ import cromwell.backend._ import cromwell.core._ -import cromwell.core.callcaching.docker.DockerHashActor.DockerHashResponseSuccess -import cromwell.core.callcaching.docker.{DockerHashRequest, DockerHashResult} +import cromwell.docker.DockerHashActor.DockerHashResponseSuccess +import cromwell.docker.{DockerHashRequest, DockerHashResult} import cromwell.core.path.BetterFileMethods.Cmds import cromwell.core.path.DefaultPathBuilder import cromwell.engine.backend.BackendConfigurationEntry diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/preparation/JobPreparationActorSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/preparation/JobPreparationActorSpec.scala index 3c1d511c2..efb38101d 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/preparation/JobPreparationActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/preparation/JobPreparationActorSpec.scala @@ -2,8 +2,8 @@ package cromwell.engine.workflow.lifecycle.execution.preparation import akka.testkit.{ImplicitSender, TestActorRef} import cromwell.core.actor.StreamIntegration.BackPressure -import cromwell.core.callcaching.docker.DockerHashActor.{DockerHashFailedResponse, DockerHashResponseSuccess} -import cromwell.core.callcaching.docker.{DockerHashRequest, DockerHashResult, DockerImageIdentifier, DockerImageIdentifierWithoutHash} +import cromwell.docker.DockerHashActor.{DockerHashFailedResponse, DockerHashResponseSuccess} +import cromwell.docker.{DockerHashRequest, DockerHashResult, DockerImageIdentifier, DockerImageIdentifierWithoutHash} import cromwell.core.callcaching.{CallCachingEligible, CallCachingIneligible} import cromwell.core.{LocallyQualifiedName, TestKitSuite} import cromwell.engine.workflow.lifecycle.execution.preparation.CallPreparation.{BackendJobPreparationSucceeded, CallPreparationFailed, Start} diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/preparation/JobPreparationTestHelper.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/preparation/JobPreparationTestHelper.scala index 3c0ff7499..a0980cad1 100644 --- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/preparation/JobPreparationTestHelper.scala +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/preparation/JobPreparationTestHelper.scala @@ -75,6 +75,7 @@ private[preparation] class TestJobPreparationActor(kvStoreKeysForPrefetch: List[ override private[preparation] lazy val expressionLanguageFunctions = NoFunctions override private[preparation] lazy val dockerHashCredentials = dockerHashCredentialsInput override private[preparation] lazy val noResponseTimeout = dockerNoResponseTimeoutInput + override private[preparation] lazy val hasDockerDefinition = true override protected def backpressureTimeout: FiniteDuration = backpressureWaitTimeInput override def scopedKey(key: String) = scopedKeyMaker.apply(key) diff --git a/src/bin/travis/resources/local_centaur.conf b/src/bin/travis/resources/local_centaur.conf index 848780e82..ec15e2ad4 100644 --- a/src/bin/travis/resources/local_centaur.conf +++ b/src/bin/travis/resources/local_centaur.conf @@ -17,6 +17,43 @@ call-caching { enabled = true } +backend.providers { + LocalNoDocker { + actor-factory = "cromwell.backend.impl.sfs.config.ConfigBackendLifecycleActorFactory" + config { + run-in-background = true + runtime-attributes = "" + submit = "/bin/bash ${script}" + root: "cromwell-executions" + + filesystems { + local { + localization: [ + "soft-link", "hard-link", "copy" + ] + + caching { + duplication-strategy: [ + "soft-link" + ] + + # Possible values: file, path + # "file" will compute an md5 hash of the file content. + # "path" will compute an md5 hash of the file path. This strategy will only be effective if the duplication-strategy (above) is set to "soft-link", + # in order to allow for the original file path to be hashed. + hashing-strategy: "path" + + # When true, will check if a sibling file with the same name and the .md5 extension exists, and if it does, use the content of this file as a hash. + # If false or the md5 does not exist, will proceed with the above-defined hashing strategy. + check-sibling-md5: false + } + } + } + } + } +} + + backend.providers.Local.config.filesystems.local.caching.duplication-strategy = ["copy"] backend.providers.Local.config.filesystems.local.localization = ["soft-link", "copy"] backend.providers.Local.config.concurrent-job-limit = 20 diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesConfiguration.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesConfiguration.scala index 636c0dad1..f92a5f818 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesConfiguration.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesConfiguration.scala @@ -2,7 +2,7 @@ package cromwell.backend.impl.jes import cromwell.backend.BackendConfigurationDescriptor import cromwell.backend.impl.jes.authentication.JesDockerCredentials -import cromwell.core.DockerConfiguration +import cromwell.core.BackendDockerConfiguration import cromwell.filesystems.gcs.{GcsPathBuilderFactory, GoogleConfiguration} class JesConfiguration(val configurationDescriptor: BackendConfigurationDescriptor) { @@ -16,7 +16,7 @@ class JesConfiguration(val configurationDescriptor: BackendConfigurationDescript val jesComputeServiceAccount = jesAttributes.computeServiceAccount val gcsPathBuilderFactory = GcsPathBuilderFactory(jesAuths.gcs, googleConfig.applicationName) val genomicsFactory = GenomicsFactory(googleConfig.applicationName, jesAuths.genomics, jesAttributes.endpointUrl) - val dockerCredentials = DockerConfiguration.build(configurationDescriptor.backendConfig).dockerCredentials map JesDockerCredentials.apply + val dockerCredentials = BackendDockerConfiguration.build(configurationDescriptor.backendConfig).dockerCredentials map JesDockerCredentials.apply val needAuthFileUpload = jesAuths.gcs.requiresAuthFile || dockerCredentials.isDefined val qps = jesAttributes.qps } From 89d0b9d5f9e36747309c0befa85be456194aa25d Mon Sep 17 00:00:00 2001 From: "Francisco M. Casares" Date: Tue, 18 Apr 2017 10:29:24 -0700 Subject: [PATCH 007/134] Initial implementation of HtCondor backend using configurable backend. (#2141) Fixed documentation on Native specs. Generalized memory validation in order to support disk validation based on information unit. Prettyfing bash scripts. Updating README. Changes after rebase. --- README.md | 146 ++++--- .../backend/validation/MemoryValidation.scala | 19 +- .../validation/RuntimeAttributesValidation.scala | 2 +- .../RuntimeAttributesValidationSpec.scala | 8 +- build.sbt | 8 - core/src/main/resources/reference.conf | 90 ++-- project/Dependencies.scala | 5 - project/Settings.scala | 5 - .../impl/htcondor/HtCondorBackendFactory.scala | 74 ---- .../htcondor/HtCondorInitializationActor.scala | 75 ---- .../impl/htcondor/HtCondorJobExecutionActor.scala | 379 ----------------- .../impl/htcondor/HtCondorRuntimeAttributes.scala | 124 ------ .../backend/impl/htcondor/HtCondorWrapper.scala | 153 ------- .../backend/impl/htcondor/caching/CacheActor.scala | 51 --- .../impl/htcondor/caching/CacheActorFactory.scala | 9 - .../CachedResultAlreadyExistException.scala | 5 - .../localization/CachedResultLocalization.scala | 38 -- .../caching/model/CachedExecutionResult.scala | 6 - .../caching/provider/mongodb/MongoCacheActor.scala | 88 ---- .../provider/mongodb/MongoCacheActorFactory.scala | 21 - .../mongodb/model/MongoCachedExecutionResult.scala | 24 -- .../provider/mongodb/serialization/KryoSerDe.scala | 55 --- .../provider/mongodb/serialization/SerDe.scala | 25 -- .../impl/htcondor/HtCondorCommandSpec.scala | 28 -- .../htcondor/HtCondorInitializationActorSpec.scala | 53 --- .../htcondor/HtCondorJobExecutionActorSpec.scala | 472 --------------------- .../htcondor/HtCondorRuntimeAttributesSpec.scala | 304 ------------- .../CachedResultLocalizationSpec.scala | 63 --- .../provider/mongodb/MongoCacheActorSpec.scala | 104 ----- .../backend/impl/jes/JesRuntimeAttributes.scala | 4 +- .../backend/impl/sfs/config/ConfigConstants.scala | 3 +- .../impl/sfs/config/DeclarationValidation.scala | 7 +- .../sfs/config/MemoryDeclarationValidation.scala | 23 +- .../config/MemoryDeclarationValidationSpec.scala | 10 +- .../backend/impl/tes/TesRuntimeAttributes.scala | 4 +- 35 files changed, 193 insertions(+), 2292 deletions(-) delete mode 100644 supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorBackendFactory.scala delete mode 100644 supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorInitializationActor.scala delete mode 100644 supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala delete mode 100644 supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributes.scala delete mode 100644 supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorWrapper.scala delete mode 100644 supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/CacheActor.scala delete mode 100644 supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/CacheActorFactory.scala delete mode 100644 supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/exception/CachedResultAlreadyExistException.scala delete mode 100644 supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/localization/CachedResultLocalization.scala delete mode 100644 supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/model/CachedExecutionResult.scala delete mode 100644 supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/MongoCacheActor.scala delete mode 100644 supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/MongoCacheActorFactory.scala delete mode 100644 supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/model/MongoCachedExecutionResult.scala delete mode 100644 supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/serialization/KryoSerDe.scala delete mode 100644 supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/serialization/SerDe.scala delete mode 100644 supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorCommandSpec.scala delete mode 100644 supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorInitializationActorSpec.scala delete mode 100644 supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActorSpec.scala delete mode 100644 supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributesSpec.scala delete mode 100644 supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/caching/localization/CachedResultLocalizationSpec.scala delete mode 100644 supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/MongoCacheActorSpec.scala diff --git a/README.md b/README.md index f272cef09..2970c3d97 100644 --- a/README.md +++ b/README.md @@ -1262,78 +1262,117 @@ The `job-id-regex` should contain one capture group while matching against the w Allows to execute jobs using HTCondor which is a specialized workload management system for compute-intensive jobs created by the Center for High Throughput Computing in the Department of Computer Sciences at the University of Wisconsin-Madison (UW-Madison). -This backend creates six files in the `` (see previous section): +The backend is specified via the actor factory `ConfigBackendLifecycleActorFactory`: -* `script` - A shell script of the job to be run. This contains the user's command from the `command` section of the WDL code. -* `stdout` - The standard output of the process -* `stderr` - The standard error of the process -* `submitfile` - A submit file that HtCondor understands in order to submit a job -* `submitfile.stdout` - The standard output of the submit file -* `submitfile.stderr` - The standard error of the submit file +``` +backend { + providers { + HtCondor { + config { + actor-factory = "cromwell.backend.impl.sfs.config.ConfigBackendLifecycleActorFactory" + # ... other configuration + } + } + } +} +``` -The `script` file contains: +This backend makes the same assumption about the filesystem that the local backend does: the Cromwell process and the jobs both have read/write access to the CWD of the job. + +The CWD will contain a `script.sh` file which will contain the same contents as the Local backend: ``` +#!/bin/sh cd echo $? > rc ``` -The `submitfile` file contains: +The job is launched with a configurable script command such as: ``` -executable=cromwell-executions/test/e950e07d-4132-4fe0-8d86-ab6925dd94ad/call-merge_files/script -output=cromwell-executions/test/e950e07d-4132-4fe0-8d86-ab6925dd94ad/call-merge_files/stdout -error=cromwell-executions/test/e950e07d-4132-4fe0-8d86-ab6925dd94ad/call-merge_files/stderr -log=cromwell-executions/test/e950e07d-4132-4fe0-8d86-ab6925dd94ad/call-merge_files/merge_files.log +chmod 755 ${script} +cat > ${cwd}/execution/submitFile < rc`, the backend will wait for the existence of this file, parse out the return code and determine success or failure and then subsequently post-process. -``` +The command used to submit the job is specified under the configuration key `backend.providers.HtCondor.config.submit`. It uses the same syntax as a command in WDL, and will be provided the variables: -* provider: it defines the provider to use based on CacheActorFactory and CacheActor interfaces. -* enabled: enables or disables cache. -* forceRewrite: it allows to invalidate the cache entry and store result again. -* db section: configuration related to MongoDB provider. It may not exist for other implementations. +* `script` - A shell script of the job to be run. This contains the user's command from the `command` section of the WDL code. +* `cwd` - The path where the script should be run. +* `out` - The path to the stdout. +* `err` - The path to the stderr. +* `job_name` - A unique name for the job. -### Docker -This backend supports the following optional runtime attributes / workflow options for working with Docker: -* docker: Docker image to use such as "Ubuntu". -* dockerWorkingDir: defines the working directory in the container. -* dockerOutputDir: defiles the output directory in the container when there is the need to define a volume for outputs within the container. By default if this attribute is not set, dockerOutputDir will be the job working directory. +This backend also supports docker as optional feature. Configuration key `backend.providers.HtCondor.config.submit-docker` is specified for this end. When the WDL contains a docker runtime attribute, this command will be provided with two additional variables: -Inputs: -HtCondor backend analyzes all inputs and do a distinct of the folders in order to mount input folders into the container. +* `docker` - The docker image name. +* `docker_cwd` - The path where `cwd` should be mounted within the docker container. -Outputs: -It will use dockerOutputDir runtime attribute / workflow option to resolve the folder in which the execution results will placed. If there is no dockerOutputDir defined it will use the current working directory. +``` +chmod 755 ${script} +cat > ${cwd}/execution/dockerScript < ${cwd}/execution/submitFile <= 64"] + cpu = 2 + memory = "1GB" + disk = "1GB" + nativeSpecs: "TARGET.Arch == \"INTEL\" && TARGET.Memory >= 64" } ``` -nativeSpecs attribute needs to be specified as an array of strings to work. +nativeSpecs attribute needs to be specified as String. ## Spark Backend diff --git a/backend/src/main/scala/cromwell/backend/validation/MemoryValidation.scala b/backend/src/main/scala/cromwell/backend/validation/MemoryValidation.scala index 7af6271b9..bc0fb32da 100644 --- a/backend/src/main/scala/cromwell/backend/validation/MemoryValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/MemoryValidation.scala @@ -25,13 +25,16 @@ import scala.util.{Failure, Success} * `withDefaultMemory` can be used to create a memory validation that defaults to a particular memory size. */ object MemoryValidation { - lazy val instance: RuntimeAttributesValidation[MemorySize] = new MemoryValidation - lazy val optional: OptionalRuntimeAttributesValidation[MemorySize] = instance.optional - def configDefaultString(config: Option[Config]): Option[String] = instance.configDefaultValue(config) - def withDefaultMemory(memorySize: String): RuntimeAttributesValidation[MemorySize] = { + def instance(attributeName: String = RuntimeAttributesKeys.MemoryKey): RuntimeAttributesValidation[MemorySize] = + new MemoryValidation(attributeName) + def optional(attributeName: String = RuntimeAttributesKeys.MemoryKey): OptionalRuntimeAttributesValidation[MemorySize] = + instance(attributeName).optional + def configDefaultString(attributeName: String = RuntimeAttributesKeys.MemoryKey, config: Option[Config]): Option[String] = + instance(attributeName).configDefaultValue(config) + def withDefaultMemory(attributeName: String = RuntimeAttributesKeys.MemoryKey, memorySize: String): RuntimeAttributesValidation[MemorySize] = { MemorySize.parse(memorySize) match { - case Success(memory) => instance.withDefault(WdlInteger(memory.bytes.toInt)) - case Failure(_) => instance.withDefault(BadDefaultAttribute(WdlString(memorySize.toString))) + case Success(memory) => instance(attributeName).withDefault(WdlInteger(memory.bytes.toInt)) + case Failure(_) => instance(attributeName).withDefault(BadDefaultAttribute(WdlString(memorySize.toString))) } } @@ -66,11 +69,11 @@ object MemoryValidation { } } -class MemoryValidation extends RuntimeAttributesValidation[MemorySize] { +class MemoryValidation(attributeName: String = RuntimeAttributesKeys.MemoryKey) extends RuntimeAttributesValidation[MemorySize] { import MemoryValidation._ - override def key = RuntimeAttributesKeys.MemoryKey + override def key = attributeName override def coercion = Seq(WdlIntegerType, WdlStringType) diff --git a/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala b/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala index 5a868f416..6635747a1 100644 --- a/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala @@ -34,7 +34,7 @@ object RuntimeAttributesValidation { } def validateMemory(value: Option[WdlValue], onMissingKey: => ErrorOr[MemorySize]): ErrorOr[MemorySize] = { - validateWithValidation(value, MemoryValidation.instance, onMissingKey) + validateWithValidation(value, MemoryValidation.instance(), onMissingKey) } def validateCpu(cpu: Option[WdlValue], onMissingKey: => ErrorOr[Int]): ErrorOr[Int] = { diff --git a/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala b/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala index c6de0b819..4cd60eef7 100644 --- a/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala +++ b/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala @@ -334,8 +334,8 @@ class RuntimeAttributesValidationSpec extends WordSpecLike with Matchers with Be val backendConfig: Config = ConfigFactory.parseString(backendConfigTemplate).getConfig("default-runtime-attributes") - val memoryVal = MemoryValidation.configDefaultString(Some(backendConfig)) - MemoryValidation.withDefaultMemory(memoryVal.get).runtimeAttributeDefinition.factoryDefault shouldBe Some((WdlInteger(2000000000))) + val memoryVal = MemoryValidation.configDefaultString(RuntimeAttributesKeys.MemoryKey, Some(backendConfig)) + MemoryValidation.withDefaultMemory(RuntimeAttributesKeys.MemoryKey, memoryVal.get).runtimeAttributeDefinition.factoryDefault shouldBe Some((WdlInteger(2000000000))) } "shouldn't throw up if the value for a default-runtime-attribute key cannot be coerced into an expected WdlType" in { @@ -348,8 +348,8 @@ class RuntimeAttributesValidationSpec extends WordSpecLike with Matchers with Be val backendConfig: Config = ConfigFactory.parseString(backendConfigTemplate).getConfig("default-runtime-attributes") - val memoryVal = MemoryValidation.configDefaultString(Some(backendConfig)) - MemoryValidation.withDefaultMemory(memoryVal.get).runtimeAttributeDefinition.factoryDefault shouldBe Some(BadDefaultAttribute(WdlString("blahblah"))) + val memoryVal = MemoryValidation.configDefaultString(RuntimeAttributesKeys.MemoryKey, Some(backendConfig)) + MemoryValidation.withDefaultMemory(RuntimeAttributesKeys.MemoryKey, memoryVal.get).runtimeAttributeDefinition.factoryDefault shouldBe Some(BadDefaultAttribute(WdlString("blahblah"))) } "should be able to coerce a list of return codes into an WdlArray" in { diff --git a/build.sbt b/build.sbt index a07ab730e..9716e9dd8 100644 --- a/build.sbt +++ b/build.sbt @@ -57,12 +57,6 @@ lazy val tesBackend = (project in backendRoot / "tes") .dependsOn(sfsBackend) .dependsOn(backend % "test->test") -lazy val htCondorBackend = (project in backendRoot / "htcondor") - .settings(htCondorBackendSettings:_*) - .withTestSettings - .dependsOn(sfsBackend) - .dependsOn(backend % "test->test") - lazy val sparkBackend = (project in backendRoot / "spark") .settings(sparkBackendSettings:_*) .withTestSettings @@ -106,7 +100,6 @@ lazy val root = (project in file(".")) .aggregate(services) .aggregate(backend) .aggregate(sfsBackend) - .aggregate(htCondorBackend) .aggregate(sparkBackend) .aggregate(jesBackend) .aggregate(tesBackend) @@ -115,7 +108,6 @@ lazy val root = (project in file(".")) .dependsOn(engine) .dependsOn(jesBackend) .dependsOn(tesBackend) - .dependsOn(htCondorBackend) .dependsOn(sparkBackend) // Dependencies for tests .dependsOn(engine % "test->test") diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf index c8b718e57..3a476c518 100644 --- a/core/src/main/resources/reference.conf +++ b/core/src/main/resources/reference.conf @@ -344,48 +344,62 @@ backend { #} #HtCondor { - # actor-factory = "cromwell.backend.impl.htcondor.HtCondorBackendFactory" + # actor-factory = "cromwell.backend.impl.sfs.config.ConfigBackendLifecycleActorFactory" # config { - # # Root directory where Cromwell writes job results. This directory must be - # # visible and writeable by the Cromwell process as well as the jobs that Cromwell - # # launches. - # root: "cromwell-executions" + # runtime-attributes = """ + # Int cpu = 1 + # Float memory_mb = 512.0 + # Float disk_kb = 256000.0 + # String? nativeSpecs + # String? docker + # """ # - # #Placeholders: - # #1. Working directory. - # #2. Working directory volume. - # #3. Inputs volumes. - # #4. Output volume. - # #5. Docker image. - # #6. Job command. - # docker { - # #Allow soft links in dockerized jobs - # cmd = "docker run -w %s %s %s %s --rm %s /bin/bash -c \"%s\"" - # defaultWorkingDir = "/workingDir/" - # defaultOutputDir = "/output/" - # } + # submit = """ + # chmod 755 ${script} + # cat > ${cwd}/execution/submitFile < ${cwd}/execution/dockerScript < ${cwd}/execution/submitFile < Try(value.toBoolean).getOrElse { - logger.warn(s"Could not get '$optionKey' attribute from workflow options. Falling back to default value.") - defaultValue - } - case Failure(_) => defaultValue - } - } -} - diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorInitializationActor.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorInitializationActor.scala deleted file mode 100644 index eb298c558..000000000 --- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorInitializationActor.scala +++ /dev/null @@ -1,75 +0,0 @@ -package cromwell.backend.impl.htcondor - -import akka.actor.{ActorRef, Props} -import cromwell.backend.impl.htcondor.HtCondorInitializationActor._ -import cromwell.backend.impl.htcondor.HtCondorRuntimeAttributes._ -import cromwell.backend.validation.{ContinueOnReturnCodeValidation, RuntimeAttributesDefault} -import cromwell.backend.validation.RuntimeAttributesKeys._ -import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendWorkflowDescriptor, BackendWorkflowInitializationActor} -import cromwell.core.WorkflowOptions -import wdl4s.TaskCall -import wdl4s.types.{WdlBooleanType, WdlIntegerType, WdlStringType} -import wdl4s.values.WdlValue - -import scala.concurrent.Future -import scala.util.Try - -object HtCondorInitializationActor { - val SupportedKeys = Set(DockerKey, DockerWorkingDirKey, DockerOutputDirKey, FailOnStderrKey, - ContinueOnReturnCodeKey, CpuKey, MemoryKey, DiskKey) - - def props(workflowDescriptor: BackendWorkflowDescriptor, - calls: Set[TaskCall], - configurationDescriptor: BackendConfigurationDescriptor, - serviceRegistryActor: ActorRef): Props = - Props(new HtCondorInitializationActor(workflowDescriptor, calls, configurationDescriptor, serviceRegistryActor)) -} - -class HtCondorInitializationActor(override val workflowDescriptor: BackendWorkflowDescriptor, - override val calls: Set[TaskCall], - override val configurationDescriptor: BackendConfigurationDescriptor, - override val serviceRegistryActor: ActorRef) extends BackendWorkflowInitializationActor { - - override protected def runtimeAttributeValidators: Map[String, (Option[WdlValue]) => Boolean] = Map( - DockerKey -> wdlTypePredicate(valueRequired = false, WdlStringType.isCoerceableFrom), - DockerWorkingDirKey -> wdlTypePredicate(valueRequired = false, WdlStringType.isCoerceableFrom), - DockerOutputDirKey -> wdlTypePredicate(valueRequired = false, WdlStringType.isCoerceableFrom), - FailOnStderrKey -> wdlTypePredicate(valueRequired = false, WdlBooleanType.isCoerceableFrom), - ContinueOnReturnCodeKey -> continueOnReturnCodePredicate(valueRequired = false), - CpuKey -> wdlTypePredicate(valueRequired = false, WdlIntegerType.isCoerceableFrom), - MemoryKey -> wdlTypePredicate(valueRequired = false, WdlStringType.isCoerceableFrom), - DiskKey -> wdlTypePredicate(valueRequired = false, WdlStringType.isCoerceableFrom) - ) - - /** - * A call which happens before anything else runs - */ - override def beforeAll(): Future[Option[BackendInitializationData]] = Future.successful(None) - - /** - * Validate that this WorkflowBackendActor can run all of the calls that it's been assigned - */ - override def validate(): Future[Unit] = { - Future { - calls foreach { call => - val runtimeAttributes = call.task.runtimeAttributes.attrs - val notSupportedAttributes = runtimeAttributes filterKeys { !SupportedKeys.contains(_) } - - if (notSupportedAttributes.nonEmpty) { - val notSupportedAttrString = notSupportedAttributes.keys mkString ", " - log.warning(s"Key/s [$notSupportedAttrString] is/are not supported by HtCondorBackend. Unsupported attributes will not be part of jobs executions.") - } - } - } - } - - override protected def coerceDefaultRuntimeAttributes(options: WorkflowOptions): Try[Map[String, WdlValue]] = { - RuntimeAttributesDefault.workflowOptionsDefault(options, HtCondorRuntimeAttributes.coercionMap) - } - - override def continueOnReturnCodePredicate(valueRequired: Boolean)(wdlExpressionMaybe: Option[WdlValue]): Boolean = { - val continueOnReturnCodeDefaultValue = HtCondorRuntimeAttributes.staticDefaults.get(ContinueOnReturnCodeKey).get - ContinueOnReturnCodeValidation.instance.withDefault(continueOnReturnCodeDefaultValue).validateOptionalExpression(wdlExpressionMaybe) - } - -} diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala deleted file mode 100644 index be5437ce4..000000000 --- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActor.scala +++ /dev/null @@ -1,379 +0,0 @@ -package cromwell.backend.impl.htcondor - -import java.nio.file.attribute.PosixFilePermission -import java.util.UUID - -import akka.actor.{ActorRef, Props} -import cromwell.backend.BackendJobExecutionActor.{BackendJobExecutionResponse, JobFailedNonRetryableResponse, JobSucceededResponse} -import cromwell.backend._ -import cromwell.backend.impl.htcondor.caching.CacheActor._ -import cromwell.backend.impl.htcondor.caching.localization.CachedResultLocalization -import cromwell.backend.io.JobPathsWithDocker -import cromwell.backend.sfs.{SharedFileSystem, SharedFileSystemExpressionFunctions} -import cromwell.backend.wdl.Command -import cromwell.core.path.JavaWriterImplicits._ -import cromwell.core.path.Obsolete._ -import cromwell.core.path.{DefaultPathBuilder, PathBuilder} -import cromwell.services.keyvalue.KeyValueServiceActor._ -import cromwell.services.metadata.CallMetadataKeys -import lenthall.util.TryUtil -import org.apache.commons.codec.digest.DigestUtils -import wdl4s.EvaluatedTaskInputs -import wdl4s.parser.MemoryUnit -import wdl4s.types.{WdlArrayType, WdlFileType} -import wdl4s.values.WdlArray - -import scala.concurrent.{Future, Promise} -import scala.sys.process.ProcessLogger -import scala.util.{Failure, Success, Try} - -object HtCondorJobExecutionActor { - val HtCondorJobIdKey = "htCondor_job_id" - - val pathBuilders = List(DefaultPathBuilder) - - def props(jobDescriptor: BackendJobDescriptor, configurationDescriptor: BackendConfigurationDescriptor, serviceRegistryActor: ActorRef, cacheActorProps: Option[Props]): Props = - Props(new HtCondorJobExecutionActor(jobDescriptor, configurationDescriptor, serviceRegistryActor, cacheActorProps)) -} - -class HtCondorJobExecutionActor(override val jobDescriptor: BackendJobDescriptor, - override val configurationDescriptor: BackendConfigurationDescriptor, - serviceRegistryActor: ActorRef, - cacheActorProps: Option[Props]) - extends BackendJobExecutionActor with CachedResultLocalization with SharedFileSystem { - - import HtCondorJobExecutionActor._ - - private val tag = s"CondorJobExecutionActor-${jobDescriptor.call.fullyQualifiedName}:" - override val pathBuilders: List[PathBuilder] = HtCondorJobExecutionActor.pathBuilders - - implicit val executionContext = context.dispatcher - - lazy val cmds = new HtCondorCommands - lazy val extProcess = new HtCondorProcess - - private val fileSystemsConfig = configurationDescriptor.backendConfig.getConfig("filesystems") - override val sharedFileSystemConfig = fileSystemsConfig.getConfig("local") - private val workflowDescriptor = jobDescriptor.workflowDescriptor - private val jobPaths = new JobPathsWithDocker(jobDescriptor.key, workflowDescriptor, configurationDescriptor.backendConfig) - - // Files - private val executionDir = jobPaths.callExecutionRoot - private val returnCodePath = jobPaths.returnCode - private val stdoutPath = jobPaths.stdout - private val stderrPath = jobPaths.stderr - private val scriptPath = jobPaths.script - - // stdout stderr writers for submit file logs - private val submitFilePath = executionDir.resolve("submitfile") - private val submitFileStderr = executionDir.resolve("submitfile.stderr") - private val submitFileStdout = executionDir.resolve("submitfile.stdout") - private val htCondorLog = executionDir.resolve(s"${jobDescriptor.call.unqualifiedName}.log") - - private lazy val stdoutWriter = extProcess.untailedWriter(submitFileStdout) - private lazy val stderrWriter = extProcess.tailedWriter(100, submitFileStderr) - - private val call = jobDescriptor.key.call - private val callEngineFunction = SharedFileSystemExpressionFunctions(jobPaths, pathBuilders) - - private val lookup = jobDescriptor.fullyQualifiedInputs.apply _ - - private val runtimeAttributes = { - val evaluateAttrs = call.task.runtimeAttributes.attrs mapValues (_.evaluate(lookup, callEngineFunction)) - // Fail the call if runtime attributes can't be evaluated - val runtimeMap = TryUtil.sequenceMap(evaluateAttrs, "Runtime attributes evaluation").get - HtCondorRuntimeAttributes(runtimeMap, jobDescriptor.workflowDescriptor.workflowOptions) - } - - private val cacheActor = cacheActorProps match { - case Some(props) => Some(context.actorOf(props, s"CacheActor-${jobDescriptor.call.fullyQualifiedName}")) - case None => None - } - - log.debug("{} Calculating hash for current job.", tag) - lazy private val jobHash = calculateHash - - private val executionResponse = Promise[BackendJobExecutionResponse]() - - // Message sent (by self, to self) wrapping over the response produced by HtCondor - private final case class JobExecutionResponse(resp: BackendJobExecutionResponse) - - // Message sent (by self, to self) to trigger a status check to HtCondor - private final case class TrackTaskStatus(id: String) - - private var condorJobId: Option[String] = None - - private val pollingInterval = configurationDescriptor.backendConfig.getInt("poll-interval") - - override def receive = super.receive orElse { - case JobExecutionResponse(resp) => - log.debug("{}: Completing job [{}] with response: [{}]", tag, jobDescriptor.key, resp) - executionResponse trySuccess resp - () - case TrackTaskStatus(id) => - // Avoid the redundant status check if the response is already completed (e.g. in case of abort) - if (!executionResponse.isCompleted) trackTask(id) - - // Messages received from Caching actor - case ExecutionResultFound(succeededResponse) => - executionResponse trySuccess localizeCachedResponse(succeededResponse) - () - case ExecutionResultNotFound => prepareAndExecute() - case ExecutionResultStored(hash) => log.debug("{} Cache entry was stored for Job with hash {}.", tag, hash) - case ExecutionResultAlreadyExist => log.warning("{} Cache entry for hash {} already exist.", tag, jobHash) - - // Messages received from KV actor - case KvPair(scopedKey, Some(jobId)) if scopedKey.key == HtCondorJobIdKey => - log.info("{} Found job id {}. Trying to recover job now.", tag, jobId) - self ! TrackTaskStatus(jobId) - case KvKeyLookupFailed(_) => - log.debug("{} Job id not found. Falling back to execute.", tag) - execute - // -Ywarn-value-discard - () - case KvFailure(_, e) => - log.error("{} Failure attempting to look up HtCondor job id. Exception message: {}. Falling back to execute.", tag, e.getMessage) - execute - // -Ywarn-value-discard - () - } - - /** - * Restart or resume a previously-started job. - */ - override def recover: Future[BackendJobExecutionResponse] = { - log.warning("{} Trying to recover job {}.", tag, jobDescriptor.key.call.fullyQualifiedName) - serviceRegistryActor ! KvGet(ScopedKey(jobDescriptor.workflowDescriptor.id, - KvJobKey(jobDescriptor.key.call.fullyQualifiedName, jobDescriptor.key.index, jobDescriptor.key.attempt), - HtCondorJobIdKey)) - executionResponse.future - } - - /** - * Execute a new job. - */ - override def execute: Future[BackendJobExecutionResponse] = { - log.debug("{} Checking if hash {{}} is in the cache.", tag, jobHash) - cacheActor match { - case Some(actorRef) => actorRef ! ReadExecutionResult(jobHash) - case None => prepareAndExecute() - } - executionResponse.future - } - - /** - * Abort a running job. - */ - override def abort(): Unit = { - // Nothing to do in case `condorJobId` is not defined - condorJobId foreach { id => - log.info("{}: Aborting job [{}:{}].", tag, jobDescriptor.key.tag, id) - val abortProcess = new HtCondorProcess - val argv = Seq(HtCondorCommands.Remove, id) - val process = abortProcess.externalProcess(argv) - val exitVal = process.exitValue() - if (exitVal == 0) - log.info("{}: Job {} successfully killed and removed from the queue.", tag, id) - else - log.error("{}: Failed to kill / remove job {}. Exit Code: {}, Stderr: {}", tag, id, exitVal, abortProcess.processStderr) - } - } - - private def executeTask(): Unit = { - val argv = Seq(HtCondorCommands.Submit, submitFilePath.toString) - val process = extProcess.externalProcess(argv, ProcessLogger(stdoutWriter.writeWithNewline, stderrWriter.writeWithNewline)) - val condorReturnCode = process.exitValue() // blocks until process (i.e. condor submission) finishes - log.debug("{} Return code of condor submit command: {}", tag, condorReturnCode) - - List(stdoutWriter.writer, stderrWriter.writer).foreach(_.flushAndClose()) - - condorReturnCode match { - case 0 if File(submitFileStderr).lines.toList.isEmpty => - log.info("{} {} submitted to HtCondor. Waiting for the job to complete via. RC file status.", tag, jobDescriptor.call.fullyQualifiedName) - val job = HtCondorCommands.SubmitOutputPattern.r - //Number of lines in stdout for submit job will be 3 at max therefore reading all lines at once. - log.debug(s"{} Output of submit process : {}", tag, File(submitFileStdout).lines.toList) - val line = File(submitFileStdout).lines.toList.last - line match { - case job(jobId, clusterId) => - val overallJobIdentifier = s"$clusterId.${jobId.toInt - 1}" // Condor has 0 based indexing on the jobs, probably won't work on stuff like `queue 150` - log.info("{} {} mapped to HtCondor JobID: {}", tag, jobDescriptor.call.fullyQualifiedName, overallJobIdentifier) - serviceRegistryActor ! KvPut(KvPair(ScopedKey(jobDescriptor.workflowDescriptor.id, - KvJobKey(jobDescriptor.key.call.fullyQualifiedName, jobDescriptor.key.index, jobDescriptor.key.attempt), - HtCondorJobIdKey), Option(overallJobIdentifier))) - condorJobId = Option(overallJobIdentifier) - self ! TrackTaskStatus(overallJobIdentifier) - - case _ => self ! JobExecutionResponse(JobFailedNonRetryableResponse(jobDescriptor.key, - new IllegalStateException("Failed to retrieve job(id) and cluster id"), Option(condorReturnCode))) - } - - case 0 => - log.error(s"Unexpected! Received return code for condor submission as 0, although stderr file is non-empty: {}", File(submitFileStderr).lines) - self ! JobExecutionResponse(JobFailedNonRetryableResponse(jobDescriptor.key, - new IllegalStateException(s"Execution process failed. HtCondor returned zero status code but non empty stderr file: $condorReturnCode"), - Option(condorReturnCode))) - - case nonZeroExitCode: Int => - self ! JobExecutionResponse(JobFailedNonRetryableResponse(jobDescriptor.key, - new IllegalStateException(s"Execution process failed. HtCondor returned non zero status code: $condorReturnCode"), Option(condorReturnCode))) - } - } - - private def trackTask(jobIdentifier: String): Unit = { - val jobReturnCode = Try(extProcess.jobReturnCode(jobIdentifier, returnCodePath)) - log.debug("{} Process complete. RC file now exists with value: {}", tag, jobReturnCode) - - jobReturnCode match { - case Success(None) => - import scala.concurrent.duration._ - // Job is still running in HtCondor. Check back again after `pollingInterval` seconds - context.system.scheduler.scheduleOnce(pollingInterval.seconds, self, TrackTaskStatus(jobIdentifier)) - () - case Success(Some(rc)) if runtimeAttributes.continueOnReturnCode.continueFor(rc) => self ! JobExecutionResponse(processSuccess(rc)) - case Success(Some(rc)) => self ! JobExecutionResponse(JobFailedNonRetryableResponse(jobDescriptor.key, - new IllegalStateException("Job exited with invalid return code: " + rc), Option(rc))) - case Failure(error) => self ! JobExecutionResponse(JobFailedNonRetryableResponse(jobDescriptor.key, error, None)) - } - } - - private def processSuccess(rc: Int): BackendJobExecutionResponse = { - evaluateOutputs(callEngineFunction, outputMapper(jobPaths)) match { - case Success(outputs) => - val succeededResponse = JobSucceededResponse(jobDescriptor.key, Some(rc), outputs, None, Seq.empty) - log.debug("{} Storing data into cache for hash {}.", tag, jobHash) - // If cache fails to store data for any reason it should not stop the workflow/task execution but log the issue. - cacheActor foreach { _ ! StoreExecutionResult(jobHash, succeededResponse) } - succeededResponse - case Failure(e) => - val message = Option(e.getMessage) map { - ": " + _ - } getOrElse "" - JobFailedNonRetryableResponse(jobDescriptor.key, new Throwable("Failed post processing of outputs" + message, e), Option(rc)) - } - } - - private def calculateHash: String = { - val cmd = Command.instantiate(jobDescriptor, callEngineFunction) match { - case Success(command) => command - case Failure(ex) => - val errMsg = s"$tag Cannot instantiate job command for caching purposes due to ${ex.getMessage}." - log.error(ex.getCause, errMsg) - throw new IllegalStateException(errMsg, ex.getCause) - } - val str = Seq[Any](cmd, - runtimeAttributes.failOnStderr, - runtimeAttributes.dockerImage.getOrElse(""), - runtimeAttributes.dockerWorkingDir.getOrElse(""), - runtimeAttributes.dockerOutputDir.getOrElse(""), - runtimeAttributes.cpu.toString, - runtimeAttributes.memory.toString, - runtimeAttributes.disk.toString).mkString - DigestUtils.md5Hex(str) - } - - private def createExecutionFolderAndScript(): Unit = { - try { - log.debug("{} Creating execution folder: {}", tag, executionDir) - executionDir.toString.toFile.createIfNotExists(asDirectory = true, createParents = true) - - log.debug("{} Resolving job command", tag) - val command = localizeInputs(jobPaths.callInputsRoot, runtimeAttributes.dockerImage.isDefined)(jobDescriptor.inputDeclarations) flatMap { - localizedInputs => resolveJobCommand(localizedInputs) - } - - log.debug("{} Creating bash script for executing command: {}", tag, command) - cmds.writeScript(command.get, scriptPath.toAbsolutePath, executionDir.toAbsolutePath) // Writes the bash script for executing the command - File(scriptPath).addPermission(PosixFilePermission.OWNER_EXECUTE) // Add executable permissions to the script. - //TODO: Need to append other runtime attributes from Wdl to Condor submit file - val attributes: Map[String, Any] = Map(HtCondorRuntimeKeys.Executable -> scriptPath.toAbsolutePath, - HtCondorRuntimeKeys.InitialWorkingDir -> jobPaths.callExecutionRoot.toAbsolutePath, - HtCondorRuntimeKeys.Output -> stdoutPath.toAbsolutePath, - HtCondorRuntimeKeys.Error -> stderrPath.toAbsolutePath, - HtCondorRuntimeKeys.Log -> htCondorLog.toAbsolutePath, - HtCondorRuntimeKeys.LogXml -> true, - HtCondorRuntimeKeys.LeaveInQueue -> true, - HtCondorRuntimeKeys.Cpu -> runtimeAttributes.cpu, - HtCondorRuntimeKeys.Memory -> runtimeAttributes.memory.to(MemoryUnit.MB).amount.toLong, - HtCondorRuntimeKeys.Disk -> runtimeAttributes.disk.to(MemoryUnit.KB).amount.toLong - ) - - cmds.generateSubmitFile(submitFilePath, attributes, runtimeAttributes.nativeSpecs) // This writes the condor submit file - () - - } catch { - case ex: Exception => - log.error(ex, "Failed to prepare task: " + ex.getMessage) - throw ex - } - } - - private def resolveJobCommand(localizedInputs: EvaluatedTaskInputs): Try[String] = { - val command = if (runtimeAttributes.dockerImage.isDefined) { - modifyCommandForDocker(call.task.instantiateCommand(localizedInputs, callEngineFunction, identity), localizedInputs) - } else { - call.task.instantiateCommand(localizedInputs, callEngineFunction, identity) - } - command match { - case Success(cmd) => tellMetadata(Map("command" -> cmd)) - case Failure(ex) => - log.error("{} failed to resolve command due to exception:{}", tag, ex) - tellMetadata(Map(s"${CallMetadataKeys.Failures}[${UUID.randomUUID().toString}]" -> ex.getMessage)) - } - command - } - - /** - * Fire and forget data to the metadata service - */ - private def tellMetadata(metadataKeyValues: Map[String, Any]): Unit = { - import cromwell.services.metadata.MetadataService.implicits.MetadataAutoPutter - serviceRegistryActor.putMetadata(jobDescriptor.workflowDescriptor.id, Option(jobDescriptor.key), metadataKeyValues) - } - - private def modifyCommandForDocker(jobCmd: Try[String], localizedInputs: EvaluatedTaskInputs): Try[String] = { - Try { - val dockerInputDataVol = localizedInputs.values.collect { - case file if file.wdlType == WdlFileType => - val limit = file.valueString.lastIndexOf("/") - Seq(file.valueString.substring(0, limit)) - case files if files.wdlType == WdlArrayType(WdlFileType) => files.asInstanceOf[WdlArray].value map { file => - val limit = file.valueString.lastIndexOf("/") - file.valueString.substring(0, limit) - } - }.flatten.toSeq - - log.debug("{} List of input volumes: {}", tag, dockerInputDataVol.mkString(",")) - val dockerCmd = configurationDescriptor.backendConfig.getString("docker.cmd") - val defaultWorkingDir = configurationDescriptor.backendConfig.getString("docker.defaultWorkingDir") - val defaultOutputDir = configurationDescriptor.backendConfig.getString("docker.defaultOutputDir") - val dockerVolume = "-v %s:%s" - val dockerVolumeInputs = s"$dockerVolume:ro" - // `v.get` is safe below since we filtered the list earlier with only defined elements - val inputVolumes = dockerInputDataVol.distinct.map(v => dockerVolumeInputs.format(v, v)).mkString(" ") - val outputVolume = dockerVolume.format(executionDir.toAbsolutePath.toString, runtimeAttributes.dockerOutputDir.getOrElse(defaultOutputDir)) - val workingDir = dockerVolume.format(executionDir.toAbsolutePath.toString, runtimeAttributes.dockerWorkingDir.getOrElse(defaultWorkingDir)) - val cmd = dockerCmd.format(runtimeAttributes.dockerWorkingDir.getOrElse(defaultWorkingDir), workingDir, inputVolumes, outputVolume, runtimeAttributes.dockerImage.get, jobCmd.get) - log.debug("{} Docker command line to be used for task execution: {}.", tag, cmd) - cmd - } - } - - private def prepareAndExecute(): Unit = { - try { - createExecutionFolderAndScript() - executeTask() - } catch { - case e: Exception => self ! JobExecutionResponse(JobFailedNonRetryableResponse(jobDescriptor.key, e, None)) - } - } - - private def localizeCachedResponse(succeededResponse: JobSucceededResponse): BackendJobExecutionResponse = { - Try(localizeCachedOutputs(executionDir, succeededResponse.jobOutputs)) match { - case Success(outputs) => - executionDir.toString.toFile.createIfNotExists(asDirectory = true, createParents = true) - JobSucceededResponse(jobDescriptor.key, succeededResponse.returnCode, outputs, None, Seq.empty) - case Failure(exception) => JobFailedNonRetryableResponse(jobDescriptor.key, exception, None) - } - } -} diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributes.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributes.scala deleted file mode 100644 index 4f72c01a6..000000000 --- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributes.scala +++ /dev/null @@ -1,124 +0,0 @@ -package cromwell.backend.impl.htcondor - - -import cats.data.Validated.{Invalid, Valid} -import cats.syntax.cartesian._ -import cats.syntax.validated._ -import cromwell.backend.MemorySize -import cromwell.backend.validation.ContinueOnReturnCode -import cromwell.backend.validation.RuntimeAttributesDefault._ -import cromwell.backend.validation.RuntimeAttributesKeys._ -import cromwell.backend.validation.RuntimeAttributesValidation._ -import cromwell.core._ -import lenthall.validation.ErrorOr._ -import lenthall.exception.MessageAggregation -import wdl4s.types._ -import wdl4s.values.{WdlArray, WdlBoolean, WdlInteger, WdlString, WdlValue} - - -object HtCondorRuntimeAttributes { - private val FailOnStderrDefaultValue = false - private val ContinueOnRcDefaultValue = 0 - private val CpuDefaultValue = 1 - private val MemoryDefaultValue = "0.512 GB" - private val DisksDefaultValue = "1.024 GB" - - val DockerWorkingDirKey = "dockerWorkingDir" - val DockerOutputDirKey = "dockerOutputDir" - val DiskKey = "disk" - val NativeSpecsKey = "nativeSpecs" - - val staticDefaults: Map[String, WdlValue] = Map( - FailOnStderrKey -> WdlBoolean(FailOnStderrDefaultValue), - ContinueOnReturnCodeKey -> WdlInteger(ContinueOnRcDefaultValue), - CpuKey -> WdlInteger(CpuDefaultValue), - MemoryKey -> WdlString(MemoryDefaultValue), - DiskKey -> WdlString(DisksDefaultValue) - ) - - private[htcondor] val coercionMap: Map[String, Set[WdlType]] = Map ( - FailOnStderrKey -> Set[WdlType](WdlBooleanType), - ContinueOnReturnCodeKey -> ContinueOnReturnCode.validWdlTypes, - DockerKey -> Set(WdlStringType), - DockerWorkingDirKey -> Set(WdlStringType), - DockerOutputDirKey -> Set(WdlStringType), - CpuKey -> Set(WdlIntegerType), - MemoryKey -> Set(WdlStringType), - DiskKey -> Set(WdlStringType), - NativeSpecsKey -> Set(WdlArrayType(WdlStringType)) - ) - - def apply(attrs: Map[String, WdlValue], options: WorkflowOptions): HtCondorRuntimeAttributes = { - // Fail now if some workflow options are specified but can't be parsed correctly - val defaultFromOptions = workflowOptionsDefault(options, coercionMap).get - val withDefaultValues = withDefaults(attrs, List(defaultFromOptions, staticDefaults)) - - val docker = validateDocker(withDefaultValues.get(DockerKey), None.validNel) - val dockerWorkingDir = validateDockerWorkingDir(withDefaultValues.get(DockerWorkingDirKey), None.validNel) - val dockerOutputDir = validateDockerOutputDir(withDefaultValues.get(DockerOutputDirKey), None.validNel) - val failOnStderr = validateFailOnStderr(withDefaultValues.get(FailOnStderrKey), noValueFoundFor(FailOnStderrKey)) - val continueOnReturnCode = validateContinueOnReturnCode(withDefaultValues.get(ContinueOnReturnCodeKey), noValueFoundFor(ContinueOnReturnCodeKey)) - val cpu = validateCpu(withDefaultValues.get(CpuKey), noValueFoundFor(CpuKey)) - val memory = validateMemory(withDefaultValues.get(MemoryKey), noValueFoundFor(MemoryKey)) - val disk = validateDisk(withDefaultValues.get(DiskKey), noValueFoundFor(DiskKey)) - val nativeSpecs = validateNativeSpecs(withDefaultValues.get(NativeSpecsKey), None.validNel) - - (continueOnReturnCode |@| docker |@| dockerWorkingDir |@| dockerOutputDir |@| failOnStderr |@| cpu |@| memory |@| disk |@| nativeSpecs) map { - new HtCondorRuntimeAttributes(_, _, _, _, _, _, _, _, _) - } match { - case Valid(x) => x - case Invalid(nel) => throw new RuntimeException with MessageAggregation { - override def exceptionContext: String = "Runtime attribute validation failed" - override def errorMessages: Traversable[String] = nel.toList - } - } - } - - private def validateDockerWorkingDir(dockerWorkingDir: Option[WdlValue], onMissingKey: => ErrorOr[Option[String]]): ErrorOr[Option[String]] = { - dockerWorkingDir match { - case Some(WdlString(s)) => Some(s).validNel - case None => onMissingKey - case _ => s"Expecting $DockerWorkingDirKey runtime attribute to be a String".invalidNel - } - } - - private def validateDockerOutputDir(dockerOutputDir: Option[WdlValue], onMissingKey: => ErrorOr[Option[String]]): ErrorOr[Option[String]] = { - dockerOutputDir match { - case Some(WdlString(s)) => Some(s).validNel - case None => onMissingKey - case _ => s"Expecting $DockerOutputDirKey runtime attribute to be a String".invalidNel - } - } - - private def validateDisk(value: Option[WdlValue], onMissingKey: => ErrorOr[MemorySize]): ErrorOr[MemorySize] = { - val diskWrongFormatMsg = s"Expecting $DiskKey runtime attribute to be an Integer or String with format '8 GB'. Exception: %s" - - value match { - case Some(i: WdlInteger) => parseMemoryInteger(i) - case Some(s: WdlString) => parseMemoryString(s) - case Some(_) => String.format(diskWrongFormatMsg, "Not supported WDL type value").invalidNel - case None => onMissingKey - } - } - - private def validateNativeSpecs(value: Option[WdlValue], onMissingKey: => ErrorOr[Option[Array[String]]]): ErrorOr[Option[Array[String]]] = { - val nativeSpecsWrongFormatMsg = s"Expecting $NativeSpecsKey runtime attribute to be an Array of Strings. Exception: %s" - value match { - case Some(ns: WdlArray) if ns.wdlType.memberType.equals(WdlStringType) => - val nsa = ns.value.map { value => value.valueString }.toArray - Option(nsa).validNel - case Some(_) => String.format(nativeSpecsWrongFormatMsg, "Not supported WDL type value").invalidNel - case None => onMissingKey - } - } -} - -case class HtCondorRuntimeAttributes(continueOnReturnCode: ContinueOnReturnCode, - dockerImage: Option[String], - dockerWorkingDir: Option[String], - dockerOutputDir: Option[String], - failOnStderr: Boolean, - cpu: Int, - memory: MemorySize, - disk: MemorySize, - nativeSpecs: Option[Array[String]]) diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorWrapper.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorWrapper.scala deleted file mode 100644 index cbc9c24e9..000000000 --- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/HtCondorWrapper.scala +++ /dev/null @@ -1,153 +0,0 @@ -package cromwell.backend.impl.htcondor - -import com.typesafe.scalalogging.StrictLogging -import cromwell.backend.impl.htcondor -import cromwell.core.path.JavaWriterImplicits._ -import cromwell.core.path.Obsolete._ -import cromwell.core.path.{Path, TailedWriter, UntailedWriter} - -import scala.sys.process._ - -object JobStatus { - val MapOfStatuses = Map( - 0 -> Created, // This is actually `unexpanded` in HtCondor, not sure what that actually means - 1 -> Created, // Idle - 2 -> Running, - 3 -> Removed, - 4 -> Completed, - 5 -> Failed, // SystemOnHold - 6 -> SubmissionError // Also the default - ) - - def fromCondorStatusCode(statusCode: Int): JobStatus = { - MapOfStatuses.getOrElse(statusCode, SubmissionError) // By default we return SubmissionError - } - - def isTerminal(jobStatus: JobStatus): Boolean = jobStatus.isInstanceOf[TerminalJobStatus] -} - -sealed trait JobStatus -sealed trait TerminalJobStatus extends JobStatus -case object Created extends JobStatus -case object Running extends JobStatus -case object Completed extends TerminalJobStatus -case object Removed extends TerminalJobStatus -case object Failed extends TerminalJobStatus -case object Aborted extends TerminalJobStatus -case object SubmissionError extends TerminalJobStatus - - -object HtCondorCommands { - val SubmitOutputPattern = "(\\d*) job\\(s\\) submitted to cluster (\\d*)\\." - val Submit = "condor_submit" - val Remove = "condor_rm" - private val JobStatus = "condor_q %s -autoformat JobStatus" - def generateJobStatusCommand(jobId: String): String = HtCondorCommands.JobStatus.format(jobId) -} - -class HtCondorCommands extends StrictLogging { - - /** - * Writes the script file containing the user's command from the WDL as well - * as some extra shell code for monitoring jobs - */ - def writeScript(instantiatedCommand: String, filePath: Path, containerRoot: Path): Unit = { - logger.debug(s"Writing bash script for execution. Command: $instantiatedCommand.") - val scriptBody = s""" - -#!/bin/sh -cd $containerRoot -$instantiatedCommand -echo $$? > rc - -""".trim + "\n" - File(filePath).write(scriptBody) - () - } - - def generateSubmitFile(path: Path, attributes: Map[String, Any], nativeSpecs: Option[Array[String]]): String = { - def htCondorSubmitCommand(filePath: Path) = { - s"${HtCondorCommands.Submit} ${filePath.toString}" - } - - val submitFileWriter = path.untailed - attributes.foreach { attribute => submitFileWriter.writeWithNewline(s"${attribute._1}=${attribute._2}") } - //Native specs is intended for attaching HtCondor native configuration such as 'requirements' and 'rank' definition - //directly to the submit file. - nativeSpecs foreach { _.foreach { submitFileWriter.writeWithNewline } } - submitFileWriter.writeWithNewline(HtCondorRuntimeKeys.Queue) - submitFileWriter.writer.flushAndClose() - logger.debug(s"submit file name is : $path") - logger.debug(s"content of file is : ${File(path).lines.toList}") - htCondorSubmitCommand(path) - } - -} - -class HtCondorProcess extends StrictLogging { - private val stdout = new StringBuilder - private val stderr = new StringBuilder - - def processLogger: ProcessLogger = ProcessLogger(s => { stdout append s; () }, s => { stderr append s; () }) - def processStdout: String = stdout.toString().trim - def processStderr: String = stderr.toString().trim - def commandList(command: String): Seq[String] = Seq("/bin/bash",command) - def untailedWriter(path: Path): UntailedWriter = path.untailed - def tailedWriter(limit: Int, path: Path): TailedWriter = path.tailed(limit) - def externalProcess(cmdList: Seq[String], processLogger: ProcessLogger = processLogger): Process = cmdList.run(processLogger) - - /** - * Returns the RC of this job if it has finished. - */ - def jobReturnCode(jobId: String, returnCodeFilePath: Path): Option[Int] = { - - checkStatus(jobId) match { - case status if JobStatus.isTerminal(status) => - Files.exists(returnCodeFilePath) match { - case true => Option(File(returnCodeFilePath).contentAsString.stripLineEnd.toInt) - case false => - val msg = s"JobStatus from Condor is terminal ($status) and no RC file exists!" - logger.debug(msg) - throw new IllegalStateException(msg) - } - case nonTerminalStatus => None - } - } - - private def checkStatus(jobId: String): JobStatus = { - val htCondorProcess = new HtCondorProcess - val commandArgv = HtCondorCommands.generateJobStatusCommand(jobId).split(" ").toSeq - val process = htCondorProcess.externalProcess(commandArgv) - val returnCode = process.exitValue() - returnCode match { - case 0 => - val stdout = htCondorProcess.processStdout - // If stdout is empty, that means the job got removed from the queue. Return Completed in that case - val status = if (stdout.isEmpty) htcondor.Completed else JobStatus.fromCondorStatusCode(htCondorProcess.processStdout.toInt) - logger.info("Condor JobId {} current status: {}", jobId, status) - status - case errorCode => - val msg = "Could not retreive status from the queue: " + htCondorProcess.processStderr - logger.error(msg) - throw new IllegalStateException(msg) - } - } - -} - -object HtCondorRuntimeKeys { - val Executable = "executable" - val Arguments = "arguments" - val Error = "error" - val Output = "output" - val Log = "log" - val Queue = "queue" - val Rank = "rank" - val Requirements = "requirements" - val Cpu = "request_cpus" - val Memory = "request_memory" - val Disk = "request_disk" - val LogXml = "log_xml" - val LeaveInQueue = "leave_in_queue" - val InitialWorkingDir = "Iwd" -} diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/CacheActor.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/CacheActor.scala deleted file mode 100644 index 74fb3ade5..000000000 --- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/CacheActor.scala +++ /dev/null @@ -1,51 +0,0 @@ -package cromwell.backend.impl.htcondor.caching - -import akka.actor.{Actor, ActorLogging} -import cromwell.backend.BackendJobExecutionActor.JobSucceededResponse -import cromwell.backend.impl.htcondor.caching.CacheActor._ -import cromwell.backend.impl.htcondor.caching.exception.{CachedResultAlreadyExistException, CachedResultNotFoundException} -import cromwell.backend.impl.htcondor.caching.model.CachedExecutionResult - -object CacheActor { - - trait CacheActorCommand - case class ReadExecutionResult(hash: String) extends CacheActorCommand - case class StoreExecutionResult(hash: String, succeededResponse: JobSucceededResponse) extends CacheActorCommand - - trait CacheActorResponse - case class ExecutionResultFound(succeededResponse: JobSucceededResponse) extends CacheActorResponse - case object ExecutionResultNotFound extends CacheActorResponse - case class ExecutionResultStored(hash: String) extends CacheActorResponse - case object ExecutionResultAlreadyExist extends CacheActorResponse - -} - -trait CacheActor extends Actor with ActorLogging { - def tag: String = "[CacheActor]" - def forceRewrite: Boolean = false - - override def receive: Receive = { - case ReadExecutionResult(hash) => - try { - val executionResult = readExecutionResult(hash) - log.info(s"{} Execution result found in cache for hash {}. Returning result: {}.", tag, hash, executionResult) - sender() ! ExecutionResultFound(executionResult.succeededResponse) - } catch { - case ex: CachedResultNotFoundException => sender() ! ExecutionResultNotFound - } - - case StoreExecutionResult(hash, succeededResult) => - try { - storeExecutionResult(CachedExecutionResult(hash, succeededResult)) - log.info(s"{} Cache entry for job [{}] stored successfully.", tag, hash) - sender() ! ExecutionResultStored(hash) - } catch { - case ex: CachedResultAlreadyExistException => sender() ! ExecutionResultAlreadyExist - } - } - - def readExecutionResult(hash: String): CachedExecutionResult - - def storeExecutionResult(cachedExecutionResult: CachedExecutionResult): Unit - -} diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/CacheActorFactory.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/CacheActorFactory.scala deleted file mode 100644 index 9dd99cc24..000000000 --- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/CacheActorFactory.scala +++ /dev/null @@ -1,9 +0,0 @@ -package cromwell.backend.impl.htcondor.caching - -import akka.actor.Props - -trait CacheActorFactory { - - def getCacheActorProps(forceRewrite: Boolean): Props - -} diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/exception/CachedResultAlreadyExistException.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/exception/CachedResultAlreadyExistException.scala deleted file mode 100644 index 387252809..000000000 --- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/exception/CachedResultAlreadyExistException.scala +++ /dev/null @@ -1,5 +0,0 @@ -package cromwell.backend.impl.htcondor.caching.exception - -class CachedResultNotFoundException(message: String) extends RuntimeException(message) - -class CachedResultAlreadyExistException(message: String) extends RuntimeException(message) diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/localization/CachedResultLocalization.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/localization/CachedResultLocalization.scala deleted file mode 100644 index 027b0b287..000000000 --- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/localization/CachedResultLocalization.scala +++ /dev/null @@ -1,38 +0,0 @@ -package cromwell.backend.impl.htcondor.caching.localization - -import cromwell.core._ -import cromwell.core.path.Obsolete._ -import cromwell.core.path.Path -import wdl4s.types.{WdlArrayType, WdlFileType} -import wdl4s.values.{WdlArray, WdlSingleFile, WdlValue} - -trait CachedResultLocalization { - private[localization] def localizePathViaSymbolicLink(originalPath: Path, executionPath: Path): Path = { - if (File(originalPath).isDirectory) throw new UnsupportedOperationException("Cannot localize directory with symbolic links.") - else { - File(executionPath).parent.createPermissionedDirectories() - Files.createSymbolicLink(executionPath, originalPath.toAbsolutePath) - } - } - - private[localization] def localizeCachedFile(executionPath: Path, output: WdlValue): WdlSingleFile = { - val origPath = Paths.get(output.valueString) - val newPath = executionPath.toAbsolutePath.resolve(origPath.getFileName) - val slPath = localizePathViaSymbolicLink(origPath, newPath) - WdlSingleFile(slPath.toString) - } - - def localizeCachedOutputs(executionPath: Path, outputs: CallOutputs): CallOutputs = { - outputs map { case (lqn, jobOutput) => - jobOutput.wdlValue.wdlType match { - case WdlFileType => (lqn -> JobOutput(localizeCachedFile(executionPath, jobOutput.wdlValue))) - case WdlArrayType(WdlFileType) => - val newArray: Seq[WdlSingleFile] = jobOutput.wdlValue.asInstanceOf[WdlArray].value map { - localizeCachedFile(executionPath, _) - } - (lqn -> JobOutput(WdlArray(WdlArrayType(WdlFileType), newArray))) - case _ => (lqn, jobOutput) - } - } - } -} diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/model/CachedExecutionResult.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/model/CachedExecutionResult.scala deleted file mode 100644 index 1b023fd41..000000000 --- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/model/CachedExecutionResult.scala +++ /dev/null @@ -1,6 +0,0 @@ -package cromwell.backend.impl.htcondor.caching.model - -import cromwell.backend.BackendJobExecutionActor.JobSucceededResponse - -case class CachedExecutionResult(hash: String, succeededResponse: JobSucceededResponse) - diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/MongoCacheActor.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/MongoCacheActor.scala deleted file mode 100644 index cf6cf1151..000000000 --- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/MongoCacheActor.scala +++ /dev/null @@ -1,88 +0,0 @@ -package cromwell.backend.impl.htcondor.caching.provider.mongodb - -import com.mongodb.DBObject -import com.mongodb.casbah.MongoCollection -import com.mongodb.casbah.commons.{MongoDBObject, TypeImports} -import com.mongodb.util.JSON -import cromwell.backend.BackendJobExecutionActor.JobSucceededResponse -import cromwell.backend.impl.htcondor.caching.CacheActor -import cromwell.backend.impl.htcondor.caching.exception.{CachedResultAlreadyExistException, CachedResultNotFoundException} -import cromwell.backend.impl.htcondor.caching.model.CachedExecutionResult -import cromwell.backend.impl.htcondor.caching.provider.mongodb.model.{KryoSerializedObject, MongoCachedExecutionResult} -import cromwell.backend.impl.htcondor.caching.provider.mongodb.serialization.KryoSerDe - -class MongoCacheActor(collection: MongoCollection, - override val forceRewrite: Boolean = false) extends CacheActor with KryoSerDe { - - import cromwell.backend.impl.htcondor.caching.provider.mongodb.model.MongoCachedExecutionResultProtocol._ - import spray.json._ - - val ErrMsg = "Got an exception when storing execution result for hash {}." - val HashIdentifier = "hash" - override val tag = s"[MongoCacheActor]" - - override def readExecutionResult(hash: String): CachedExecutionResult = { - val query = MongoDBObject(HashIdentifier -> hash) - val result = collection.findOne(query) - result match { - case Some(mongoDbObject) => - if (forceRewrite) throwCachedResultNotFoundException(hash) - else deserializeSucceededResponse(mongoDbObject) - case None => throwCachedResultNotFoundException(hash) - } - } - - override def storeExecutionResult(cachedExecutionResult: CachedExecutionResult): Unit = { - try { - readExecutionResult(cachedExecutionResult.hash) - val warnMsg = s"$tag Execution result hash {${cachedExecutionResult.hash}} is already defined in database." - log.warning(warnMsg) - if (forceRewrite) { - removeExecutionResult(cachedExecutionResult.hash) - storeExecutionResultInMongoDb(cachedExecutionResult) - } - else throw new CachedResultAlreadyExistException(warnMsg) - } catch { - case e: CachedResultNotFoundException => - storeExecutionResultInMongoDb(cachedExecutionResult) - case e: CachedResultAlreadyExistException => - throw e - case e: Exception => - log.error(e, "{} Got an unhandled exception when trying to store execution result for hash {}.", tag, cachedExecutionResult.hash) - throw e - } - } - - private def deserializeSucceededResponse(mongoDbObject: TypeImports.DBObject): CachedExecutionResult = { - val cachedResult = JsonParser(mongoDbObject.toString).convertTo[MongoCachedExecutionResult] - val succeededResponse = deserialize(cachedResult.succeededResponse.byteArray, classOf[JobSucceededResponse]) - CachedExecutionResult(cachedResult.hash, succeededResponse) - } - - private def removeExecutionResult(hash: String): Unit = { - val query = MongoDBObject(HashIdentifier -> hash) - val result = collection.remove(query) - if (result.getN == 0) throwCachedResultNotFoundException(hash) - log.info("{} Removed execution result for hash: {}.", tag, hash) - } - - private def storeExecutionResultInMongoDb(cachedExecutionResult: CachedExecutionResult): Unit = { - val cachedResult = MongoCachedExecutionResult(cachedExecutionResult.hash, KryoSerializedObject(serialize(cachedExecutionResult.succeededResponse))) - val result = collection.insert(constructDbObjectForCachedExecutionResult(cachedResult)) - if (!result.wasAcknowledged()) { - log.error(ErrMsg, cachedExecutionResult) - throw new IllegalStateException(ErrMsg) - } - } - - private def constructDbObjectForCachedExecutionResult(cachedExecutionResult: MongoCachedExecutionResult): DBObject = { - val resultAsJsonString = cachedExecutionResult.toJson.toString() - JSON.parse(resultAsJsonString).asInstanceOf[DBObject] - } - - private def throwCachedResultNotFoundException(hash: String): Nothing = { - val warnMsg = s"$tag Execution result hash {$hash} does not exist in database." - log.warning(warnMsg) - throw new CachedResultNotFoundException(warnMsg) - } -} diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/MongoCacheActorFactory.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/MongoCacheActorFactory.scala deleted file mode 100644 index bdb1354f9..000000000 --- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/MongoCacheActorFactory.scala +++ /dev/null @@ -1,21 +0,0 @@ -package cromwell.backend.impl.htcondor.caching.provider.mongodb - -import akka.actor.Props -import com.mongodb.casbah.{MongoClient, MongoCollection} -import com.typesafe.config.Config -import cromwell.backend.impl.htcondor.caching.CacheActorFactory - -class MongoCacheActorFactory(config: Config) extends CacheActorFactory { - val dbHost = config.getString("cache.db.host") - val dbPort = config.getInt("cache.db.port") - val dbName = config.getString("cache.db.name") - val dbCollectionName = config.getString("cache.db.collection") - val dbInstance: MongoClient = MongoClient(dbHost, dbPort) - val db = dbInstance(dbName) - val collection: MongoCollection = db(dbCollectionName) - - override def getCacheActorProps(forceRewrite: Boolean): Props = { - Props(new MongoCacheActor(collection, forceRewrite)) - } - -} diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/model/MongoCachedExecutionResult.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/model/MongoCachedExecutionResult.scala deleted file mode 100644 index f13326257..000000000 --- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/model/MongoCachedExecutionResult.scala +++ /dev/null @@ -1,24 +0,0 @@ -package cromwell.backend.impl.htcondor.caching.provider.mongodb.model - -import spray.json - -/** - * Wrapper over the byte array that is stored in db - * - * @param byteArray Serialized data - */ -case class KryoSerializedObject(byteArray: Array[Byte]) - -/** - * SucceededResponse to be stored in MongoDB. - * - * @param hash Calculated hash for the Job. - * @param succeededResponse Serialized succeeded response. - */ -case class MongoCachedExecutionResult(hash: String, succeededResponse: KryoSerializedObject) - -object MongoCachedExecutionResultProtocol extends json.DefaultJsonProtocol { - implicit val kryoSerializedObject = jsonFormat1(KryoSerializedObject) - implicit val cachedExecutionResultProtocol = jsonFormat2(MongoCachedExecutionResult) -} - diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/serialization/KryoSerDe.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/serialization/KryoSerDe.scala deleted file mode 100644 index d3f564c61..000000000 --- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/serialization/KryoSerDe.scala +++ /dev/null @@ -1,55 +0,0 @@ -package cromwell.backend.impl.htcondor.caching.provider.mongodb.serialization - -import java.io.{ByteArrayInputStream, ByteArrayOutputStream} - -import com.esotericsoftware.kryo.io.{Input, Output} -import com.twitter.chill.ScalaKryoInstantiator - -/** - * This mixin provides access to `serialize` and `deserialize` methods that use Kryo underneath to - * perform the actual conversion to / from bytes - */ -trait KryoSerDe extends SerDe { - - /** - * - * @param data Any Scala / Java object that needs to be serialized - * @return A serialized byte array that can be transported across Network moved around with - */ - override def serialize[A <: AnyRef](data: A): Array[Byte] = { - try { - val instantiator = new ScalaKryoInstantiator - instantiator.setRegistrationRequired(false) // This makes it unnecessary to register all classes - val kryo = instantiator.newKryo() - val buffer = new ByteArrayOutputStream() - val output = new Output(buffer) - kryo.writeObject(output, data) - output.close() - buffer.toByteArray - } catch { - case exception: Exception => throw SerDeException("Failed to serialize data.", exception) - } - } - - /** - * - * @param byteArray Kryo serialized data. Expects only results of `writeObject` - * @param toClass Class to which the `byteArray` will be deserialized - * @return The deserialized object as an instance of A - */ - override def deserialize[A <: AnyRef](byteArray: Array[Byte], toClass: Class[A]): A = { - try { - val instantiator = new ScalaKryoInstantiator - instantiator.setRegistrationRequired(false) - val kryo = instantiator.newKryo() - val buffer = new ByteArrayInputStream(byteArray) - val input = new Input(buffer) - val result = kryo.readObject(input, toClass) - input.close() - result - } catch { - case exception: Exception => throw SerDeException("Failed to deserialize data.", exception) - } - } - -} diff --git a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/serialization/SerDe.scala b/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/serialization/SerDe.scala deleted file mode 100644 index 7ab8883d0..000000000 --- a/supportedBackends/htcondor/src/main/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/serialization/SerDe.scala +++ /dev/null @@ -1,25 +0,0 @@ -package cromwell.backend.impl.htcondor.caching.provider.mongodb.serialization - -/** - * A vanilla Serialization / Deserialization interface - */ -trait SerDe { - - case class SerDeException(message: String, error: Throwable) extends IllegalStateException(message, error) - - /** - * Serialize the given data - * @param data Any Scala / Java object that needs to be serialized - * @return A serialized byte array that can be transported across Network moved around with - */ - def serialize[A <: AnyRef](data: A): Array[Byte] - - /** - * Deserializes an array of Bytes to the given class - * @param byteArray Kryo serialized data. Expects only results of `writeObject` - * @param toClass Class to which the `byteArray` will be deserialized - * @return The deserialized object as an instance of A - */ - def deserialize[A <: AnyRef](byteArray: Array[Byte], toClass: Class[A]): A - -} diff --git a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorCommandSpec.scala b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorCommandSpec.scala deleted file mode 100644 index d6778fadd..000000000 --- a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorCommandSpec.scala +++ /dev/null @@ -1,28 +0,0 @@ -package cromwell.backend.impl.htcondor - -import cromwell.core.path.Obsolete._ -import org.scalatest.{Matchers, WordSpecLike} - -class HtCondorCommandSpec extends WordSpecLike with Matchers { - private val attributes = Map("executable" -> "test.sh", "input" -> "/temp/test", "error"->"stderr") - private val resultAttributes = List("executable=test.sh","input=/temp/test","error=stderr", "spec1", "spec2", "queue") - private val htCondorCommands = new HtCondorCommands - private val nativeSpecs = Option(Array("spec1", "spec2")) - - "submitCommand method" should { - "return submit file with content passed to it" in { - val file = File.newTemporaryFile() - val command = htCondorCommands.generateSubmitFile(file.path, attributes, nativeSpecs) - resultAttributes shouldEqual file.lines.toList - file.delete() - command shouldEqual s"condor_submit ${file.path}" - } - } - - "statusCommand method" should { - "return status command" in { - val command = HtCondorCommands.generateJobStatusCommand("96.0") - command shouldEqual s"condor_q 96.0 -autoformat JobStatus" - } - } -} \ No newline at end of file diff --git a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorInitializationActorSpec.scala b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorInitializationActorSpec.scala deleted file mode 100644 index ba569802f..000000000 --- a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorInitializationActorSpec.scala +++ /dev/null @@ -1,53 +0,0 @@ -package cromwell.backend.impl.htcondor - -import akka.testkit.{EventFilter, ImplicitSender, TestDuration} -import cromwell.backend.BackendWorkflowInitializationActor.Initialize -import cromwell.backend.{BackendConfigurationDescriptor, BackendSpec, BackendWorkflowDescriptor, TestConfig} -import cromwell.core.TestKitSuite -import org.scalatest.{Matchers, WordSpecLike} -import wdl4s.TaskCall - -import scala.concurrent.duration._ - -class HtCondorInitializationActorSpec extends TestKitSuite("HtCondorInitializationActorSpec") with WordSpecLike - with Matchers with ImplicitSender { - val Timeout = 5.second.dilated - - import BackendSpec._ - - val HelloWorld = - s""" - |task hello { - | String addressee = "you" - | command { - | echo "Hello $${addressee}!" - | } - | output { - | String salutation = read_string(stdout()) - | } - | - | RUNTIME - |} - | - |workflow wf_hello { - | call hello - |} - """.stripMargin - - private def getHtCondorBackend(workflowDescriptor: BackendWorkflowDescriptor, calls: Set[TaskCall], conf: BackendConfigurationDescriptor) = { - system.actorOf(HtCondorInitializationActor.props(workflowDescriptor, calls, conf, emptyActor)) - } - - "HtCondorInitializationActor" should { - "log a warning message when there are unsupported runtime attributes" in { - within(Timeout) { - EventFilter.warning(message = s"Key/s [proc] is/are not supported by HtCondorBackend. Unsupported attributes will not be part of jobs executions.", occurrences = 1) intercept { - val workflowDescriptor = buildWorkflowDescriptor(HelloWorld, runtime = """runtime { proc: 1 }""") - val backend = getHtCondorBackend(workflowDescriptor, workflowDescriptor.workflow.taskCalls, - TestConfig.emptyBackendConfigDescriptor) - backend ! Initialize - } - } - } - } -} diff --git a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActorSpec.scala b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActorSpec.scala deleted file mode 100644 index 3ce2f499e..000000000 --- a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorJobExecutionActorSpec.scala +++ /dev/null @@ -1,472 +0,0 @@ -package cromwell.backend.impl.htcondor - -import java.io.Writer - -import akka.actor.{Actor, Props} -import akka.testkit.{ImplicitSender, TestActorRef} -import com.typesafe.config.ConfigFactory -import cromwell.backend.BackendJobExecutionActor.{JobFailedNonRetryableResponse, JobSucceededResponse} -import cromwell.backend.impl.htcondor.caching.CacheActor -import cromwell.backend.impl.htcondor.caching.exception.CachedResultNotFoundException -import cromwell.backend.impl.htcondor.caching.model.CachedExecutionResult -import cromwell.backend.io.JobPathsWithDocker -import cromwell.backend.{BackendConfigurationDescriptor, BackendJobDescriptor, BackendSpec} -import cromwell.core._ -import cromwell.core.path.Obsolete._ -import cromwell.core.path.{Path, PathWriter, TailedWriter, UntailedWriter} -import cromwell.services.keyvalue.KeyValueServiceActor.{KvGet, KvPair, KvPut} -import org.mockito.Mockito -import org.mockito.Mockito._ -import org.scalatest.concurrent.PatienceConfiguration.Timeout -import org.scalatest.mockito.MockitoSugar -import org.scalatest.{BeforeAndAfter, Matchers, WordSpecLike} -import wdl4s.types.{WdlArrayType, WdlFileType} -import wdl4s.values.{WdlArray, WdlFile, WdlValue} - -import scala.concurrent.duration._ -import scala.io.Source -import scala.sys.process.{Process, ProcessLogger} - -class HtCondorJobExecutionActorSpec extends TestKitSuite("HtCondorJobExecutionActorSpec") - with WordSpecLike - with Matchers - with MockitoSugar - with BeforeAndAfter - with ImplicitSender { - - import BackendSpec._ - - private val htCondorCommands: HtCondorCommands = new HtCondorCommands - private val htCondorProcess: HtCondorProcess = mock[HtCondorProcess] - private val cacheActorMockProps = Props(new CacheActorMock()) - - private val helloWorldWdl = - """ - |task hello { - | - | command { - | echo "Hello World!" - | } - | output { - | String salutation = read_string(stdout()) - | } - | RUNTIME - |} - | - |workflow wf_hello { - | call hello - |} - """.stripMargin - - private val helloWorldWdlWithFileInput = - s""" - |task hello { - | File inputFile - | - | command { - | echo $${inputFile} - | } - | output { - | String salutation = read_string(stdout()) - | } - | RUNTIME - |} - | - |workflow wf_hello { - | call hello - |} - """.stripMargin - - private val helloWorldWdlWithFileArrayInput = - s""" - |task hello { - | Array[File] inputFiles - | - | command { - | echo $${sep=' ' inputFiles} - | } - | output { - | String salutation = read_string(stdout()) - | } - | RUNTIME - |} - | - |workflow wf_hello { - | call hello - |} - """.stripMargin - - private val backendConfig = ConfigFactory.parseString( - s"""{ - | root = "local-cromwell-executions" - | - | docker { - | cmd = "docker run -w %s %s %s %s --rm %s /bin/bash -c \\"%s\\"" - | defaultWorkingDir = "/workingDir/" - | defaultOutputDir = "/output/" - | } - | - | filesystems { - | local { - | localization = [ - | "hard-link", "soft-link", "copy" - | ] - | } - | } - | poll-interval = 3 - |} - """.stripMargin) - - private val timeout = Timeout(1.seconds) - - after { - Mockito.reset(htCondorProcess) - } - - "executeTask method" should { - "return succeeded task status with stdout" in { - val jobDescriptor = prepareJob() - val (job, jobPaths, backendConfigDesc) = (jobDescriptor.jobDescriptor, jobDescriptor.jobPaths, jobDescriptor.backendConfigurationDescriptor) - val stubProcess = mock[Process] - val stubUntailed = new UntailedWriter(jobPaths.stdout) with MockPathWriter - val stubTailed = new TailedWriter(jobPaths.stderr, 100) with MockPathWriter - val stderrResult = "" - - when(htCondorProcess.commandList(any[String])).thenReturn(Seq.empty[String]) - when(htCondorProcess.externalProcess(any[Seq[String]], any[ProcessLogger])).thenReturn(stubProcess) - when(stubProcess.exitValue()).thenReturn(0) - when(htCondorProcess.tailedWriter(any[Int], any[Path])).thenReturn(stubTailed) - when(htCondorProcess.untailedWriter(any[Path])).thenReturn(stubUntailed) - when(htCondorProcess.processStderr).thenReturn(stderrResult) - when(htCondorProcess.jobReturnCode(any[String], any[Path])).thenReturn(Option(0)) - - val backend = TestActorRef(new HtCondorJobExecutionActor(job, backendConfigDesc, system.deadLetters, None) { - override lazy val cmds = htCondorCommands - override lazy val extProcess = htCondorProcess - }).underlyingActor - - whenReady(backend.execute, timeout) { response => - response shouldBe a[JobSucceededResponse] - verify(htCondorProcess, times(1)).externalProcess(any[Seq[String]], any[ProcessLogger]) - verify(htCondorProcess, times(1)).tailedWriter(any[Int], any[Path]) - verify(htCondorProcess, times(1)).untailedWriter(any[Path]) - } - - cleanUpJob(jobPaths) - } - - "return succeeded task status when it recovers from a shutdown" in { - val jobDescriptor = prepareJob() - val (job, jobPaths, backendConfigDesc) = (jobDescriptor.jobDescriptor, jobDescriptor.jobPaths, jobDescriptor.backendConfigurationDescriptor) - val stubProcess = mock[Process] - val stubUntailed = new UntailedWriter(jobPaths.stdout) with MockPathWriter - val stubTailed = new TailedWriter(jobPaths.stderr, 100) with MockPathWriter - val stderrResult = "" - val kVServiceActor = system.actorOf(Props(new KVServiceActor())) - - when(htCondorProcess.commandList(any[String])).thenReturn(Seq.empty[String]) - when(htCondorProcess.externalProcess(any[Seq[String]], any[ProcessLogger])).thenReturn(stubProcess) - when(stubProcess.exitValue()).thenReturn(0) - when(htCondorProcess.tailedWriter(any[Int], any[Path])).thenReturn(stubTailed) - when(htCondorProcess.untailedWriter(any[Path])).thenReturn(stubUntailed) - when(htCondorProcess.processStderr).thenReturn(stderrResult) - when(htCondorProcess.jobReturnCode(any[String], any[Path])).thenReturn(Option(0)) - - val backend = TestActorRef(new HtCondorJobExecutionActor(job, backendConfigDesc, kVServiceActor, None) { - override lazy val cmds = htCondorCommands - override lazy val extProcess = htCondorProcess - }).underlyingActor - - whenReady(backend.recover, timeout) { response => - response shouldBe a[JobSucceededResponse] - } - - cleanUpJob(jobPaths) - } - - "return succeeded task status with stdout when cache is enabled" in { - val jobDescriptor = prepareJob() - val (job, jobPaths, backendConfigDesc) = (jobDescriptor.jobDescriptor, jobDescriptor.jobPaths, jobDescriptor.backendConfigurationDescriptor) - val stubProcess = mock[Process] - val stubUntailed = new UntailedWriter(jobPaths.stdout) with MockPathWriter - val stubTailed = new TailedWriter(jobPaths.stderr, 100) with MockPathWriter - val stderrResult = "" - - when(htCondorProcess.commandList(any[String])).thenReturn(Seq.empty[String]) - when(htCondorProcess.externalProcess(any[Seq[String]], any[ProcessLogger])).thenReturn(stubProcess) - when(stubProcess.exitValue()).thenReturn(0) - when(htCondorProcess.tailedWriter(any[Int], any[Path])).thenReturn(stubTailed) - when(htCondorProcess.untailedWriter(any[Path])).thenReturn(stubUntailed) - when(htCondorProcess.processStderr).thenReturn(stderrResult) - when(htCondorProcess.jobReturnCode(any[String], any[Path])).thenReturn(Option(0)) - - val backend = TestActorRef(new HtCondorJobExecutionActor(job, backendConfigDesc, system.deadLetters, Some(cacheActorMockProps)) { - override lazy val cmds = htCondorCommands - override lazy val extProcess = htCondorProcess - }).underlyingActor - - whenReady(backend.execute, timeout) { response => - response shouldBe a[JobSucceededResponse] - verify(htCondorProcess, times(1)).externalProcess(any[Seq[String]], any[ProcessLogger]) - verify(htCondorProcess, times(1)).tailedWriter(any[Int], any[Path]) - verify(htCondorProcess, times(1)).untailedWriter(any[Path]) - } - - cleanUpJob(jobPaths) - } - - "return failed task status with stderr on non-zero process exit" in { - val jobDescriptor = prepareJob() - val (job, jobPaths, backendConfigDesc) = (jobDescriptor.jobDescriptor, jobDescriptor.jobPaths, jobDescriptor.backendConfigurationDescriptor) - - val backend = TestActorRef(new HtCondorJobExecutionActor(job, backendConfigDesc, system.deadLetters, Some(cacheActorMockProps)) { - override lazy val cmds = htCondorCommands - override lazy val extProcess = htCondorProcess - }).underlyingActor - val stubProcess = mock[Process] - val stubUntailed = new UntailedWriter(jobPaths.stdout) with MockPathWriter - val stubTailed = new TailedWriter(jobPaths.stderr, 100) with MockPathWriter - val stderrResult = "" - - when(htCondorProcess.externalProcess(any[Seq[String]], any[ProcessLogger])).thenReturn(stubProcess) - when(stubProcess.exitValue()).thenReturn(0) - when(htCondorProcess.tailedWriter(any[Int], any[Path])).thenReturn(stubTailed) - when(htCondorProcess.untailedWriter(any[Path])).thenReturn(stubUntailed) - when(htCondorProcess.processStderr).thenReturn(stderrResult) - when(htCondorProcess.jobReturnCode(any[String], any[Path])).thenReturn(Option(-1)) - - whenReady(backend.execute, timeout) { response => - response shouldBe a[JobFailedNonRetryableResponse] - assert(response.asInstanceOf[JobFailedNonRetryableResponse].throwable.getMessage.contains("Job exited with invalid return code")) - } - - cleanUpJob(jobPaths) - } - - "return a successful task status even with a non-zero process exit" in { - val runtime = - """ - |runtime { - | continueOnReturnCode: [911] - |} - """.stripMargin - val jobDescriptor = prepareJob(runtimeString = runtime) - val (job, jobPaths, backendConfigDesc) = (jobDescriptor.jobDescriptor, jobDescriptor.jobPaths, jobDescriptor.backendConfigurationDescriptor) - - val backend = TestActorRef(new HtCondorJobExecutionActor(job, backendConfigDesc, system.deadLetters, Some(cacheActorMockProps)) { - override lazy val cmds = htCondorCommands - override lazy val extProcess = htCondorProcess - }).underlyingActor - val stubProcess = mock[Process] - val stubUntailed = new UntailedWriter(jobPaths.stdout) with MockPathWriter - val stubTailed = new TailedWriter(jobPaths.stderr, 100) with MockPathWriter - val stderrResult = "" - - when(htCondorProcess.externalProcess(any[Seq[String]], any[ProcessLogger])).thenReturn(stubProcess) - when(stubProcess.exitValue()).thenReturn(0) - when(htCondorProcess.tailedWriter(any[Int], any[Path])).thenReturn(stubTailed) - when(htCondorProcess.untailedWriter(any[Path])).thenReturn(stubUntailed) - when(htCondorProcess.processStderr).thenReturn(stderrResult) - when(htCondorProcess.jobReturnCode(any[String], any[Path])).thenReturn(Option(911)) - - whenReady(backend.execute, timeout) { response => - response shouldBe a[JobSucceededResponse] - } - - cleanUpJob(jobPaths) - } - - "return a successful task status when it runs a docker command with working and output directory" in { - val runtime = - """ - |runtime { - | docker: "ubuntu/latest" - | dockerWorkingDir: "/workingDir/" - | dockerOutputDir: "/outputDir/" - |} - """.stripMargin - val jsonInputFile = createCannedFile("testFile", "some content").pathAsString - val inputs = Map( - "wf_hello.hello.inputFile" -> WdlFile(jsonInputFile) - ) - val jobDescriptor = prepareJob(helloWorldWdlWithFileInput, runtime, Option(inputs)) - val (job, jobPaths, backendConfigDesc) = (jobDescriptor.jobDescriptor, jobDescriptor.jobPaths, jobDescriptor.backendConfigurationDescriptor) - - val backend = TestActorRef(new HtCondorJobExecutionActor(job, backendConfigDesc, system.deadLetters, Some(cacheActorMockProps)) { - override lazy val cmds = htCondorCommands - override lazy val extProcess = htCondorProcess - }).underlyingActor - val stubProcess = mock[Process] - val stubUntailed = new UntailedWriter(jobPaths.stdout) with MockPathWriter - val stubTailed = new TailedWriter(jobPaths.stderr, 100) with MockPathWriter - val stderrResult = "" - - when(htCondorProcess.externalProcess(any[Seq[String]], any[ProcessLogger])).thenReturn(stubProcess) - when(stubProcess.exitValue()).thenReturn(0) - when(htCondorProcess.tailedWriter(any[Int], any[Path])).thenReturn(stubTailed) - when(htCondorProcess.untailedWriter(any[Path])).thenReturn(stubUntailed) - when(htCondorProcess.processStderr).thenReturn(stderrResult) - when(htCondorProcess.jobReturnCode(any[String], any[Path])).thenReturn(Option(0)) - - whenReady(backend.execute) { response => - response shouldBe a[JobSucceededResponse] - } - - val bashScript = Source.fromFile(jobPaths.script.toFile).getLines.mkString - - assert(bashScript.contains("docker run -w /workingDir/ -v")) - assert(bashScript.contains(":/workingDir/")) - assert(bashScript.contains(":ro")) - assert(bashScript.contains("/call-hello/execution:/outputDir/ --rm ubuntu/latest /bin/bash -c \"echo")) - - cleanUpJob(jobPaths) - } - - "return failed when cmds fails to write script" in { - val htCondorCommandsMock: HtCondorCommands = mock[HtCondorCommands] - val jobDescriptor = prepareJob() - val (job, jobPaths, backendConfigDesc) = (jobDescriptor.jobDescriptor, jobDescriptor.jobPaths, jobDescriptor.backendConfigurationDescriptor) - - val backend = TestActorRef(new HtCondorJobExecutionActor(job, backendConfigDesc, system.deadLetters, Some(cacheActorMockProps)) { - override lazy val cmds = htCondorCommandsMock - override lazy val extProcess = htCondorProcess - }).underlyingActor - val stubProcess = mock[Process] - val stubUntailed = new UntailedWriter(jobPaths.stdout) with MockPathWriter - val stubTailed = new TailedWriter(jobPaths.stderr, 100) with MockPathWriter - val stderrResult = "" - - when(htCondorCommandsMock.writeScript(any[String], any[Path], any[Path])).thenThrow(new IllegalStateException("Could not write the file.")) - when(htCondorProcess.externalProcess(any[Seq[String]], any[ProcessLogger])).thenReturn(stubProcess) - when(stubProcess.exitValue()).thenReturn(0) - when(htCondorProcess.tailedWriter(any[Int], any[Path])).thenReturn(stubTailed) - when(htCondorProcess.untailedWriter(any[Path])).thenReturn(stubUntailed) - when(htCondorProcess.processStderr).thenReturn(stderrResult) - when(htCondorProcess.jobReturnCode(any[String], any[Path])).thenReturn(Option(-1)) - - whenReady(backend.execute, timeout) { response => - response shouldBe a[JobFailedNonRetryableResponse] - assert(response.asInstanceOf[JobFailedNonRetryableResponse].throwable.getMessage.contains("Could not write the file.")) - } - - cleanUpJob(jobPaths) - } - } - - "return a successful task status when it tries to run a docker command containing file data from a WDL file array" in { - val runtime = - """ - |runtime { - | docker: "ubuntu/latest" - | dockerWorkingDir: "/workingDir/" - | dockerOutputDir: "/outputDir/" - |} - """.stripMargin - - val tempDir1 = Files.createTempDirectory("dir1") - val tempDir2 = Files.createTempDirectory("dir2") - val jsonInputFile = - createCannedFile(prefix = "testFile", contents = "some content", dir = Some(tempDir1)).pathAsString - val jsonInputFile2 = - createCannedFile(prefix = "testFile2", contents = "some other content", dir = Some(tempDir2)).pathAsString - - val inputs = Map( - "wf_hello.hello.inputFiles" -> WdlArray(WdlArrayType(WdlFileType), Seq(WdlFile(jsonInputFile), WdlFile(jsonInputFile2))) - ) - val jobDescriptor = prepareJob(helloWorldWdlWithFileArrayInput, runtime, Option(inputs)) - val (job, jobPaths, backendConfigDesc) = (jobDescriptor.jobDescriptor, jobDescriptor.jobPaths, jobDescriptor.backendConfigurationDescriptor) - - val backend = TestActorRef(new HtCondorJobExecutionActor(job, backendConfigDesc, system.deadLetters, Some(cacheActorMockProps)) { - override lazy val cmds = htCondorCommands - override lazy val extProcess = htCondorProcess - }).underlyingActor - val stubProcess = mock[Process] - val stubUntailed = new UntailedWriter(jobPaths.stdout) with MockPathWriter - val stubTailed = new TailedWriter(jobPaths.stderr, 100) with MockPathWriter - val stderrResult = "" - - when(htCondorProcess.externalProcess(any[Seq[String]], any[ProcessLogger])).thenReturn(stubProcess) - when(stubProcess.exitValue()).thenReturn(0) - when(htCondorProcess.tailedWriter(any[Int], any[Path])).thenReturn(stubTailed) - when(htCondorProcess.untailedWriter(any[Path])).thenReturn(stubUntailed) - when(htCondorProcess.processStderr).thenReturn(stderrResult) - when(htCondorProcess.jobReturnCode(any[String], any[Path])).thenReturn(Option(0)) - - whenReady(backend.execute) { response => - response shouldBe a[JobSucceededResponse] - } - - val bashScript = Source.fromFile(jobPaths.script.toFile).getLines.mkString - - assert(bashScript.contains("docker run -w /workingDir/ -v")) - assert(bashScript.contains(":/workingDir/")) - assert(bashScript.contains(tempDir1.toAbsolutePath.toString)) - assert(bashScript.contains(tempDir2.toAbsolutePath.toString)) - assert(bashScript.contains("/call-hello/execution:/outputDir/ --rm ubuntu/latest /bin/bash -c \"echo")) - - cleanUpJob(jobPaths) - } - - private def cleanUpJob(jobPaths: JobPathsWithDocker): Unit = { - File(jobPaths.workflowRoot).delete(true) - () - } - - private def createCannedFile(prefix: String, contents: String, dir: Option[Path] = None): File = { - val suffix = ".out" - val file = File.newTemporaryFile(prefix, suffix, dir.map(File.apply)) - file.write(contents) - } - - val emptyWorkflowOptions = WorkflowOptions.fromMap(Map.empty).get - - private def prepareJob(source: String = helloWorldWdl, runtimeString: String = "", inputFiles: Option[Map[String, WdlValue]] = None): TestJobDescriptor = { - val backendWorkflowDescriptor = buildWorkflowDescriptor(wdl = source, inputs = inputFiles.getOrElse(Map.empty), runtime = runtimeString) - val backendConfigurationDescriptor = BackendConfigurationDescriptor(backendConfig, ConfigFactory.load) - val jobDesc = jobDescriptorFromSingleCallWorkflow(backendWorkflowDescriptor, inputFiles.getOrElse(Map.empty), emptyWorkflowOptions, Set.empty) - val jobPaths = new JobPathsWithDocker(jobDesc.key, backendWorkflowDescriptor, backendConfig) - val executionDir = File(jobPaths.callExecutionRoot) - val stdout = File(executionDir.pathAsString, "stdout") - stdout.createIfNotExists(asDirectory = false, createParents = true) - val submitFileStderr = executionDir./("submitfile.stderr") - val submitFileStdout = executionDir./("submitfile.stdout") - submitFileStdout.createIfNotExists(asDirectory = false, createParents = true) - submitFileStdout << - """Submitting job(s).. - |1 job(s) submitted to cluster 88. - """.stripMargin.trim - submitFileStderr.createIfNotExists(asDirectory = false, createParents = true) - TestJobDescriptor(jobDesc, jobPaths, backendConfigurationDescriptor) - } - - private case class TestJobDescriptor(jobDescriptor: BackendJobDescriptor, jobPaths: JobPathsWithDocker, backendConfigurationDescriptor: BackendConfigurationDescriptor) - - trait MockWriter extends Writer { - var closed = false - - override def close() = closed = true - - override def flush() = {} - - override def write(a: Array[Char], b: Int, c: Int) = {} - } - - trait MockPathWriter extends PathWriter { - override lazy val writer: Writer = new MockWriter {} - override val path: Path = mock[Path] - } - - class CacheActorMock extends CacheActor { - override def readExecutionResult(hash: String): CachedExecutionResult = throw new CachedResultNotFoundException("Entry not found.") - - override def storeExecutionResult(cachedExecutionResult: CachedExecutionResult): Unit = () - } - - class KVServiceActor extends Actor { - override def receive: Receive = { - case _: KvPut => // Do nothing - case KvGet(kvKey) => sender ! KvPair(kvKey, Option("123")) - } - } - -} diff --git a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributesSpec.scala b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributesSpec.scala deleted file mode 100644 index 9812c44e9..000000000 --- a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/HtCondorRuntimeAttributesSpec.scala +++ /dev/null @@ -1,304 +0,0 @@ -package cromwell.backend.impl.htcondor - -import cromwell.backend.{BackendSpec, MemorySize} -import cromwell.backend.validation.ContinueOnReturnCodeSet -import cromwell.backend.validation.RuntimeAttributesKeys._ -import cromwell.core.WorkflowOptions -import lenthall.util.TryUtil -import org.scalatest.{Matchers, WordSpecLike} -import spray.json._ -import wdl4s.WdlExpression._ -import wdl4s._ -import wdl4s.expression.NoFunctions -import wdl4s.values.WdlValue - -class HtCondorRuntimeAttributesSpec extends WordSpecLike with Matchers { - - import BackendSpec._ - - val HelloWorld = - s""" - |task hello { - | String addressee = "you" - | command { - | echo "Hello $${addressee}!" - | } - | output { - | String salutation = read_string(stdout()) - | } - | - | RUNTIME - |} - | - |workflow wf_hello { - | call hello - |} - """.stripMargin - - val emptyWorkflowOptions = WorkflowOptions(JsObject(Map.empty[String, JsValue])) - - val memorySize = MemorySize.parse("0.512 GB").get - val diskSize = MemorySize.parse("1.024 GB").get - val staticDefaults = new HtCondorRuntimeAttributes(ContinueOnReturnCodeSet(Set(0)), None, None, None, false, 1, - memorySize, diskSize, None) - - def workflowOptionsWithDefaultRA(defaults: Map[String, JsValue]) = { - WorkflowOptions(JsObject(Map( - "default_runtime_attributes" -> JsObject(defaults) - ))) - } - - "HtCondorRuntimeAttributes" should { - "return an instance of itself when there are no runtime attributes defined." in { - val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { }""").head - assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, emptyWorkflowOptions, staticDefaults) - } - - "return an instance of itself when tries to validate a valid Docker entry" in { - val expectedRuntimeAttributes = staticDefaults.copy(dockerImage = Option("ubuntu:latest")) - val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { docker: "ubuntu:latest" }""").head - assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, emptyWorkflowOptions, expectedRuntimeAttributes) - } - - "return an instance of itself when tries to validate a valid Docker entry based on input" in { - val expectedRuntimeAttributes = staticDefaults.copy(dockerImage = Option("you")) - val runtimeAttributes = createRuntimeAttributes(HelloWorld, s"""runtime { docker: "\\$${addressee}" }""").head - assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, emptyWorkflowOptions, expectedRuntimeAttributes) - } - - "use workflow options as default if docker key is missing" in { - val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { }""").head - val workflowOptions = workflowOptionsWithDefaultRA(Map(DockerKey -> JsString("ubuntu:latest"))) - assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, workflowOptions, staticDefaults.copy(dockerImage = Some("ubuntu:latest"))) - } - - "throw an exception when tries to validate an invalid Docker entry" in { - val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { docker: 1 }""").head - assertHtCondorRuntimeAttributesFailedCreation(runtimeAttributes, "Expecting docker runtime attribute to be a String") - } - - "return an instance of itself when tries to validate a valid docker working directory entry" in { - val expectedRuntimeAttributes = staticDefaults.copy(dockerWorkingDir = Option("/workingDir")) - val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { dockerWorkingDir: "/workingDir" }""").head - assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, emptyWorkflowOptions, expectedRuntimeAttributes) - } - - "return an instance of itself when tries to validate a valid docker working directory entry based on input" in { - val expectedRuntimeAttributes = staticDefaults.copy(dockerWorkingDir = Option("you")) - val runtimeAttributes = createRuntimeAttributes(HelloWorld, s"""runtime { dockerWorkingDir: "\\$${addressee}" }""").head - assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, emptyWorkflowOptions, expectedRuntimeAttributes) - } - - "use workflow options as default if docker working directory key is missing" in { - val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { }""").head - val workflowOptions = workflowOptionsWithDefaultRA(Map("dockerWorkingDir" -> JsString("/workingDir"))) - assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, workflowOptions, staticDefaults.copy(dockerWorkingDir = Some("/workingDir"))) - } - - "throw an exception when tries to validate an invalid docker working directory entry" in { - val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { dockerWorkingDir: 1 }""").head - assertHtCondorRuntimeAttributesFailedCreation(runtimeAttributes, "Expecting dockerWorkingDir runtime attribute to be a String") - } - - "return an instance of itself when tries to validate a valid docker output directory entry" in { - val expectedRuntimeAttributes = staticDefaults.copy(dockerOutputDir = Option("/outputDir")) - val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { dockerOutputDir: "/outputDir" }""").head - assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, emptyWorkflowOptions, expectedRuntimeAttributes) - } - - "return an instance of itself when tries to validate a valid docker output directory entry based on input" in { - val expectedRuntimeAttributes = staticDefaults.copy(dockerOutputDir = Option("you")) - val runtimeAttributes = createRuntimeAttributes(HelloWorld, s"""runtime { dockerOutputDir: "\\$${addressee}" }""").head - assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, emptyWorkflowOptions, expectedRuntimeAttributes) - } - - "use workflow options as default if docker output directory key is missing" in { - val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { }""").head - val workflowOptions = workflowOptionsWithDefaultRA(Map("dockerOutputDir" -> JsString("/outputDir"))) - assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, workflowOptions, staticDefaults.copy(dockerOutputDir = Some("/outputDir"))) - } - - "throw an exception when tries to validate an invalid docker output directory entry" in { - val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { dockerOutputDir: 1 }""").head - assertHtCondorRuntimeAttributesFailedCreation(runtimeAttributes, "Expecting dockerOutputDir runtime attribute to be a String") - } - - "return an instance of itself when tries to validate a valid failOnStderr entry" in { - val expectedRuntimeAttributes = staticDefaults.copy(failOnStderr = true) - val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { failOnStderr: "true" }""").head - val shouldBeIgnored = workflowOptionsWithDefaultRA(Map(FailOnStderrKey -> JsBoolean(false))) - assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, shouldBeIgnored, expectedRuntimeAttributes) - } - - "throw an exception when tries to validate an invalid failOnStderr entry" in { - val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { failOnStderr: "yes" }""").head - assertHtCondorRuntimeAttributesFailedCreation(runtimeAttributes, "Expecting failOnStderr runtime attribute to be a Boolean or a String with values of 'true' or 'false'") - } - - "use workflow options as default if failOnStdErr key is missing" in { - val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { }""").head - val workflowOptions = workflowOptionsWithDefaultRA(Map(FailOnStderrKey -> JsBoolean(true))) - assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, workflowOptions, staticDefaults.copy(failOnStderr = true)) - } - - "return an instance of itself when tries to validate a valid continueOnReturnCode entry" in { - val expectedRuntimeAttributes = staticDefaults.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1))) - val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { continueOnReturnCode: 1 }""").head - val shouldBeIgnored = workflowOptionsWithDefaultRA(Map(ContinueOnReturnCodeKey -> JsBoolean(false))) - assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, shouldBeIgnored, expectedRuntimeAttributes) - } - - "throw an exception when tries to validate an invalid continueOnReturnCode entry" in { - val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { continueOnReturnCode: "value" }""").head - assertHtCondorRuntimeAttributesFailedCreation(runtimeAttributes, "Expecting continueOnReturnCode runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]") - } - - "use workflow options as default if continueOnReturnCode key is missing" in { - val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { }""").head - val workflowOptions = workflowOptionsWithDefaultRA(Map(ContinueOnReturnCodeKey -> JsArray(Vector(JsNumber(1), JsNumber(2))))) - assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, workflowOptions, staticDefaults.copy(continueOnReturnCode = ContinueOnReturnCodeSet(Set(1, 2)))) - } - - "return an instance of itself when tries to validate a valid cpu entry" in { - val expectedRuntimeAttributes = staticDefaults.copy(cpu = 2) - val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { cpu: 2 }""").head - val shouldBeIgnored = workflowOptionsWithDefaultRA(Map(CpuKey -> JsString("6"))) - assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, shouldBeIgnored, expectedRuntimeAttributes) - } - - "throw an exception when tries to validate an invalid cpu entry" in { - val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { cpu: "value" }""").head - assertHtCondorRuntimeAttributesFailedCreation(runtimeAttributes, "Expecting cpu runtime attribute to be an Integer") - } - - "use workflow options as default if cpu key is missing" in { - val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { }""").head - val expectedRuntimeAttributes = staticDefaults.copy(cpu = 6) - val workflowOptions = workflowOptionsWithDefaultRA(Map(CpuKey -> JsString("6"))) - val workflowOptions2 = workflowOptionsWithDefaultRA(Map(CpuKey -> JsNumber("6"))) - assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, workflowOptions, expectedRuntimeAttributes) - assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, workflowOptions2, expectedRuntimeAttributes) - } - - "use default cpu value when there is no cpu key entry" in { - val expectedRuntimeAttributes = staticDefaults.copy(cpu = 1) - val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { }""").head - val shouldBeIgnored = workflowOptionsWithDefaultRA(Map()) - assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, shouldBeIgnored, expectedRuntimeAttributes) - } - - "return an instance of itself when tries to validate a valid memory entry" in { - val expectedRuntimeAttributes = staticDefaults.copy(memory = MemorySize.parse("1 GB").get) - val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { memory: "1 GB" }""").head - val shouldBeIgnored = workflowOptionsWithDefaultRA(Map(MemoryKey -> JsString("blahaha"))) - assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, shouldBeIgnored, expectedRuntimeAttributes) - } - - "throw an exception when tries to validate an invalid memory entry" in { - val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { docker: "ubuntu:latest" memory: "value" }""").head - assertHtCondorRuntimeAttributesFailedCreation(runtimeAttributes, "Expecting memory runtime attribute to be an Integer or String with format '8 GB'") - } - - "use workflow options as default if memory key is missing" in { - val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { }""").head - val expectedRuntimeAttributes = staticDefaults.copy(memory = MemorySize.parse("65 GB").get) - val workflowOptions = workflowOptionsWithDefaultRA(Map(MemoryKey -> JsString("65 GB"))) - assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, workflowOptions, expectedRuntimeAttributes) - } - - "use default memory value when there is no memory key entry" in { - val expectedRuntimeAttributes = staticDefaults.copy(memory = MemorySize.parse("0.512 GB").get) - val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { }""").head - val shouldBeIgnored = workflowOptionsWithDefaultRA(Map()) - assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, shouldBeIgnored, expectedRuntimeAttributes) - } - - "return an instance of itself when tries to validate a valid disk entry" in { - val expectedRuntimeAttributes = staticDefaults.copy(disk = MemorySize.parse("1 GB").get) - val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { disk: "1 GB" }""").head - val shouldBeIgnored = workflowOptionsWithDefaultRA(Map("disk" -> JsString("blahaha"))) - assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, shouldBeIgnored, expectedRuntimeAttributes) - } - - "throw an exception when tries to validate an invalid String disk entry" in { - val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { docker: "ubuntu:latest" disk: "value" }""").head - assertHtCondorRuntimeAttributesFailedCreation(runtimeAttributes, "Expecting memory runtime attribute to be an Integer or String with format '8 GB'") - } - - "throw an exception when tries to validate an invalid Integer array disk entry" in { - val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { docker: "ubuntu:latest" disk: [1] }""").head - assertHtCondorRuntimeAttributesFailedCreation(runtimeAttributes, "Expecting disk runtime attribute to be an Integer or String with format '8 GB'") - } - - "use workflow options as default if disk key is missing" in { - val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { }""").head - val expectedRuntimeAttributes = staticDefaults.copy(disk = MemorySize.parse("65 GB").get) - val workflowOptions = workflowOptionsWithDefaultRA(Map("disk" -> JsString("65 GB"))) - assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, workflowOptions, expectedRuntimeAttributes) - } - - "use default disk value when there is no disk key entry" in { - val expectedRuntimeAttributes = staticDefaults.copy(disk = MemorySize.parse("1.024 GB").get) - val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { }""").head - val shouldBeIgnored = workflowOptionsWithDefaultRA(Map()) - assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, shouldBeIgnored, expectedRuntimeAttributes) - } - - "return an instance of itself when tries to validate a valid native specs entry" in { - val expectedRuntimeAttributes = staticDefaults.copy(nativeSpecs = Option(Array("spec1", "spec2"))) - val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { nativeSpecs: ["spec1", "spec2"] }""").head - assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes, emptyWorkflowOptions, expectedRuntimeAttributes) - } - - "throw an exception when tries to validate an invalid native specs entry" in { - val runtimeAttributes = createRuntimeAttributes(HelloWorld, """runtime { nativeSpecs: [1, 2] }""").head - assertHtCondorRuntimeAttributesFailedCreation(runtimeAttributes, "Expecting nativeSpecs runtime attribute to be an Array of Strings.") - } - } - - private def assertHtCondorRuntimeAttributesSuccessfulCreation(runtimeAttributes: Map[String, WdlValue], - workflowOptions: WorkflowOptions, - expectedRuntimeAttributes: HtCondorRuntimeAttributes) = { - try { - val actualRuntimeAttr = HtCondorRuntimeAttributes(runtimeAttributes, workflowOptions) - assert(actualRuntimeAttr.cpu == expectedRuntimeAttributes.cpu) - assert(actualRuntimeAttr.disk == expectedRuntimeAttributes.disk) - assert(actualRuntimeAttr.memory == expectedRuntimeAttributes.memory) - assert(actualRuntimeAttr.continueOnReturnCode == expectedRuntimeAttributes.continueOnReturnCode) - assert(actualRuntimeAttr.failOnStderr == expectedRuntimeAttributes.failOnStderr) - assert(actualRuntimeAttr.dockerWorkingDir == expectedRuntimeAttributes.dockerWorkingDir) - assert(actualRuntimeAttr.dockerImage == expectedRuntimeAttributes.dockerImage) - assert(actualRuntimeAttr.dockerOutputDir == expectedRuntimeAttributes.dockerOutputDir) - expectedRuntimeAttributes.nativeSpecs match { - case Some(ns) => assert(ns.deep == expectedRuntimeAttributes.nativeSpecs.get.deep) - case None => assert(expectedRuntimeAttributes.nativeSpecs.isEmpty) - } - } catch { - case ex: RuntimeException => fail(s"Exception was not expected but received: ${ex.getMessage}") - } - } - - private def assertHtCondorRuntimeAttributesFailedCreation(runtimeAttributes: Map[String, WdlValue], exMsg: String) = { - try { - HtCondorRuntimeAttributes(runtimeAttributes, emptyWorkflowOptions) - fail("A RuntimeException was expected.") - } catch { - case ex: RuntimeException => assert(ex.getMessage.contains(exMsg)) - } - } - - private def createRuntimeAttributes(wdlSource: WdlSource, runtimeAttributes: String): Seq[Map[String, WdlValue]] = { - val workflowDescriptor = buildWorkflowDescriptor(wdlSource, runtime = runtimeAttributes) - - def createLookup(call: Call): ScopedLookupFunction = { - val knownInputs = workflowDescriptor.knownValues - call.lookupFunction(knownInputs, NoFunctions) - } - - workflowDescriptor.workflow.taskCalls.toSeq map { - call => - val ra = call.task.runtimeAttributes.attrs mapValues { _.evaluate(createLookup(call), NoFunctions) } - TryUtil.sequenceMap(ra, "Runtime attributes evaluation").get - } - } -} diff --git a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/caching/localization/CachedResultLocalizationSpec.scala b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/caching/localization/CachedResultLocalizationSpec.scala deleted file mode 100644 index 2e3b66656..000000000 --- a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/caching/localization/CachedResultLocalizationSpec.scala +++ /dev/null @@ -1,63 +0,0 @@ -package cromwell.backend.impl.htcondor.caching.localization - -import cromwell.core.path.Obsolete._ -import cromwell.core.{CallOutputs, JobOutput} -import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpecLike} -import wdl4s.types.{WdlArrayType, WdlFileType} -import wdl4s.values.{WdlArray, WdlSingleFile, WdlString} - -class CachedResultLocalizationSpec extends WordSpecLike with Matchers with BeforeAndAfterAll { - private class CachedResultLocalizationMock extends CachedResultLocalization - private val defaultTmpDir = Files.createTempDirectory("cachedFiles").toAbsolutePath - private val defaultCachedFile = defaultTmpDir.resolve("input.txt") - private val newTmpDir = Files.createTempDirectory("newFiles").toAbsolutePath - private val newTmpFile = newTmpDir.resolve(defaultCachedFile.getFileName()) - private val cachedResults = new CachedResultLocalizationMock() - private val defaultFileArray = Seq("arrInput1.txt", "arrInput2.txt", "arrInput3.txt").map(defaultTmpDir.resolve(_).toAbsolutePath) - private val newFileArray = Seq("arrInput1.txt", "arrInput2.txt", "arrInput3.txt").map(newTmpDir.resolve(_).toAbsolutePath) - - override def afterAll() = { - Seq(defaultCachedFile, newTmpFile) ++ newFileArray ++ Seq(defaultTmpDir, newTmpDir) foreach { _.toFile.delete() } - } - - "CachedResultLocalization" should { - "localize file path via symbolic link" in { - val slPath = cachedResults.localizePathViaSymbolicLink(defaultCachedFile, newTmpFile) - assert(Files.isSymbolicLink(slPath)) - Files.delete(newTmpFile) - } - - "not localize dir path via symbolic link" in { - assertThrows[UnsupportedOperationException](cachedResults.localizePathViaSymbolicLink(defaultTmpDir, newTmpFile)) - } - - "localize cached job outputs which are WDL files using symbolic link" in { - val outputs: CallOutputs = Map("File1" -> JobOutput(WdlSingleFile(defaultCachedFile.toAbsolutePath.toString))) - val newJobOutputs = cachedResults.localizeCachedOutputs(newTmpDir, outputs) - newJobOutputs foreach { case (lqn, jobOutput) => - assert(jobOutput.wdlValue.valueString == newTmpFile.toString) - } - } - - "localize cached job outputs which are WDL File Array using symbolic link" in { - val wdlArray = WdlArray(WdlArrayType(WdlFileType), defaultFileArray.map(file => WdlSingleFile(file.toString()))) - val outputs = Map("File1" -> JobOutput(wdlArray)) - val newJobOutputs = cachedResults.localizeCachedOutputs(newTmpDir, outputs) - newJobOutputs foreach { case (lqn, jobOutput) => - val wdlArray = jobOutput.wdlValue.asInstanceOf[WdlArray].value - wdlArray foreach { entry => - assert(!entry.valueString.contains(defaultTmpDir.toString)) - assert(entry.valueString.contains(newTmpDir.toString)) - } - } - } - - "not localize cached job outputs which are not WDL files" in { - val outputs = Map("String1" -> JobOutput(WdlString(defaultCachedFile.toAbsolutePath.toString))) - val newJobOutputs = cachedResults.localizeCachedOutputs(newTmpDir, outputs) - newJobOutputs foreach { case (lqn, jobOutput) => - assert(jobOutput.wdlValue.valueString == defaultCachedFile.toString) - } - } - } -} diff --git a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/MongoCacheActorSpec.scala b/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/MongoCacheActorSpec.scala deleted file mode 100644 index d638b82d9..000000000 --- a/supportedBackends/htcondor/src/test/scala/cromwell/backend/impl/htcondor/caching/provider/mongodb/MongoCacheActorSpec.scala +++ /dev/null @@ -1,104 +0,0 @@ -package cromwell.backend.impl.htcondor.caching.provider.mongodb - -import akka.actor.ActorSystem -import akka.testkit.{ImplicitSender, TestActorRef, TestKit} -import com.mongodb.casbah.MongoCollection -import com.mongodb.casbah.commons.MongoDBObject -import com.mongodb.util.JSON -import com.mongodb.{DBObject, WriteResult} -import com.typesafe.config.{Config, ConfigFactory} -import cromwell.backend.{MemorySize, BackendJobDescriptorKey} -import cromwell.backend.BackendJobExecutionActor.JobSucceededResponse -import cromwell.backend.impl.htcondor.HtCondorRuntimeAttributes -import cromwell.backend.impl.htcondor.caching.CacheActor._ -import cromwell.backend.impl.htcondor.caching.exception.CachedResultNotFoundException -import cromwell.backend.impl.htcondor.caching.provider.mongodb.model.{KryoSerializedObject, MongoCachedExecutionResult} -import cromwell.backend.impl.htcondor.caching.provider.mongodb.serialization.KryoSerDe -import cromwell.backend.validation.ContinueOnReturnCodeSet -import cromwell.core.JobOutput -import org.mockito.Mockito -import org.mockito.Mockito._ -import org.scalatest.mockito.MockitoSugar -import org.scalatest.{BeforeAndAfter, BeforeAndAfterAll, MustMatchers, WordSpecLike} -import wdl4s.TaskCall -import wdl4s.values.WdlString - -class MongoCacheActorSpec extends TestKit(ActorSystem("MongoCacheProviderActorSpecSystem")) with WordSpecLike with MustMatchers - with BeforeAndAfter with BeforeAndAfterAll with ImplicitSender with MockitoSugar with KryoSerDe { - - import spray.json._ - import cromwell.backend.impl.htcondor.caching.provider.mongodb.model.MongoCachedExecutionResultProtocol._ - - val config: Config = ConfigFactory.load() - val mongoDbCollectionMock = mock[MongoCollection] - val memorySize = MemorySize.parse("0.512 GB").get - val diskSize = MemorySize.parse("1.024 GB").get - val runtimeConfig = HtCondorRuntimeAttributes(ContinueOnReturnCodeSet(Set(0)), Some("tool-name"), Some("/workingDir"), Some("/outputDir"), true, 1, memorySize, diskSize, None) - val jobHash = "88dde49db10f1551299fb9937f313c10" - val taskStatus = "done" - val succeededResponseMock = JobSucceededResponse(BackendJobDescriptorKey(TaskCall(Option("taskName"), null, null, null), None, 0), None, Map("test" -> JobOutput(WdlString("Test"))), None, Seq.empty) - val serSucceededRespMock = KryoSerializedObject(serialize(succeededResponseMock)) - val cachedExecutionResult = MongoCachedExecutionResult(jobHash, serSucceededRespMock) - val cachedExecutionResultDbObject = JSON.parse(cachedExecutionResult.toJson.toString).asInstanceOf[DBObject] - val query = MongoDBObject("hash" -> jobHash) - - after { - Mockito.reset(mongoDbCollectionMock) - } - - override def afterAll = shutdown() - - "A CacheActor" should { - "return an ExecutionResultFound when read an execution result from cache" in { - when(mongoDbCollectionMock.findOne(query)) thenReturn Some(cachedExecutionResultDbObject) - val cacheActor = TestActorRef(new MongoCacheActor(mongoDbCollectionMock)) - cacheActor ! ReadExecutionResult(jobHash) - expectMsg(ExecutionResultFound(succeededResponseMock)) - verify(mongoDbCollectionMock, atLeastOnce).findOne(query) - } - - "return an ExecutionResultNotFound when it can't find an execution result in cache" in { - when(mongoDbCollectionMock.findOne(query)) thenReturn None - val cacheActor = TestActorRef(new MongoCacheActor(mongoDbCollectionMock)) - cacheActor ! ReadExecutionResult(jobHash) - expectMsg(ExecutionResultNotFound) - verify(mongoDbCollectionMock, atLeastOnce).findOne(query) - } - - "return ExecutionResultStored when it stores an execution result" in { - when(mongoDbCollectionMock.findOne(query)) thenThrow new CachedResultNotFoundException("") - when(mongoDbCollectionMock.insert(cachedExecutionResultDbObject)) thenReturn new WriteResult(0, true, "") - val cacheActor = TestActorRef(new MongoCacheActor(mongoDbCollectionMock)) - cacheActor ! StoreExecutionResult(jobHash, succeededResponseMock) - expectMsg(ExecutionResultStored("88dde49db10f1551299fb9937f313c10")) - verify(mongoDbCollectionMock, atLeastOnce).insert(cachedExecutionResultDbObject) - } - - "return ExecutionResultAlreadyExist when it tries to store an existing execution result" in { - when(mongoDbCollectionMock.findOne(query)) thenReturn Some(cachedExecutionResultDbObject) - val cacheActor = TestActorRef(new MongoCacheActor(mongoDbCollectionMock)) - cacheActor ! StoreExecutionResult(jobHash, succeededResponseMock) - expectMsg(ExecutionResultAlreadyExist) - verify(mongoDbCollectionMock, atLeastOnce).findOne(query) - } - - "return ExecutionResultNotFound when try to read and force re-write flag is enabled" in { - when(mongoDbCollectionMock.findOne(query)) thenReturn Some(cachedExecutionResultDbObject) - val cacheActor = TestActorRef(new MongoCacheActor(mongoDbCollectionMock, true)) - cacheActor ! ReadExecutionResult(jobHash) - expectMsg(ExecutionResultNotFound) - verify(mongoDbCollectionMock, atLeastOnce).findOne(query) - } - - "return ExecutionResultStored when try to store and force re-write flag is enabled" in { - when(mongoDbCollectionMock.findOne(query)) thenThrow new CachedResultNotFoundException("") - when(mongoDbCollectionMock.insert(cachedExecutionResultDbObject)) thenReturn new WriteResult(0, true, "") - when(mongoDbCollectionMock.remove(query)) thenReturn new WriteResult(0, true, "") - val cacheActor = TestActorRef(new MongoCacheActor(mongoDbCollectionMock)) - cacheActor ! StoreExecutionResult(jobHash, succeededResponseMock) - expectMsg(ExecutionResultStored("88dde49db10f1551299fb9937f313c10")) - verify(mongoDbCollectionMock, atLeastOnce).insert(cachedExecutionResultDbObject) - verify(mongoDbCollectionMock, atLeastOnce).remove(query) - } - } -} diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesRuntimeAttributes.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesRuntimeAttributes.scala index 311d9f4b7..2cc7cb17d 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesRuntimeAttributes.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesRuntimeAttributes.scala @@ -63,7 +63,9 @@ object JesRuntimeAttributes { .withDefault(preemptibleValidationInstance.configDefaultWdlValue(runtimeConfig) getOrElse PreemptibleDefaultValue) private def memoryValidation(runtimeConfig: Option[Config]): RuntimeAttributesValidation[MemorySize] = { - MemoryValidation.withDefaultMemory(MemoryValidation.configDefaultString(runtimeConfig) getOrElse MemoryDefaultValue) + MemoryValidation.withDefaultMemory( + RuntimeAttributesKeys.MemoryKey, + MemoryValidation.configDefaultString(RuntimeAttributesKeys.MemoryKey, runtimeConfig) getOrElse MemoryDefaultValue) } private def bootDiskSizeValidation(runtimeConfig: Option[Config]): RuntimeAttributesValidation[Int] = bootDiskValidationInstance diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigConstants.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigConstants.scala index 3a951be9b..682b794f4 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigConstants.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigConstants.scala @@ -25,7 +25,8 @@ object ConfigConstants { val MemoryRuntimeAttribute = "memory" // See: MemoryDeclarationValidation val MemoryRuntimeAttributePrefix = "memory_" - + val DiskRuntimeAttribute = "disk" + val DiskRuntimeAttributePrefix = "disk_" /* List of task names used internally. NOTE: underscore separated diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/DeclarationValidation.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/DeclarationValidation.scala index c13279d6d..480e2143c 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/DeclarationValidation.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/DeclarationValidation.scala @@ -5,6 +5,7 @@ import wdl4s.expression.NoFunctions import wdl4s.types._ import wdl4s.values.WdlValue import wdl4s.{Declaration, NoLookup, WdlExpression} +import cromwell.backend.impl.sfs.config.ConfigConstants._ /** * Creates instances of runtime attribute validations from WDL declarations. @@ -27,8 +28,10 @@ object DeclarationValidation { new DeclarationValidation(declaration, DockerValidation.instance) case name if name == CpuValidation.instance.key => new DeclarationValidation(declaration, CpuValidation.instance) // See MemoryDeclarationValidation for more info - case name if MemoryDeclarationValidation.isMemoryDeclaration(name) => - new MemoryDeclarationValidation(declaration) + case name if MemoryDeclarationValidation.isMemoryDeclaration(name, MemoryRuntimeAttribute, MemoryRuntimeAttributePrefix) => + new MemoryDeclarationValidation(declaration, MemoryRuntimeAttribute, MemoryRuntimeAttributePrefix) + case name if MemoryDeclarationValidation.isMemoryDeclaration(name, DiskRuntimeAttribute, DiskRuntimeAttributePrefix) => + new MemoryDeclarationValidation(declaration, DiskRuntimeAttribute, DiskRuntimeAttributePrefix) // All other declarations must be a Boolean, Float, Integer, or String. case _ => val validatedRuntimeAttr = validator(declaration.wdlType, declaration.unqualifiedName) diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/MemoryDeclarationValidation.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/MemoryDeclarationValidation.scala index 39fd88046..ae2003ee0 100644 --- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/MemoryDeclarationValidation.scala +++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/MemoryDeclarationValidation.scala @@ -1,7 +1,6 @@ package cromwell.backend.impl.sfs.config import cromwell.backend.MemorySize -import cromwell.backend.impl.sfs.config.ConfigConstants._ import cromwell.backend.validation._ import wdl4s.expression.NoFunctions import wdl4s.parser.MemoryUnit @@ -31,8 +30,8 @@ import wdl4s.{Declaration, NoLookup, WdlExpression} * * @param declaration The declaration used to create this memory validation. */ -class MemoryDeclarationValidation(declaration: Declaration) - extends DeclarationValidation(declaration, MemoryValidation.instance) { +class MemoryDeclarationValidation(declaration: Declaration, attributeName: String, attributeNamePrefix: String) + extends DeclarationValidation(declaration, MemoryValidation.instance(attributeName)) { import MemoryDeclarationValidation._ @@ -64,7 +63,7 @@ class MemoryDeclarationValidation(declaration: Declaration) } private lazy val declarationMemoryUnit: MemoryUnit = { - val suffix = memoryUnitSuffix(declaration.unqualifiedName) + val suffix = memoryUnitSuffix(declaration.unqualifiedName, attributeName, attributeNamePrefix) val memoryUnitOption = MemoryUnit.values.find(_.suffixes.map(_.toLowerCase).contains(suffix.toLowerCase)) memoryUnitOption match { case Some(memoryUnit) => memoryUnit @@ -79,7 +78,7 @@ class MemoryDeclarationValidation(declaration: Declaration) * @return The value from the collection wrapped in `Some`, or `None` if the value wasn't found. */ override def extractWdlValueOption(validatedRuntimeAttributes: ValidatedRuntimeAttributes): Option[WdlValue] = { - RuntimeAttributesValidation.extractOption(MemoryValidation.instance, validatedRuntimeAttributes) map + RuntimeAttributesValidation.extractOption(MemoryValidation.instance(attributeName), validatedRuntimeAttributes) map coerceMemorySize(declaration.wdlType) } @@ -94,11 +93,11 @@ class MemoryDeclarationValidation(declaration: Declaration) } object MemoryDeclarationValidation { - def isMemoryDeclaration(name: String): Boolean = { + def isMemoryDeclaration(name: String, attributeName: String, attributeNamePrefix: String): Boolean = { name match { - case MemoryRuntimeAttribute => true - case prefixed if prefixed.startsWith(MemoryRuntimeAttributePrefix) => - val suffix = memoryUnitSuffix(name) + case `attributeName` => true + case prefixed if prefixed.startsWith(attributeNamePrefix) => + val suffix = memoryUnitSuffix(name, attributeName, attributeNamePrefix) MemoryUnit.values exists { _.suffixes.map(_.toLowerCase).contains(suffix) } @@ -106,10 +105,10 @@ object MemoryDeclarationValidation { } } - private def memoryUnitSuffix(name: String) = { - if (name == MemoryRuntimeAttribute) + private def memoryUnitSuffix(name: String, attributeName: String, attributeNamePrefix: String) = { + if (name == attributeName) MemoryUnit.Bytes.suffixes.head else - name.substring(MemoryRuntimeAttributePrefix.length) + name.substring(attributeNamePrefix.length) } } diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/impl/sfs/config/MemoryDeclarationValidationSpec.scala b/supportedBackends/sfs/src/test/scala/cromwell/backend/impl/sfs/config/MemoryDeclarationValidationSpec.scala index 3cc444c53..bd57a9e77 100644 --- a/supportedBackends/sfs/src/test/scala/cromwell/backend/impl/sfs/config/MemoryDeclarationValidationSpec.scala +++ b/supportedBackends/sfs/src/test/scala/cromwell/backend/impl/sfs/config/MemoryDeclarationValidationSpec.scala @@ -7,6 +7,7 @@ import org.scalatest.prop.TableDrivenPropertyChecks import org.scalatest.{FlatSpec, Matchers} import wdl4s.parser.MemoryUnit import wdl4s.values.{WdlFloat, WdlInteger} +import ConfigConstants._ class MemoryDeclarationValidationSpec extends FlatSpec with Matchers with TableDrivenPropertyChecks { behavior of "MemoryDeclarationValidation" @@ -40,7 +41,8 @@ class MemoryDeclarationValidationSpec extends FlatSpec with Matchers with TableD val configWdlNamespace = new ConfigWdlNamespace(config) val runtimeDeclaration = configWdlNamespace.runtimeDeclarations.head - val memoryDeclarationValidation = new MemoryDeclarationValidation(runtimeDeclaration) + val memoryDeclarationValidation = new MemoryDeclarationValidation(runtimeDeclaration, + MemoryRuntimeAttribute, MemoryRuntimeAttributePrefix) val attributes = runtimeAmount .map(amount => RuntimeAttributesKeys.MemoryKey -> MemorySize(amount.toDouble, MemoryUnit.GB)) .toMap @@ -52,7 +54,8 @@ class MemoryDeclarationValidationSpec extends FlatSpec with Matchers with TableD val expectedDefault = expectedDefaultAmount .map(amount => WdlInteger(MemorySize(amount.toDouble, MemoryUnit.GB).bytes.toInt)) - MemoryDeclarationValidation.isMemoryDeclaration(runtimeDeclaration.unqualifiedName) should be(true) + MemoryDeclarationValidation.isMemoryDeclaration(runtimeDeclaration.unqualifiedName, + MemoryRuntimeAttribute, MemoryRuntimeAttributePrefix) should be(true) default should be(expectedDefault) extracted should be(expectedExtracted) } @@ -92,7 +95,8 @@ class MemoryDeclarationValidationSpec extends FlatSpec with Matchers with TableD val configWdlNamespace = new ConfigWdlNamespace(config) val runtimeDeclaration = configWdlNamespace.runtimeDeclarations.head - MemoryDeclarationValidation.isMemoryDeclaration(runtimeDeclaration.unqualifiedName) should be(false) + MemoryDeclarationValidation.isMemoryDeclaration(runtimeDeclaration.unqualifiedName, + MemoryRuntimeAttribute, MemoryRuntimeAttributePrefix) should be(false) } } } diff --git a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesRuntimeAttributes.scala b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesRuntimeAttributes.scala index b159fee47..2d28e6212 100644 --- a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesRuntimeAttributes.scala +++ b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesRuntimeAttributes.scala @@ -39,7 +39,9 @@ object TesRuntimeAttributes { .withDefaultDiskSize(DiskSizeValidation.configDefaultString(runtimeConfig) getOrElse DiskSizeDefaultValue) private def memoryValidation(runtimeConfig: Option[Config]): RuntimeAttributesValidation[MemorySize] = { - MemoryValidation.withDefaultMemory(MemoryValidation.configDefaultString(runtimeConfig) getOrElse MemoryDefaultValue) + MemoryValidation.withDefaultMemory( + RuntimeAttributesKeys.MemoryKey, + MemoryValidation.configDefaultString(RuntimeAttributesKeys.MemoryKey, runtimeConfig) getOrElse MemoryDefaultValue) } private val dockerValidation: RuntimeAttributesValidation[String] = DockerValidation.instance From 6d79d0afe3e3603168766ea27f5a33403b644851 Mon Sep 17 00:00:00 2001 From: Kate Voss Date: Wed, 19 Apr 2017 15:29:27 -0400 Subject: [PATCH 008/134] Removing CLI from the Readme (#2161) * Removing CLI from the Readme * making it one line in the TOC * updating to the published url --- README.md | 172 +------------------------------------------------------------- 1 file changed, 1 insertion(+), 171 deletions(-) diff --git a/README.md b/README.md index 2970c3d97..403c444e9 100644 --- a/README.md +++ b/README.md @@ -17,10 +17,7 @@ A [Workflow Management System](https://en.wikipedia.org/wiki/Workflow_management * [Building](#building) * [Installing](#installing) * [Upgrading from 0.19 to 0.21](#upgrading-from-019-to-021) -* [Command Line Usage](#command-line-usage) - * [run](#run) - * [server](#server) - * [version](#version) +* [**NEW** Command Line Usage](http://gatkforums.broadinstitute.org/wdl/discussion/8782/command-line-cromwell) (on the WDL/Cromwell Website) * [Getting Started with WDL](#getting-started-with-wdl) * [WDL Support](#wdl-support) * [Configuring Cromwell](#configuring-cromwell) @@ -139,173 +136,6 @@ OS X users can install Cromwell with Homebrew: `brew install cromwell`. See the [migration document](MIGRATION.md) for more details. -# Command Line Usage - -Run the JAR file with no arguments to get the usage message: - -``` - - -$ java -jar cromwell.jar -java -jar cromwell.jar - -Actions: -run [] [] - [] [] [] - - Given a WDL file and JSON file containing the value of the - workflow inputs, this will run the workflow locally and - print out the outputs in JSON format. The workflow - options file specifies some runtime configuration for the - workflow (see README for details). The workflow metadata - output is an optional file path to output the metadata. The - directory of WDL files is optional. However, it is required - if the primary workflow imports workflows that are outside - of the root directory of the Cromwell project. - - Use a single dash ("-") to skip optional files. Ex: - run noinputs.wdl - - metadata.json - - - server - - Starts a web server on port 8000. See the web server - documentation for more details about the API endpoints. - - -version - - Returns the version of the Cromwell engine. - -``` - -## run - -Given a WDL file and a JSON inputs file (see `inputs` subcommand), Run the workflow and print the outputs: - -``` -$ java -jar cromwell.jar run 3step.wdl inputs.json -... play-by-play output ... -{ - "three_step.ps.procs": "/var/folders/kg/c7vgxnn902lc3qvc2z2g81s89xhzdz/T/stdout1272284837004786003.tmp", - "three_step.cgrep.count": 0, - "three_step.wc.count": 13 -} -``` - -The JSON inputs can be left off if there's a file with the same name as the WDL file but with a `.inputs` extension. For example, this will assume that `3step.inputs` exists: - -``` -$ java -jar cromwell.jar run 3step.wdl -``` - -If your workflow has no inputs, you can specify `-` as the value for the inputs parameter: - -``` -$ java -jar cromwell.jar run my_workflow.wdl - -``` - -The third, optional parameter to the 'run' subcommand is a JSON file of workflow options. By default, the command line will look for a file with the same name as the WDL file but with the extension `.options`. But one can also specify a value of `-` manually to specify that there are no workflow options. - -See the section [workflow options](#workflow-options) for more details. - -``` -$ java -jar cromwell.jar run my_jes_wf.wdl my_jes_wf.json wf_options.json -``` - -The fourth, optional parameter to the 'run' subcommand is a path where the workflow metadata will be written. By default, no workflow metadata will be written. - -``` -$ java -jar cromwell.jar run my_wf.wdl - - my_wf.metadata.json -... play-by-play output ... -$ cat my_wf.metadata.json -{ - "workflowName": "w", - "calls": { - "w.x": [{ - "executionStatus": "Done", - "stdout": "/Users/jdoe/projects/cromwell/cromwell-executions/w/a349534f-137b-4809-9425-1893ac272084/call-x/stdout", - "shardIndex": -1, - "outputs": { - "o": "local\nremote" - }, - "runtimeAttributes": { - "failOnStderr": "false", - "continueOnReturnCode": "0" - }, - "cache": { - "allowResultReuse": true - }, - "inputs": { - "remote": "/Users/jdoe/remote.txt", - "local": "local.txt" - }, - "returnCode": 0, - "backend": "Local", - "end": "2016-07-11T10:27:56.074-04:00", - "stderr": "/Users/jdoe/projects/cromwell/cromwell-executions/w/a349534f-137b-4809-9425-1893ac272084/call-x/stderr", - "callRoot": "cromwell-executions/w/a349534f-137b-4809-9425-1893ac272084/call-x", - "attempt": 1, - "start": "2016-07-11T10:27:55.992-04:00" - }] - }, - "outputs": { - "w.x.o": "local\nremote" - }, - "workflowRoot": "cromwell-executions/w/a349534f-137b-4809-9425-1893ac272084", - "id": "a349534f-137b-4809-9425-1893ac272084", - "inputs": { - "w.x.remote": "/Users/jdoe/remote.txt", - "w.x.local": "local.txt" - }, - "submission": "2016-07-11T10:27:54.907-04:00", - "status": "Succeeded", - "end": "2016-07-11T10:27:56.108-04:00", - "start": "2016-07-11T10:27:54.919-04:00" -} -``` - -The fifth, optional parameter to the 'run' subcommand is a zip file which contains WDL source files. This zip file can be passed -and your primary workflow can import any WDL's from that collection and re-use those tasks. - -For example, consider you have a directory of WDL files: -``` -my_WDLs -└──cgrep.wdl -└──ps.wdl -└──wc.wdl -``` - -If you zip that directory to my_WDLs.zip, you have the option to pass it in as the last parameter in your run command -and be able to reference these WDLs as imports in your primary WDL. For example, your primary WDL can look like this: -``` -import "ps.wdl" as ps -import "cgrep.wdl" -import "wc.wdl" as wordCount - -workflow threestep { - -call ps.ps as getStatus -call cgrep.cgrep { input: str = getStatus.x } -call wordCount { input: str = ... } - -} - -``` -The command to run this WDL, without needing any inputs, workflow options or metadata files would look like: - -``` -$ java -jar cromwell.jar run threestep.wdl - - - /path/to/my_WDLs.zip -``` - -The sixth optional parameter is a path to a labels file. See [Labels](#labels) for information and the expected format. - -## server - -Start a server on port 8000, the API for the server is described in the [REST API](#rest-api) section. - -## version - -Returns the version of Cromwell engine. - # Getting Started with WDL For many examples on how to use WDL see [the WDL site](https://github.com/broadinstitute/wdl#getting-started-with-wdl) From 6eb104b44a03eed972c886e28a95cefa4632d019 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Wed, 5 Apr 2017 15:20:09 -0400 Subject: [PATCH 009/134] Cromwell API client (for Centaur) --- build.sbt | 4 + .../main/scala/cromwell/api/CromwellClient.scala | 117 +++++++++++++++++++++ .../cromwell/api/model/CromwellBackends.scala | 9 ++ .../scala/cromwell/api/model/CromwellStatus.scala | 9 ++ .../api/model/FailedWorkflowSubmission.scala | 9 ++ .../scala/cromwell/api/model/OutputResponse.scala | 9 ++ .../cromwell/api/model/SubmittedWorkflow.scala | 8 ++ .../main/scala/cromwell/api/model/WorkflowId.scala | 20 ++++ .../cromwell/api/model/WorkflowMetadata.scala | 3 + .../scala/cromwell/api/model/WorkflowStatus.scala | 39 +++++++ .../cromwell/api/model/WorkflowSubmission.scala | 26 +++++ project/Dependencies.scala | 11 ++ project/Settings.scala | 12 +++ project/plugins.sbt | 2 +- 14 files changed, 277 insertions(+), 1 deletion(-) create mode 100644 cromwellApiClient/src/main/scala/cromwell/api/CromwellClient.scala create mode 100644 cromwellApiClient/src/main/scala/cromwell/api/model/CromwellBackends.scala create mode 100644 cromwellApiClient/src/main/scala/cromwell/api/model/CromwellStatus.scala create mode 100644 cromwellApiClient/src/main/scala/cromwell/api/model/FailedWorkflowSubmission.scala create mode 100644 cromwellApiClient/src/main/scala/cromwell/api/model/OutputResponse.scala create mode 100644 cromwellApiClient/src/main/scala/cromwell/api/model/SubmittedWorkflow.scala create mode 100644 cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowId.scala create mode 100644 cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowMetadata.scala create mode 100644 cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowStatus.scala create mode 100644 cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowSubmission.scala diff --git a/build.sbt b/build.sbt index 9716e9dd8..f3d84863a 100644 --- a/build.sbt +++ b/build.sbt @@ -26,6 +26,10 @@ lazy val dockerHashing = (project in file("dockerHashing")) .dependsOn(core % "test->test") .withTestSettings +lazy val cromwellApiClient = (project in file("cromwellApiClient")) + .settings(cromwellApiClientSettings: _*) + .withTestSettings + lazy val services = (project in file("services")) .settings(servicesSettings:_*) .withTestSettings diff --git a/cromwellApiClient/src/main/scala/cromwell/api/CromwellClient.scala b/cromwellApiClient/src/main/scala/cromwell/api/CromwellClient.scala new file mode 100644 index 000000000..5aca78918 --- /dev/null +++ b/cromwellApiClient/src/main/scala/cromwell/api/CromwellClient.scala @@ -0,0 +1,117 @@ +package cromwell.api + +import java.net.URL + +import akka.http.scaladsl.Http +import akka.actor.ActorSystem +import akka.http.scaladsl.model.{HttpEntity, _} +import akka.http.scaladsl.unmarshalling.{Unmarshal, Unmarshaller} +import akka.stream.ActorMaterializer +import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._ +import akka.util.ByteString +import cromwell.api.model._ +import spray.json._ + +import scala.concurrent.{ExecutionContext, Future} +import cromwell.api.CromwellClient._ + +import scala.util.{Failure, Success, Try} + +class CromwellClient(val cromwellUrl: URL, val apiVersion: String)(implicit actorSystem: ActorSystem, materializer: ActorMaterializer) { + + lazy val submitEndpoint = s"$cromwellUrl/api/workflows/$apiVersion" + // Everything else is a suffix off the submit endpoint: + lazy val batchSubmitEndpoint = s"$submitEndpoint/batch" + private def workflowSpecificEndpoint(workflowId: WorkflowId, endpoint: String) = s"$submitEndpoint/$workflowId/$endpoint" + def abortEndpoint(workflowId: WorkflowId) = workflowSpecificEndpoint(workflowId, "abort") + def statusEndpoint(workflowId: WorkflowId) = workflowSpecificEndpoint(workflowId, "status") + def metadataEndpoint(workflowId: WorkflowId) = workflowSpecificEndpoint(workflowId, "metadata") + lazy val backendsEndpoint = s"$submitEndpoint/backends" + + import model.CromwellStatusJsonSupport._ + import model.CromwellBackendsJsonSupport._ + + private def requestEntityForSubmit(workflowSubmission: WorkflowSubmission) = { + val sourceBodyParts = Map( + "wdlSource" -> Option(workflowSubmission.wdl), + "workflowInputs" -> workflowSubmission.inputsJson, + "workflowOptions" -> insertSecrets(workflowSubmission.options, workflowSubmission.refreshToken) + ) collect { case (name, Some(source)) => Multipart.FormData.BodyPart(name, HttpEntity(MediaTypes.`application/json`, ByteString(source))) } + + val zipBodyParts = Map( + "wdlDependencies" -> workflowSubmission.zippedImports + ) collect { case (name, Some(file)) => Multipart.FormData.BodyPart.fromPath(name, MediaTypes.`application/zip`, file.path) } + + val multipartFormData = Multipart.FormData((sourceBodyParts ++ zipBodyParts).toSeq : _*) + multipartFormData.toEntity() + } + + def submit(workflow: WorkflowSubmission)(implicit ec: ExecutionContext): Future[SubmittedWorkflow] = { + val requestEntity = requestEntityForSubmit(workflow) + + makeRequest[CromwellStatus](HttpRequest(HttpMethods.POST, submitEndpoint, List.empty[HttpHeader], requestEntity)) map { status => + SubmittedWorkflow(WorkflowId.fromString(status.id), cromwellUrl, workflow) + } + } + + def submitBatch(workflow: WorkflowBatchSubmission)(implicit ec: ExecutionContext): Future[List[SubmittedWorkflow]] = { + import DefaultJsonProtocol._ + + val requestEntity = requestEntityForSubmit(workflow) + + // Make a set of submissions that represent the batch (so we can zip with the results later): + val submissionSet = workflow.inputsBatch.map(inputs => WorkflowSingleSubmission(workflow.wdl, Option(inputs), workflow.options, workflow.zippedImports, workflow.refreshToken)) + + makeRequest[List[CromwellStatus]](HttpRequest(HttpMethods.POST, batchSubmitEndpoint, List.empty[HttpHeader], requestEntity)) map { statuses => + val zipped = submissionSet.zip(statuses) + zipped map { case (submission, status) => + SubmittedWorkflow(WorkflowId.fromString(status.id), cromwellUrl, submission) + } + } + } + + def abort(workflowId: WorkflowId)(implicit ec: ExecutionContext): Future[WorkflowStatus] = getRequest[CromwellStatus](abortEndpoint(workflowId)) map WorkflowStatus.apply + def status(workflowId: WorkflowId)(implicit ec: ExecutionContext): Future[WorkflowStatus] = getRequest[CromwellStatus](statusEndpoint(workflowId)) map WorkflowStatus.apply + def metadata(workflowId: WorkflowId)(implicit ec: ExecutionContext): Future[WorkflowMetadata] = getRequest[String](metadataEndpoint(workflowId)) map WorkflowMetadata + def backends(implicit ec: ExecutionContext): Future[CromwellBackends] = getRequest[CromwellBackends](backendsEndpoint) + + /** + * + * @tparam A The type of response expected. Must be supported by an implicit unmarshaller from ResponseEntity. + */ + private def makeRequest[A](request: HttpRequest)(implicit um: Unmarshaller[ResponseEntity, A], ec: ExecutionContext): Future[A] = for { + response <- Http().singleRequest(request) + entity <- Future.fromTry(response.toEntity) + unmarshalled <- entity.to[A] + } yield unmarshalled + + private def getRequest[A](uri: String)(implicit um: Unmarshaller[ResponseEntity, A], ec: ExecutionContext): Future[A] = makeRequest[A](HttpRequest(uri = uri)) + + private def insertSecrets(options: Option[String], refreshToken: Option[String]): Option[String] = { + import DefaultJsonProtocol._ + val tokenKey = "refresh_token" + + def addToken(optionsMap: Map[String, JsValue]): Map[String, JsValue] = { + refreshToken match { + case Some(token) if optionsMap.get(tokenKey).isDefined => optionsMap + (tokenKey -> JsString(token)) + case _ => optionsMap + } + } + + options map (o => addToken(o.parseJson.asJsObject.convertTo[Map[String, JsValue]]).toJson.toString) + } +} + +object CromwellClient { + final implicit class EnhancedHttpResponse(val response: HttpResponse) extends AnyVal { + + def toEntity: Try[Unmarshal[ResponseEntity]] = response match { + case HttpResponse(_: StatusCodes.Success, _, entity, _) => Success(Unmarshal(entity)) + case other => Failure(new UnsuccessfulRequestException(other)) + } + } + + final case class UnsuccessfulRequestException(httpResponse: HttpResponse) extends Exception { + override def getMessage = httpResponse.toString + } +} diff --git a/cromwellApiClient/src/main/scala/cromwell/api/model/CromwellBackends.scala b/cromwellApiClient/src/main/scala/cromwell/api/model/CromwellBackends.scala new file mode 100644 index 000000000..007723310 --- /dev/null +++ b/cromwellApiClient/src/main/scala/cromwell/api/model/CromwellBackends.scala @@ -0,0 +1,9 @@ +package cromwell.api.model + +import spray.json.DefaultJsonProtocol + +object CromwellBackendsJsonSupport extends DefaultJsonProtocol { + implicit val CromwellBackendsFormat = jsonFormat2(CromwellBackends) +} + +final case class CromwellBackends(defaultBackend: String, supportedBackends: List[String]) diff --git a/cromwellApiClient/src/main/scala/cromwell/api/model/CromwellStatus.scala b/cromwellApiClient/src/main/scala/cromwell/api/model/CromwellStatus.scala new file mode 100644 index 000000000..31433e5c7 --- /dev/null +++ b/cromwellApiClient/src/main/scala/cromwell/api/model/CromwellStatus.scala @@ -0,0 +1,9 @@ +package cromwell.api.model + +import spray.json.DefaultJsonProtocol + +object CromwellStatusJsonSupport extends DefaultJsonProtocol { + implicit val CromwellStatusFormat = jsonFormat2(CromwellStatus) +} + +case class CromwellStatus(id: String, status: String) diff --git a/cromwellApiClient/src/main/scala/cromwell/api/model/FailedWorkflowSubmission.scala b/cromwellApiClient/src/main/scala/cromwell/api/model/FailedWorkflowSubmission.scala new file mode 100644 index 000000000..c9e14cb37 --- /dev/null +++ b/cromwellApiClient/src/main/scala/cromwell/api/model/FailedWorkflowSubmission.scala @@ -0,0 +1,9 @@ +package cromwell.api.model + +import spray.json.DefaultJsonProtocol + +object FailedWorkflowSubmissionJsonSupport extends DefaultJsonProtocol { + implicit val FailedWorkflowSubmissionFormat = jsonFormat2(FailedWorkflowSubmission) +} + +case class FailedWorkflowSubmission(status: String, message: String) diff --git a/cromwellApiClient/src/main/scala/cromwell/api/model/OutputResponse.scala b/cromwellApiClient/src/main/scala/cromwell/api/model/OutputResponse.scala new file mode 100644 index 000000000..657aca668 --- /dev/null +++ b/cromwellApiClient/src/main/scala/cromwell/api/model/OutputResponse.scala @@ -0,0 +1,9 @@ +package cromwell.api.model + +import spray.json.DefaultJsonProtocol + +object OutputResponseJsonSupport extends DefaultJsonProtocol { + implicit val OutputResponseFormat = jsonFormat2(OutputResponse) +} + +case class OutputResponse(id: String, outputs: Map[String, String]) diff --git a/cromwellApiClient/src/main/scala/cromwell/api/model/SubmittedWorkflow.scala b/cromwellApiClient/src/main/scala/cromwell/api/model/SubmittedWorkflow.scala new file mode 100644 index 000000000..bcebcd5c9 --- /dev/null +++ b/cromwellApiClient/src/main/scala/cromwell/api/model/SubmittedWorkflow.scala @@ -0,0 +1,8 @@ +package cromwell.api.model + +import java.net.URL + +/** + * Represents information which we need to capture about a workflow sent to Cromwell. + */ +case class SubmittedWorkflow(id: WorkflowId, cromwellServer: URL, workflow: WorkflowSubmission) diff --git a/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowId.scala b/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowId.scala new file mode 100644 index 000000000..2ad4760bb --- /dev/null +++ b/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowId.scala @@ -0,0 +1,20 @@ +package cromwell.api.model + +import java.util.UUID + +// ********* !!!!!!!!!! ******** +// +// WARNING! This is the Cromwell API version of WorkflowId. If you aren't changing the API client, you probably +// want cromwell.core.WorkflowId instead! +// +// ********* !!!!!!!!!! ******** + +case class WorkflowId(id: UUID) { + override def toString = id.toString + def shortString = id.toString.split("-")(0) +} + +object WorkflowId { + def fromString(id: String): WorkflowId = new WorkflowId(UUID.fromString(id)) + def randomId() = WorkflowId(UUID.randomUUID()) +} diff --git a/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowMetadata.scala b/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowMetadata.scala new file mode 100644 index 000000000..5b7ff88ab --- /dev/null +++ b/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowMetadata.scala @@ -0,0 +1,3 @@ +package cromwell.api.model + +case class WorkflowMetadata(value: String) extends AnyVal diff --git a/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowStatus.scala b/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowStatus.scala new file mode 100644 index 000000000..adadea912 --- /dev/null +++ b/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowStatus.scala @@ -0,0 +1,39 @@ +package cromwell.api.model + +// ********* !!!!!!!!!! ******** +// +// WARNING! This is a Cromwell API class. If you aren't changing the API client, you probably +// want to look elsewhere (maybe in cromwell.core?) +// +// ********* !!!!!!!!!! ******** + +/** + * ADT tree to describe Cromwell workflow statuses, both terminal and non-terminal + */ +sealed trait WorkflowStatus + +sealed trait TerminalStatus extends WorkflowStatus +case object Aborted extends TerminalStatus +case object Failed extends TerminalStatus +case object Succeeded extends TerminalStatus + +sealed trait NonTerminalStatus extends WorkflowStatus +case object Submitted extends NonTerminalStatus +case object Running extends NonTerminalStatus +case object Aborting extends NonTerminalStatus + +object WorkflowStatus { + def apply(status: String): WorkflowStatus = { + status match { + case "Submitted" => Submitted + case "Running" => Running + case "Aborting" => Aborting + case "Aborted" => Aborted + case "Failed" => Failed + case "Succeeded" => Succeeded + case bad => throw new IllegalArgumentException(s"No such status: $bad") + } + } + + def apply(workflowStatus: CromwellStatus): WorkflowStatus = apply(workflowStatus.status) +} diff --git a/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowSubmission.scala b/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowSubmission.scala new file mode 100644 index 000000000..49ce10bc0 --- /dev/null +++ b/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowSubmission.scala @@ -0,0 +1,26 @@ +package cromwell.api.model + +import better.files.File + +sealed trait WorkflowSubmission { + val wdl: String + val inputsJson: Option[String] + val options: Option[String] + val zippedImports: Option[File] + val refreshToken: Option[String] +} + +final case class WorkflowSingleSubmission(wdl: String, + inputsJson: Option[String], + options: Option[String], + zippedImports: Option[File], + refreshToken: Option[String]) extends WorkflowSubmission + +final case class WorkflowBatchSubmission(wdl: String, + inputsBatch: List[String], + options: Option[String], + zippedImports: Option[File], + refreshToken: Option[String]) extends WorkflowSubmission { + + override val inputsJson: Option[String] = Option(inputsBatch.mkString(start="[", sep=",", end="]")) +} diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 266b8cafd..ff553b40d 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -15,6 +15,9 @@ object Dependencies { lazy val akkaV = "2.4.16" lazy val akkaHttpV = "2.4.11" lazy val slickV = "3.2.0" + // TODO: Re-combine these when cromwell is 2.12: + lazy val cromwellApiClientAkkaV = "2.4.17" + lazy val cromwellApiClientAkkaHttpV = "10.0.5" lazy val googleClientApiV = "1.22.0" lazy val googleGenomicsServicesApiV = "1.22.0" lazy val betterFilesV = "2.17.1" @@ -139,6 +142,14 @@ object Dependencies { "com.github.pathikrit" %% "better-files" % betterFilesV % Test ) ++ liquibaseDependencies ++ dbmsDependencies + val cromwellApiClientDependencies = List( + "com.typesafe.akka" %% "akka-actor" % cromwellApiClientAkkaV, + "com.typesafe.akka" %% "akka-http" % cromwellApiClientAkkaHttpV, + "com.typesafe.akka" %% "akka-http-spray-json" % cromwellApiClientAkkaHttpV, + "com.github.pathikrit" %% "better-files" % "3.0.0", + "org.scalatest" %% "scalatest" % "3.0.1" % Test + ) + val jesBackendDependencies = refinedTypeDependenciesList val tesBackendDependencies = List( diff --git a/project/Settings.scala b/project/Settings.scala index 938a81233..56303da63 100644 --- a/project/Settings.scala +++ b/project/Settings.scala @@ -135,6 +135,18 @@ object Settings { libraryDependencies ++= databaseMigrationDependencies ) ++ commonSettings + val cromwellApiClientSettings = List( + name := "cromwell-api-client", + libraryDependencies ++= cromwellApiClientDependencies, + organization := "org.broadinstitute", + scalaVersion := "2.12.1", + resolvers ++= commonResolvers + // scalacOptions ++= compilerSettings, + // scalacOptions in (Compile, doc) ++= docSettings, + // parallelExecution := false + ) ++ ReleasePlugin.projectSettings ++ testSettings ++ assemblySettings ++ + cromwellVersionWithGit ++ publishingSettings + val dockerHashingSettings = List( name := "cromwell-docker-hashing" ) ++ commonSettings diff --git a/project/plugins.sbt b/project/plugins.sbt index 4de0d6862..ef96d4a95 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -7,5 +7,5 @@ See https://github.com/broadinstitute/cromwell/issues/645 */ addSbtPlugin("com.typesafe.sbt" % "sbt-git" % "0.7.1") addSbtPlugin("com.github.gseitz" % "sbt-release" % "1.0.3") -addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.3.5") +addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.5.0") addSbtPlugin("org.scoverage" % "sbt-coveralls" % "1.1.0") From 2033421c0bc37ec0c36bfeb481c7fe75c19e76e4 Mon Sep 17 00:00:00 2001 From: Thib Date: Fri, 21 Apr 2017 11:36:34 -0400 Subject: [PATCH 010/134] Support for docker hash lookup of public quay.io images (#2184) * add docker hash lookup for public quay.io images * custom centaur --- .travis.yml | 2 +- CHANGELOG.md | 3 ++ .../registryv2/DockerRegistryV2AbstractFlow.scala | 35 ++++++++++++++-------- .../docker/registryv2/flows/quay/QuayFlow.scala | 28 +++++++++++++++++ .../scala/cromwell/server/CromwellRootActor.scala | 4 ++- 5 files changed, 58 insertions(+), 14 deletions(-) create mode 100644 dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/quay/QuayFlow.scala diff --git a/.travis.yml b/.travis.yml index 1be54097a..3d705e045 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,7 +19,7 @@ before_install: - openssl aes-256-cbc -K "$encrypted_5ebd3ff04788_key" -iv "$encrypted_5ebd3ff04788_iv" -in src/bin/travis/resources/jesConf.tar.enc -out jesConf.tar -d || true env: global: - - CENTAUR_BRANCH=develop + - CENTAUR_BRANCH=quayDockerLookup matrix: # Setting this variable twice will cause the 'script' section to run twice with the respective env var invoked - BUILD_TYPE=sbt diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e9efc661..c67a84b64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,9 @@ An option to specify how a Docker hash should be looked up has been added. Two m "remote" will try to look up the image hash directly on the remote repository where the image is located (Docker Hub and GCR are supported) Note that the "local" option will require docker to be installed on the machine running cromwell, in order for it to call the docker CLI. +### Docker Hash Lookup for public Quay.io images + +* Adds hash lookup support for public [quay.io](https://quay.io/) images. ## 26 diff --git a/dockerHashing/src/main/scala/cromwell/docker/registryv2/DockerRegistryV2AbstractFlow.scala b/dockerHashing/src/main/scala/cromwell/docker/registryv2/DockerRegistryV2AbstractFlow.scala index 9429d74f9..2f1d8fec3 100644 --- a/dockerHashing/src/main/scala/cromwell/docker/registryv2/DockerRegistryV2AbstractFlow.scala +++ b/dockerHashing/src/main/scala/cromwell/docker/registryv2/DockerRegistryV2AbstractFlow.scala @@ -43,12 +43,17 @@ abstract class DockerRegistryV2AbstractFlow(httpClientFlow: HttpDockerFlow)(impl // Wraps the Http flow in a retryable flow to enable auto retries final private val httpFlowWithRetry = new HttpFlowWithRetry[DockerHashContext](httpClientFlow).flow - final private val tokenFlow = { - val responseHandlerFlow = Flow[(HttpResponse, DockerHashContext)].mapAsync(1)(Function.tupled(tokenResponseHandler)) + private [registryv2] val tokenFlow = { + val responseHandlerFlow = Flow[(HttpResponse, DockerHashContext)] + .mapAsync(1)(Function.tupled(tokenResponseHandler)) + // Map the Try[String] token to a Try[Option[String]]. + // This allows potential users of this class to override this flow and return an empty token + .map { case (tryToken, context) => (tryToken map Option.apply, context) } + requestTransformFlow(buildTokenRequest, responseHandlerFlow) } - final private val manifestFlow = { + private [registryv2] val manifestFlow = { val responseHandlerFlow = Flow[(HttpResponse, DockerHashContext)].map(Function.tupled(manifestResponseHandler)) requestTransformFlow(Function.tupled(buildManifestRequest _), responseHandlerFlow) } @@ -206,7 +211,7 @@ abstract class DockerRegistryV2AbstractFlow(httpClientFlow: HttpDockerFlow)(impl /** * Extract the access token from the json body of the http response */ - private def extractToken(jsObject: JsObject) = { + private def extractToken(jsObject: JsObject): Try[String] = { jsObject.fields.get("token") match { case Some(token: JsString) => Success(token.value) case Some(other) => Failure(new Exception("Token response contains a non-string token field")) @@ -224,14 +229,20 @@ abstract class DockerRegistryV2AbstractFlow(httpClientFlow: HttpDockerFlow)(impl /** * Builds the manifest http request */ - private def buildManifestRequest(token: String, dockerHashContext: DockerHashContext) = { - val authorizationHeader = Authorization(OAuth2BearerToken(token)) - - val manifestRequest = HttpRequest( - method = manifestRequestHttpMethod, - uri = buildManifestUri(dockerHashContext.dockerImageID), - headers = scala.collection.immutable.Seq(AcceptHeader, authorizationHeader) - ) + private def buildManifestRequest(token: Option[String], dockerHashContext: DockerHashContext) = { + val manifestRequest = token match { + case Some(authToken) => + HttpRequest( + method = manifestRequestHttpMethod, + uri = buildManifestUri(dockerHashContext.dockerImageID), + headers = scala.collection.immutable.Seq(AcceptHeader, Authorization(OAuth2BearerToken(authToken))) + ) + case None => + HttpRequest( + method = manifestRequestHttpMethod, + uri = buildManifestUri(dockerHashContext.dockerImageID) + ) + } (manifestRequest, dockerHashContext) } diff --git a/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/quay/QuayFlow.scala b/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/quay/QuayFlow.scala new file mode 100644 index 000000000..113ba5ca8 --- /dev/null +++ b/dockerHashing/src/main/scala/cromwell/docker/registryv2/flows/quay/QuayFlow.scala @@ -0,0 +1,28 @@ +package cromwell.docker.registryv2.flows.quay + +import akka.actor.Scheduler +import akka.http.scaladsl.model.HttpHeader +import akka.stream.scaladsl.{Flow, GraphDSL, Source} +import akka.stream.{ActorMaterializer, FanOutShape2} +import cromwell.docker.DockerHashActor.{DockerHashContext, DockerHashResponse} +import cromwell.docker.registryv2.DockerRegistryV2AbstractFlow +import cromwell.docker.registryv2.DockerRegistryV2AbstractFlow.HttpDockerFlow + +import scala.concurrent.ExecutionContext + +class QuayFlow(httpClientFlow: HttpDockerFlow)(implicit ec: ExecutionContext, materializer: ActorMaterializer, scheduler: Scheduler) extends DockerRegistryV2AbstractFlow(httpClientFlow) { + override protected def registryHostName: String = "quay.io" + // Not used for now because we bypass the token part as quay doesn't require one for public images + override protected def authorizationServerHostName: String = "quay.io" + // Not used for now, same reason as above + override protected def buildTokenRequestHeaders(dockerHashContext: DockerHashContext): List[HttpHeader] = List.empty + + override private [registryv2] val tokenFlow = GraphDSL.create() { implicit b => + // Flow that always returns a successful empty token + val noTokenFlow = b.add(Flow[DockerHashContext].map((None, _))) + // this token flow never fails as it returns an empty token. + // However it still needs a failure port. This creates an empty source that will act as the failure output port. + val emptyFailurePort = b.add(Source.empty[(DockerHashResponse, DockerHashContext)]) + new FanOutShape2(noTokenFlow.in, noTokenFlow.out, emptyFailurePort.out) + } +} diff --git a/engine/src/main/scala/cromwell/server/CromwellRootActor.scala b/engine/src/main/scala/cromwell/server/CromwellRootActor.scala index 10a531e0c..3da296544 100644 --- a/engine/src/main/scala/cromwell/server/CromwellRootActor.scala +++ b/engine/src/main/scala/cromwell/server/CromwellRootActor.scala @@ -16,6 +16,7 @@ import cromwell.docker.registryv2.flows.HttpFlowWithRetry.ContextWithRequest import cromwell.docker.registryv2.flows.dockerhub.DockerHubFlow import cromwell.docker.registryv2.flows.gcr.GoogleFlow import cromwell.core.io.Throttle +import cromwell.docker.registryv2.flows.quay.QuayFlow import cromwell.engine.backend.{BackendSingletonCollection, CromwellBackends} import cromwell.engine.io.IoActor import cromwell.engine.workflow.WorkflowManagerActor @@ -90,10 +91,11 @@ import scala.language.postfixOps lazy val dockerHttpPool = Http().superPool[ContextWithRequest[DockerHashContext]]() lazy val googleFlow = new GoogleFlow(dockerHttpPool, dockerConf.gcrApiQueriesPer100Seconds)(ioEc, materializer, system.scheduler) lazy val dockerHubFlow = new DockerHubFlow(dockerHttpPool)(ioEc, materializer, system.scheduler) + lazy val quayFlow = new QuayFlow(dockerHttpPool)(ioEc, materializer, system.scheduler) lazy val dockerCliFlow = new DockerCliFlow()(ioEc, materializer, system.scheduler) lazy val dockerFlows = dockerConf.method match { case DockerLocalLookup => Seq(dockerCliFlow) - case DockerRemoteLookup => Seq(dockerHubFlow, googleFlow) + case DockerRemoteLookup => Seq(dockerHubFlow, googleFlow, quayFlow) } lazy val dockerHashActor = context.actorOf(DockerHashActor.props(dockerFlows, dockerActorQueueSize, dockerConf.cacheEntryTtl, dockerConf.cacheSize)(materializer).withDispatcher(Dispatcher.IoDispatcher)) From 229e9748e4dd316a0d1b9595b7b8266af82b2b57 Mon Sep 17 00:00:00 2001 From: Thibault Jeandet Date: Fri, 21 Apr 2017 11:37:12 -0400 Subject: [PATCH 011/134] centaur develop --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3d705e045..1be54097a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,7 +19,7 @@ before_install: - openssl aes-256-cbc -K "$encrypted_5ebd3ff04788_key" -iv "$encrypted_5ebd3ff04788_iv" -in src/bin/travis/resources/jesConf.tar.enc -out jesConf.tar -d || true env: global: - - CENTAUR_BRANCH=quayDockerLookup + - CENTAUR_BRANCH=develop matrix: # Setting this variable twice will cause the 'script' section to run twice with the respective env var invoked - BUILD_TYPE=sbt From d1c8ea0c868c87dee8c7ed4eb09c33f78ebf137b Mon Sep 17 00:00:00 2001 From: Thib Date: Fri, 21 Apr 2017 11:40:05 -0400 Subject: [PATCH 012/134] Some Joint Genotyping related improvements (#2180) * Add 410 as retryable error and rework metadata building a little. * custom centaur * add CRDT test for call executions status * update timing diagram * eventuallify test * PR comments * oops * centaur develop --- .../main/scala/cromwell/core/ExecutionStatus.scala | 14 ++ .../main/scala/cromwell/core/WorkflowState.scala | 4 +- .../slick/tables/MetadataEntryComponent.scala | 28 +-- .../resources/workflowTimings/workflowTimings.html | 2 +- .../main/scala/cromwell/engine/io/IoActor.scala | 10 +- .../workflow/SingleWorkflowRunnerActor.scala | 2 +- .../lifecycle/execution/CallMetadataHelper.scala | 2 +- .../cromwell/webservice/CromwellApiService.scala | 3 +- .../scala/cromwell/webservice/PerRequest.scala | 2 +- .../webservice/metadata/IndexedJsonValue.scala | 63 ------ .../webservice/metadata/MetadataBuilderActor.scala | 213 +++++++++++---------- .../webservice/metadata/MetadataComponent.scala | 50 +++++ .../webservice/CromwellApiServiceSpec.scala | 13 +- .../webservice/MetadataBuilderActorSpec.scala | 43 ++++- project/plugins.sbt | 2 +- .../services/metadata/WorkflowQueryKey.scala | 2 +- .../metadata/impl/MetadataDatabaseAccess.scala | 6 +- .../metadata/impl/MetadataDatabaseAccessSpec.scala | 26 +++ .../metadata/impl/WriteMetadataActorSpec.scala | 25 ++- 19 files changed, 296 insertions(+), 214 deletions(-) delete mode 100644 engine/src/main/scala/cromwell/webservice/metadata/IndexedJsonValue.scala create mode 100644 engine/src/main/scala/cromwell/webservice/metadata/MetadataComponent.scala diff --git a/core/src/main/scala/cromwell/core/ExecutionStatus.scala b/core/src/main/scala/cromwell/core/ExecutionStatus.scala index 652a9995b..25319404c 100644 --- a/core/src/main/scala/cromwell/core/ExecutionStatus.scala +++ b/core/src/main/scala/cromwell/core/ExecutionStatus.scala @@ -5,6 +5,20 @@ object ExecutionStatus extends Enumeration { val NotStarted, QueuedInCromwell, Starting, Running, Failed, RetryableFailure, Done, Bypassed, Aborted = Value val TerminalStatuses = Set(Failed, Done, Aborted, RetryableFailure, Bypassed) + implicit val ExecutionStatusOrdering = Ordering.by { status: ExecutionStatus => + status match { + case NotStarted => 0 + case QueuedInCromwell => 1 + case Starting => 2 + case Running => 3 + case Aborted => 4 + case Bypassed => 5 + case RetryableFailure => 6 + case Failed => 7 + case Done => 8 + } + } + implicit class EnhancedExecutionStatus(val status: ExecutionStatus) extends AnyVal { def isTerminal: Boolean = { TerminalStatuses contains status diff --git a/core/src/main/scala/cromwell/core/WorkflowState.scala b/core/src/main/scala/cromwell/core/WorkflowState.scala index 41ac2bb97..98cef7896 100644 --- a/core/src/main/scala/cromwell/core/WorkflowState.scala +++ b/core/src/main/scala/cromwell/core/WorkflowState.scala @@ -10,9 +10,9 @@ sealed trait WorkflowState { } object WorkflowState { - private lazy val WorkflowState = Seq(WorkflowSubmitted, WorkflowRunning, WorkflowFailed, WorkflowSucceeded, WorkflowAborting, WorkflowAborted) + private lazy val WorkflowStateValues = Seq(WorkflowSubmitted, WorkflowRunning, WorkflowFailed, WorkflowSucceeded, WorkflowAborting, WorkflowAborted) - def fromString(str: String): WorkflowState = WorkflowState.find(_.toString == str).getOrElse( + def withName(str: String): WorkflowState = WorkflowStateValues.find(_.toString == str).getOrElse( throw new NoSuchElementException(s"No such WorkflowState: $str")) implicit val WorkflowStateSemigroup = new Semigroup[WorkflowState] { diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/MetadataEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/MetadataEntryComponent.scala index 745a1e593..dd1b4d3e6 100644 --- a/database/sql/src/main/scala/cromwell/database/slick/tables/MetadataEntryComponent.scala +++ b/database/sql/src/main/scala/cromwell/database/slick/tables/MetadataEntryComponent.scala @@ -62,10 +62,10 @@ trait MetadataEntryComponent { val metadataEntryIdsAutoInc = metadataEntries returning metadataEntries.map(_.metadataEntryId) val metadataEntriesForWorkflowExecutionUuid = Compiled( - (workflowExecutionUuid: Rep[String]) => for { + (workflowExecutionUuid: Rep[String]) => (for { metadataEntry <- metadataEntries if metadataEntry.workflowExecutionUuid === workflowExecutionUuid - } yield metadataEntry + } yield metadataEntry).sortBy(_.metadataTimestamp) ) val metadataEntryExistsForWorkflowExecutionUuid = Compiled( @@ -76,31 +76,31 @@ trait MetadataEntryComponent { ) val metadataEntriesForWorkflowExecutionUuidAndMetadataKey = Compiled( - (workflowExecutionUuid: Rep[String], metadataKey: Rep[String]) => for { + (workflowExecutionUuid: Rep[String], metadataKey: Rep[String]) => (for { metadataEntry <- metadataEntries if metadataEntry.workflowExecutionUuid === workflowExecutionUuid if metadataEntry.metadataKey === metadataKey if metadataEntry.callFullyQualifiedName.isEmpty if metadataEntry.jobIndex.isEmpty if metadataEntry.jobAttempt.isEmpty - } yield metadataEntry + } yield metadataEntry).sortBy(_.metadataTimestamp) ) val metadataEntriesForJobKey = Compiled( (workflowExecutionUuid: Rep[String], callFullyQualifiedName: Rep[String], jobIndex: Rep[Option[Int]], - jobAttempt: Rep[Int]) => for { + jobAttempt: Rep[Int]) => (for { metadataEntry <- metadataEntries if metadataEntry.workflowExecutionUuid === workflowExecutionUuid if metadataEntry.callFullyQualifiedName === callFullyQualifiedName if (metadataEntry.jobIndex === jobIndex) || (metadataEntry.jobIndex.isEmpty && jobIndex.isEmpty) if metadataEntry.jobAttempt === jobAttempt - } yield metadataEntry + } yield metadataEntry).sortBy(_.metadataTimestamp) ) val metadataEntriesForJobKeyAndMetadataKey = Compiled( (workflowExecutionUuid: Rep[String], metadataKey: Rep[String], callFullyQualifiedName: Rep[String], - jobIndex: Rep[Option[Int]], jobAttempt: Rep[Int]) => for { + jobIndex: Rep[Option[Int]], jobAttempt: Rep[Int]) => (for { metadataEntry <- metadataEntries if metadataEntry.workflowExecutionUuid === workflowExecutionUuid if metadataEntry.metadataKey === metadataKey @@ -108,39 +108,39 @@ trait MetadataEntryComponent { if (metadataEntry.jobIndex === jobIndex) || (metadataEntry.jobIndex.isEmpty && jobIndex.isEmpty) if metadataEntry.jobAttempt === jobAttempt - } yield metadataEntry + } yield metadataEntry).sortBy(_.metadataTimestamp) ) val metadataEntriesForIdGreaterThanOrEqual = Compiled( (metadataEntryId: Rep[Long], metadataKey1: Rep[String], metadataKey2: Rep[String], metadataKey3: Rep[String], - metadataKey4: Rep[String]) => for { + metadataKey4: Rep[String]) => (for { metadataEntry <- metadataEntries if metadataEntry.metadataEntryId >= metadataEntryId if (metadataEntry.metadataKey === metadataKey1 || metadataEntry.metadataKey === metadataKey2 || metadataEntry.metadataKey === metadataKey3 || metadataEntry.metadataKey === metadataKey4) && (metadataEntry.callFullyQualifiedName.isEmpty && metadataEntry.jobIndex.isEmpty && metadataEntry.jobAttempt.isEmpty) - } yield metadataEntry + } yield metadataEntry).sortBy(_.metadataTimestamp) ) def metadataEntriesLikeMetadataKeys(workflowExecutionUuid: String, metadataKeys: NonEmptyList[String], requireEmptyJobKey: Boolean) = { - for { + (for { metadataEntry <- metadataEntries if metadataEntry.workflowExecutionUuid === workflowExecutionUuid if metadataEntryHasMetadataKeysLike(metadataEntry, metadataKeys) if metadataEntryHasEmptyJobKey(metadataEntry, requireEmptyJobKey) - } yield metadataEntry + } yield metadataEntry).sortBy(_.metadataTimestamp) } def metadataEntriesNotLikeMetadataKeys(workflowExecutionUuid: String, metadataKeys: NonEmptyList[String], requireEmptyJobKey: Boolean) = { - for { + (for { metadataEntry <- metadataEntries if metadataEntry.workflowExecutionUuid === workflowExecutionUuid if !metadataEntryHasMetadataKeysLike(metadataEntry, metadataKeys) if metadataEntryHasEmptyJobKey(metadataEntry, requireEmptyJobKey) - } yield metadataEntry + } yield metadataEntry).sortBy(_.metadataTimestamp) } private[this] def metadataEntryHasMetadataKeysLike(metadataEntry: MetadataEntries, diff --git a/engine/src/main/resources/workflowTimings/workflowTimings.html b/engine/src/main/resources/workflowTimings/workflowTimings.html index 3e1df152c..368c7bf5d 100644 --- a/engine/src/main/resources/workflowTimings/workflowTimings.html +++ b/engine/src/main/resources/workflowTimings/workflowTimings.html @@ -69,7 +69,7 @@ var firstEventStart = null; var finalEventEnd = null; - if(callStatus == "Done" || callStatus == "Failed" || callStatus == "Preempted") { + if(callStatus == "Done" || callStatus == "Failed" || callStatus == "RetryableFailure") { executionCallsCount++; for (var executionEventIndex in executionEvents) { var executionEvent = callList[callIndex].executionEvents[executionEventIndex]; diff --git a/engine/src/main/scala/cromwell/engine/io/IoActor.scala b/engine/src/main/scala/cromwell/engine/io/IoActor.scala index 4e3eedd0b..3c845a653 100644 --- a/engine/src/main/scala/cromwell/engine/io/IoActor.scala +++ b/engine/src/main/scala/cromwell/engine/io/IoActor.scala @@ -136,12 +136,20 @@ object IoActor { case _ => false } + val AdditionalRetryablHttpCodes = List( + // HTTP 410: Gone + // From Google doc (https://cloud.google.com/storage/docs/json_api/v1/status-codes): + // "You have attempted to use a resumable upload session that is no longer available. + // If the reported status code was not successful and you still wish to upload the file, you must start a new session." + 410 + ) + /** * Failures that are considered retryable. * Retrying them should increase the "retry counter" */ def isRetryable(failure: Throwable): Boolean = failure match { - case gcs: StorageException => gcs.isRetryable + case gcs: StorageException => gcs.isRetryable || AdditionalRetryablHttpCodes.contains(gcs.getCode) case _: BatchFailedException => true case _: SocketException => true case _: SocketTimeoutException => true diff --git a/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala b/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala index a1f9fcb0d..cb9e3791a 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala @@ -244,7 +244,7 @@ object SingleWorkflowRunnerActor { id: WorkflowId) extends TerminalSwraData { override val terminalState = WorkflowAborted } implicit class EnhancedJsObject(val jsObject: JsObject) extends AnyVal { - def state: WorkflowState = WorkflowState.fromString(jsObject.fields("status").asInstanceOf[JsString].value) + def state: WorkflowState = WorkflowState.withName(jsObject.fields("status").asInstanceOf[JsString].value) } private val Tag = "SingleWorkflowRunnerActor" diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/CallMetadataHelper.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/CallMetadataHelper.scala index b5ab329aa..ccd4244b8 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/CallMetadataHelper.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/CallMetadataHelper.scala @@ -27,7 +27,7 @@ trait CallMetadataHelper { serviceRegistryActor ! PutMetadataAction(startEvents) } - def pushQueuedCallMetadata(diffs: Seq[WorkflowExecutionDiff]) = { + def pushQueuedCallMetadata(diffs: List[WorkflowExecutionDiff]) = { val startingEvents = for { diff <- diffs (jobKey, executionState) <- diff.executionStoreChanges if jobKey.isInstanceOf[BackendJobDescriptorKey] && executionState == ExecutionStatus.QueuedInCromwell diff --git a/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala b/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala index c889ef61a..38624b853 100644 --- a/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala +++ b/engine/src/main/scala/cromwell/webservice/CromwellApiService.scala @@ -237,7 +237,7 @@ trait CromwellApiService extends HttpService with PerRequestCreator { } } - def metadataRoute = + def metadataRoute = compressResponse() { path("workflows" / Segment / Segment / "metadata") { (version, possibleWorkflowId) => parameterMultiMap { parameters => val includeKeysOption = NonEmptyList.fromList(parameters.getOrElse("includeKey", List.empty)) @@ -260,6 +260,7 @@ trait CromwellApiService extends HttpService with PerRequestCreator { } } } + } def timingRoute = path("workflows" / Segment / Segment / "timing") { (version, possibleWorkflowId) => diff --git a/engine/src/main/scala/cromwell/webservice/PerRequest.scala b/engine/src/main/scala/cromwell/webservice/PerRequest.scala index f4fb9e876..415005785 100644 --- a/engine/src/main/scala/cromwell/webservice/PerRequest.scala +++ b/engine/src/main/scala/cromwell/webservice/PerRequest.scala @@ -107,7 +107,7 @@ trait PerRequestCreator { def perRequest(r: RequestContext, props: Props, message: AnyRef, - timeout: Duration = 1 minutes, + timeout: Duration = 2 minutes, name: String = PerRequestCreator.endpointActorName): Unit = { actorRefFactory.actorOf(Props(WithProps(r, props, message, timeout, name)).withDispatcher(ApiDispatcher), name) () diff --git a/engine/src/main/scala/cromwell/webservice/metadata/IndexedJsonValue.scala b/engine/src/main/scala/cromwell/webservice/metadata/IndexedJsonValue.scala deleted file mode 100644 index f51e64187..000000000 --- a/engine/src/main/scala/cromwell/webservice/metadata/IndexedJsonValue.scala +++ /dev/null @@ -1,63 +0,0 @@ -package cromwell.webservice.metadata - -import java.time.OffsetDateTime - -import cats.{Monoid, Semigroup} -import cats.instances.map._ -import cromwell.services.metadata.CallMetadataKeys -import spray.json._ - - -object IndexedJsonValue { - private implicit val dateTimeOrdering: Ordering[OffsetDateTime] = scala.Ordering.fromLessThan(_ isBefore _) - private val timestampedJsValueOrdering: Ordering[TimestampedJsValue] = scala.Ordering.by(_.timestamp) - - implicit val TimestampedJsonMonoid: Monoid[TimestampedJsValue] = new Monoid[TimestampedJsValue] { - def combine(f1: TimestampedJsValue, f2: TimestampedJsValue): TimestampedJsValue = { - (f1, f2) match { - case (o1: TimestampedJsObject, o2: TimestampedJsObject) => - val sg = implicitly[Semigroup[Map[String, TimestampedJsValue]]] - TimestampedJsObject(sg.combine(o1.v, o2.v), dateTimeOrdering.max(o1.timestamp, o2.timestamp)) - case (o1: TimestampedJsList, o2: TimestampedJsList) => - val sg = implicitly[Semigroup[Map[Int, TimestampedJsValue]]] - TimestampedJsList(sg.combine(o1.v, o2.v), dateTimeOrdering.max(o1.timestamp, o2.timestamp)) - case (o1, o2) => timestampedJsValueOrdering.max(o1, o2) - } - } - - override def empty: TimestampedJsValue = TimestampedJsObject(Map.empty, OffsetDateTime.now) - } -} - -/** Customized version of Json data structure, to account for timestamped values and lazy array creation */ -sealed trait TimestampedJsValue { - def toJson(expandedValues: Map[String, JsValue]): JsValue - def timestamp: OffsetDateTime -} - -private case class TimestampedJsList(v: Map[Int, TimestampedJsValue], timestamp: OffsetDateTime) extends TimestampedJsValue { - override def toJson(expandedValues: Map[String, JsValue]) = JsArray(v.values.toVector map { _.toJson(expandedValues) }) -} - -private case class TimestampedJsObject(v: Map[String, TimestampedJsValue], timestamp: OffsetDateTime) extends TimestampedJsValue { - override def toJson(expandedValues: Map[String, JsValue]) = { - val mappedValues = v map { - case (key, subWorkflowId: TimestampedJsPrimitive) if key == CallMetadataKeys.SubWorkflowId => - val subId = subWorkflowId.v.asInstanceOf[JsString] - expandedValues.get(subId.value) map { subMetadata => - CallMetadataKeys.SubWorkflowMetadata -> subMetadata - } getOrElse { - key -> subWorkflowId.v - } - case (key, value) => key -> value.toJson(expandedValues) - } - - JsObject(mappedValues) - } -} - -private class TimestampedJsPrimitive(val v: JsValue, val timestamp: OffsetDateTime) extends TimestampedJsValue { - override def toJson(expandedValues: Map[String, JsValue]) = v -} - -private case class TimestampedEmptyJson(override val timestamp: OffsetDateTime) extends TimestampedJsPrimitive(JsObject(Map.empty[String, JsValue]), timestamp) \ No newline at end of file diff --git a/engine/src/main/scala/cromwell/webservice/metadata/MetadataBuilderActor.scala b/engine/src/main/scala/cromwell/webservice/metadata/MetadataBuilderActor.scala index 272e94c75..7a991aef7 100644 --- a/engine/src/main/scala/cromwell/webservice/metadata/MetadataBuilderActor.scala +++ b/engine/src/main/scala/cromwell/webservice/metadata/MetadataBuilderActor.scala @@ -3,7 +3,7 @@ package cromwell.webservice.metadata import java.time.OffsetDateTime import akka.actor.{ActorRef, LoggingFSM, Props} -import cromwell.webservice.metadata.IndexedJsonValue._ +import cromwell.webservice.metadata.MetadataComponent._ import cats.instances.list._ import cats.syntax.foldable._ import cromwell.core.Dispatcher.ApiDispatcher @@ -22,7 +22,7 @@ import spray.json._ import scala.collection.immutable.TreeMap import scala.language.postfixOps -import scala.util.{Failure, Success, Try} +import scala.util.{Failure, Random, Success, Try} object MetadataBuilderActor { @@ -52,108 +52,97 @@ object MetadataBuilderActor { private val KeySeparator = MetadataKey.KeySeparator private val bracketMatcher = """\[(\d*)\]""".r - private val startMatcher = """^([^\[]+)\[""".r private val AttemptKey = "attempt" private val ShardKey = "shardIndex" - /** Types of element supported in a dotted key notation */ - private sealed trait KeyElement { - def toIndexedJson(value: TimestampedJsValue): TimestampedJsValue - } - - private case class ListElement(name: String, indexes: List[String]) extends KeyElement { - def toIndexedJson(innerValue: TimestampedJsValue) = { - if (indexes.isEmpty) { - TimestampedJsObject(Map(name -> innerValue), innerValue.timestamp) - } else { - /* - * The last index is the one that the innerValue should have in the innerList. - * From there lists are fold into one another until we reach the first index. - * e.g l[1][2] = "a" means - * "l": [ - * [ - * "a" <- will have index 2 in the inner list - * ] <- inner list: will have index 1 in the outer list - * ] - * - * Important note: Indexes are used for sorting purposes ONLY - * An index of 2 DOES NOT guarantee that a value will be at index 2 in a list - */ - val list = innerValue match { - // Empty value in a list means empty list - case TimestampedEmptyJson(timestamp) => TimestampedJsList(Map.empty, timestamp) - case nonEmptyValue => - /* - * This creates a (possibly nested) list, by folding over the indexes. - * The resulting list will be as deep as there are elements in "indexes" - * First we create the deepest list, that will contain innerValue (the actual value we want in the list) - * e.g with l[1][2] = "a". indexes will be List[1, 2]. innerValue will be "a". - * innerList is TimestampedJsList(Map(2 -> "a"), [timestamp of a]) - */ - val innerList = TimestampedJsList(TreeMap(indexes.last.toInt -> innerValue), nonEmptyValue.timestamp) - /* Then, stating with this innerList, we wrap around it as many lists as (indexes.length - 1) (because we used the last index for the innerValue above) - * Continuing with this example, result will be TimestampedJsList(Map(1 -> TimestampedJsList(Map(2 -> "a")))) - */ - indexes.init.foldRight(innerList)((index, acc) => { - TimestampedJsList(TreeMap(index.toInt -> acc), acc.timestamp) - }) + def parseKeyChunk(chunk: String, innerValue: MetadataComponent): MetadataComponent = { + chunk.indexOf('[') match { + // If there's no bracket, it's an object. e.g.: "calls" + case -1 => MetadataObject(Map(chunk -> innerValue)) + // If there's a bracket it's a named list. e.g.: "executionEvents[0][1]" + case bracketIndex => + // Name: "executionEvents" + val objectName = chunk.substring(0, bracketIndex) + + // Empty value means empty list + if (innerValue == MetadataEmpty) MetadataObject(Map(objectName -> MetadataList(Map.empty))) + else { + // Brackets: "[0][1]" + val brackets = chunk.substring(bracketIndex) + // Indices as a list: List(0, 1) + val listIndices = for { + m <- bracketMatcher.findAllMatchIn(brackets) + // It's possible for a bracket pair to be empty, in which case we just give it a random number + asInt = if (m.group(1).isEmpty) Random.nextInt() else m.group(1).toInt + } yield asInt + // Fold into a MetadataList: MetadataList(0 -> MetadataList(1 -> innerValue)) + val metadataList = listIndices.toList.foldRight(innerValue)((index, acc) => MetadataList(TreeMap(index -> acc))) + + MetadataObject(Map(objectName -> metadataList)) } - - TimestampedJsObject(Map(name -> list), list.timestamp) - } - } - } - private case class ObjectElement(name: String) extends KeyElement { - def toIndexedJson(value: TimestampedJsValue) = TimestampedJsObject(Map(name -> value), value.timestamp) - } - - private def parseKeyChunk(chunk: String): KeyElement = { - startMatcher.findFirstMatchIn(chunk) match { - case Some(listNameRegex) => - val indexes = bracketMatcher.findAllMatchIn(chunk).map(_.group(1)).toList - ListElement(listNameRegex.group(1), indexes) - case _ => ObjectElement(chunk) } } - private def metadataValueToIndexedJson(value: Option[MetadataValue], timestamp: OffsetDateTime): TimestampedJsValue = { + private def metadataValueToComponent(value: Option[MetadataValue], customOrdering: Option[Ordering[MetadataPrimitive]]): MetadataComponent = { value map { someValue => - val coerced: Try[TimestampedJsPrimitive] = someValue.valueType match { - case MetadataInt => Try(new TimestampedJsPrimitive(JsNumber(someValue.value.toInt), timestamp)) - case MetadataNumber => Try(new TimestampedJsPrimitive(JsNumber(someValue.value.toDouble), timestamp)) - case MetadataBoolean => Try(new TimestampedJsPrimitive(JsBoolean(someValue.value.toBoolean), timestamp)) - case MetadataString => Try(new TimestampedJsPrimitive(JsString(someValue.value), timestamp)) + val coerced: Try[MetadataPrimitive] = someValue.valueType match { + case MetadataInt => Try(MetadataPrimitive(JsNumber(someValue.value.toInt), customOrdering)) + case MetadataNumber => Try(MetadataPrimitive(JsNumber(someValue.value.toDouble), customOrdering)) + case MetadataBoolean => Try(MetadataPrimitive(JsBoolean(someValue.value.toBoolean), customOrdering)) + case MetadataString => Try(MetadataPrimitive(JsString(someValue.value), customOrdering)) } coerced match { case Success(v) => v case Failure(e) => log.warn(s"Failed to coerce ${someValue.value} to ${someValue.valueType}. Falling back to String.", e) - new TimestampedJsPrimitive(JsString(someValue.value), timestamp) + MetadataPrimitive(JsString(someValue.value), customOrdering) } - } getOrElse TimestampedEmptyJson(timestamp) + } getOrElse MetadataEmpty } - private def keyValueToIndexedJson(str: String, value: Option[MetadataValue], timestamp: OffsetDateTime): TimestampedJsValue = { - val innerValue: TimestampedJsValue = metadataValueToIndexedJson(value, timestamp) - str.split(KeySeparator).foldRight(innerValue)((chunk, acc) => { parseKeyChunk(chunk).toIndexedJson(acc) }) + def customOrdering(event: MetadataEvent): Option[Ordering[MetadataPrimitive]] = event match { + case MetadataEvent(MetadataKey(_, Some(_), key), _, _) if key == CallMetadataKeys.ExecutionStatus => Option(MetadataPrimitive.ExecutionStatusOrdering) + case MetadataEvent(MetadataKey(_, None, key), _, _) if key == WorkflowMetadataKeys.Status => Option(MetadataPrimitive.WorkflowStateOrdering) + case _ => None + } + + private def toMetadataComponent(subWorkflowMetadata: Map[String, JsValue])(event: MetadataEvent) = { + lazy val originalKeyAndPrimitive = (event.key.key, metadataValueToComponent(event.value, customOrdering(event))) + + val keyAndPrimitive = if (event.key.key.endsWith(CallMetadataKeys.SubWorkflowId)) { + (for { + metadataValue <- event.value + subWorkflowMetadata <- subWorkflowMetadata.get(metadataValue.value) + keyWithSubWorkflowMetadata = event.key.key.replace(CallMetadataKeys.SubWorkflowId, CallMetadataKeys.SubWorkflowMetadata) + subWorkflowComponent = MetadataPrimitive(subWorkflowMetadata) + } yield (keyWithSubWorkflowMetadata, subWorkflowComponent)) getOrElse originalKeyAndPrimitive + } else originalKeyAndPrimitive + + keyAndPrimitive._1.split(KeySeparator).foldRight(keyAndPrimitive._2)(parseKeyChunk) } + /** + * Metadata for a call attempt + */ private case class MetadataForAttempt(attempt: Int, metadata: JsObject) - /** There's one TimestampedJsValue per attempt, hence the list. */ + + /** + * Metadata objects of all attempts for one shard + */ private case class MetadataForIndex(index: Int, metadata: List[JsObject]) implicit val dateTimeOrdering: Ordering[OffsetDateTime] = scala.Ordering.fromLessThan(_ isBefore _) /** Sort events by timestamp, transform them into TimestampedJsValues, and merge them together. */ - private def eventsToIndexedJson(events: Seq[MetadataEvent]): TimestampedJsValue = { + private def eventsToIndexedJson(events: Seq[MetadataEvent], subWorkflowMetadata: Map[String, JsValue]): MetadataComponent = { // The `List` has a `Foldable` instance defined in scope, and because the `List`'s elements have a `Monoid` instance // defined in scope, `combineAll` can derive a sane `TimestampedJsValue` value even if the `List` of events is empty. - events.toList map { e => keyValueToIndexedJson(e.key.key, e.value, e.offsetDateTime) } combineAll + events.toList map toMetadataComponent(subWorkflowMetadata) combineAll } - private def eventsToAttemptMetadata(expandedValues: Map[String, JsValue])(attempt: Int, events: Seq[MetadataEvent]) = { - val withAttemptField = JsObject(eventsToIndexedJson(events).toJson(expandedValues).asJsObject.fields + (AttemptKey -> JsNumber(attempt))) + private def eventsToAttemptMetadata(subWorkflowMetadata: Map[String, JsValue])(attempt: Int, events: Seq[MetadataEvent]) = { + val withAttemptField = JsObject(eventsToIndexedJson(events, subWorkflowMetadata).toJson.asJsObject.fields + (AttemptKey -> JsNumber(attempt))) MetadataForAttempt(attempt, withAttemptField) } @@ -163,38 +152,62 @@ object MetadataBuilderActor { MetadataForIndex(index.getOrElse(-1), metadata) } - private def reduceWorkflowEvents(workflowEvents: Seq[MetadataEvent]): Seq[MetadataEvent] = { - // This handles state specially so a sensible final value is returned irrespective of the order in which raw state - // events were recorded in the journal. - val (workflowStatusEvents, workflowNonStatusEvents) = workflowEvents partition(_.key.key == WorkflowMetadataKeys.Status) - - val ordering = implicitly[Ordering[WorkflowState]] - // This orders by value in WorkflowState CRDT resolution, not necessarily the chronologically most recent state. - val sortedStateEvents = workflowStatusEvents.filter(_.value.isDefined) sortWith { case (a, b) => ordering.gt(a.value.get.toWorkflowState, b.value.get.toWorkflowState) } - workflowNonStatusEvents ++ sortedStateEvents.headOption.toList - } - - private def parseWorkflowEventsToTimestampedJsValue(events: Seq[MetadataEvent], includeCallsIfEmpty: Boolean, expandedValues: Map[String, JsValue]): JsObject = { - // Partition if sequence of events in a pair of (Workflow level events, Call level events) + private def buildMetadataJson(events: Seq[MetadataEvent], includeCallsIfEmpty: Boolean, expandedValues: Map[String, JsValue]): JsObject = { + // Partition events into workflow level and call level events val (workflowLevel, callLevel) = events partition { _.key.jobKey.isEmpty } - val foldedWorkflowValues = eventsToIndexedJson(reduceWorkflowEvents(workflowLevel)).toJson(expandedValues).asJsObject - + val workflowLevelJson = eventsToIndexedJson(workflowLevel, Map.empty).toJson.asJsObject + + /* + * Map( + * "fqn" -> Seq[Events], + * "fqn2" -> Seq[Events], + * ... + * ) + * Note that groupBy will preserve the ordering of the events in the Seq, which means that as long as the DB sorts them by timestamp, we can always assume the last one is the newest one. + * This is guaranteed by the groupBy invariant and the fact that filter preservers the ordering. (See scala doc for groupBy and filter) + */ val callsGroupedByFQN = callLevel groupBy { _.key.jobKey.get.callFqn } + /* + * Map( + * "fqn" -> Map( //Shard index + * Option(0) -> Seq[Events], + * Option(1) -> Seq[Events] + * ... + * ), + * ... + * ) + */ val callsGroupedByFQNAndIndex = callsGroupedByFQN mapValues { _ groupBy { _.key.jobKey.get.index } } + /* + * Map( + * "fqn" -> Map( + * Option(0) -> Map( //Attempt + * 1 -> Seq[Events], + * 2 -> Seq[Events], + * ... + * ), + * ... + * ), + * ... + * ) + */ val callsGroupedByFQNAndIndexAndAttempt = callsGroupedByFQNAndIndex mapValues { _ mapValues { _ groupBy { _.key.jobKey.get.attempt } } } + + val eventsToAttemptFunction = Function.tupled(eventsToAttemptMetadata(expandedValues) _) + val attemptToIndexFunction = (attemptMetadataToIndexMetadata _).tupled - val callsMap = callsGroupedByFQNAndIndexAndAttempt mapValues { eventsForIndex => - eventsForIndex mapValues { eventsForAttempt => - eventsForAttempt map Function.tupled(eventsToAttemptMetadata(expandedValues)) - } map { Function.tupled(attemptMetadataToIndexMetadata) } - } mapValues { md => JsArray(md.toVector.sortBy(_.index) flatMap { _.metadata }) } + val callsMap = callsGroupedByFQNAndIndexAndAttempt mapValues { _ mapValues { _ map eventsToAttemptFunction } map attemptToIndexFunction } mapValues { md => + JsArray(md.toVector.sortBy(_.index) flatMap { _.metadata }) + } val wrappedCalls = JsObject(Map(WorkflowMetadataKeys.Calls -> JsObject(callsMap))) val callData = if (callsMap.isEmpty && !includeCallsIfEmpty) Nil else wrappedCalls.fields - JsObject(foldedWorkflowValues.fields ++ callData) + JsObject(workflowLevelJson.fields ++ callData) } - private def parseWorkflowEvents(includeCallsIfEmpty: Boolean, expandedValues: Map[String, JsValue])(events: Seq[MetadataEvent]): JsObject = parseWorkflowEventsToTimestampedJsValue(events, includeCallsIfEmpty, expandedValues) + private def parseWorkflowEvents(includeCallsIfEmpty: Boolean, expandedValues: Map[String, JsValue])(events: Seq[MetadataEvent]): JsObject = { + buildMetadataJson(events, includeCallsIfEmpty, expandedValues) + } /** * Parse a Seq of MetadataEvent into a full Json metadata response. @@ -202,10 +215,6 @@ object MetadataBuilderActor { private def parse(events: Seq[MetadataEvent], expandedValues: Map[String, JsValue]): JsObject = { JsObject(events.groupBy(_.key.workflowId.toString) mapValues parseWorkflowEvents(includeCallsIfEmpty = true, expandedValues)) } - - implicit class EnhancedMetadataValue(val value: MetadataValue) extends AnyVal { - def toWorkflowState: WorkflowState = WorkflowState.fromString(value.value) - } } class MetadataBuilderActor(serviceRegistryActor: ActorRef) extends LoggingFSM[MetadataBuilderActorState, Option[MetadataBuilderActorData]] @@ -215,7 +224,7 @@ class MetadataBuilderActor(serviceRegistryActor: ActorRef) extends LoggingFSM[Me startWith(Idle, None) val tag = self.path.name - + when(Idle) { case Event(action: MetadataServiceAction, _) => serviceRegistryActor ! action diff --git a/engine/src/main/scala/cromwell/webservice/metadata/MetadataComponent.scala b/engine/src/main/scala/cromwell/webservice/metadata/MetadataComponent.scala new file mode 100644 index 000000000..ce089c420 --- /dev/null +++ b/engine/src/main/scala/cromwell/webservice/metadata/MetadataComponent.scala @@ -0,0 +1,50 @@ +package cromwell.webservice.metadata + +import cats.{Monoid, Semigroup} +import cats.instances.map._ +import cromwell.core.{ExecutionStatus, WorkflowState} +import spray.json.{JsArray, _} + +object MetadataComponent { + implicit val MetadataComponentMonoid: Monoid[MetadataComponent] = new Monoid[MetadataComponent] { + private lazy val stringKeyMapSg = implicitly[Semigroup[Map[String, MetadataComponent]]] + private lazy val intKeyMapSg = implicitly[Semigroup[Map[Int, MetadataComponent]]] + + def combine(f1: MetadataComponent, f2: MetadataComponent): MetadataComponent = { + (f1, f2) match { + case (MetadataObject(v1), MetadataObject(v2)) => MetadataObject(stringKeyMapSg.combine(v1, v2)) + case (MetadataList(v1), MetadataList(v2)) => MetadataList(intKeyMapSg.combine(v1, v2)) + // If there's a custom ordering, use it + case (v1 @ MetadataPrimitive(_, Some(o1)), v2 @ MetadataPrimitive(_, Some(o2))) if o1 == o2 => o1.max(v1, v2) + // Otherwise assume it's ordered by default and take the new one + case (o1, o2) => o2 + } + } + + override def empty: MetadataComponent = MetadataObject.empty + } + + implicit val jsonWriter: JsonWriter[MetadataComponent] = JsonWriter.func2Writer[MetadataComponent] { + case MetadataList(values) => JsArray(values.values.toVector map { _.toJson(this.jsonWriter) }) + case MetadataObject(values) => JsObject(values.mapValues(_.toJson(this.jsonWriter))) + case MetadataPrimitive(value, _) => value + case MetadataEmpty => JsObject.empty + } +} + +sealed trait MetadataComponent +case object MetadataEmpty extends MetadataComponent +object MetadataObject { def empty = MetadataObject(Map.empty) } +case class MetadataObject(v: Map[String, MetadataComponent]) extends MetadataComponent +case class MetadataList(v: Map[Int, MetadataComponent]) extends MetadataComponent + +object MetadataPrimitive { + val ExecutionStatusOrdering: Ordering[MetadataPrimitive] = Ordering.by { primitive: MetadataPrimitive => + ExecutionStatus.withName(primitive.v.asInstanceOf[JsString].value) + } + + val WorkflowStateOrdering: Ordering[MetadataPrimitive] = Ordering.by { primitive: MetadataPrimitive => + WorkflowState.withName(primitive.v.asInstanceOf[JsString].value) + } +} +case class MetadataPrimitive(v: JsValue, customOrdering: Option[Ordering[MetadataPrimitive]] = None) extends MetadataComponent diff --git a/engine/src/test/scala/cromwell/webservice/CromwellApiServiceSpec.scala b/engine/src/test/scala/cromwell/webservice/CromwellApiServiceSpec.scala index b360b8e09..0cd2220e1 100644 --- a/engine/src/test/scala/cromwell/webservice/CromwellApiServiceSpec.scala +++ b/engine/src/test/scala/cromwell/webservice/CromwellApiServiceSpec.scala @@ -15,10 +15,13 @@ import spray.json.{JsString, _} import spray.routing._ import cromwell.engine.workflow.workflowstore.WorkflowStoreSubmitActor.{WorkflowSubmittedToStore, WorkflowsBatchSubmittedToStore} import cromwell.util.SampleWdl.HelloWorld +import spray.httpx.ResponseTransformation import spray.httpx.SprayJsonSupport._ +import spray.httpx.encoding.Gzip import spray.testkit.ScalatestRouteTest +import spray.routing.Directives._ -class CromwellApiServiceSpec extends FlatSpec with ScalatestRouteTest with Matchers { +class CromwellApiServiceSpec extends FlatSpec with ScalatestRouteTest with Matchers with ResponseTransformation { import CromwellApiServiceSpec._ val cromwellApiService = new MockApiService() @@ -286,7 +289,7 @@ class CromwellApiServiceSpec extends FlatSpec with ScalatestRouteTest with Match behavior of "REST API /metadata endpoint" it should "return with full metadata from the metadata route" in { Get(s"/workflows/$version/${MockApiService.ExistingWorkflowId}/metadata") ~> - cromwellApiService.metadataRoute ~> + mapHttpResponse(decode(Gzip))(cromwellApiService.metadataRoute) ~> check { status should be(StatusCodes.OK) val result = responseAs[JsObject] @@ -299,7 +302,7 @@ class CromwellApiServiceSpec extends FlatSpec with ScalatestRouteTest with Match it should "return with included metadata from the metadata route" in { Get(s"/workflows/$version/${MockApiService.ExistingWorkflowId}/metadata?includeKey=testKey1&includeKey=testKey2a") ~> - cromwellApiService.metadataRoute ~> + mapHttpResponse(decode(Gzip))(cromwellApiService.metadataRoute) ~> check { status should be(StatusCodes.OK) val result = responseAs[JsObject] @@ -313,7 +316,7 @@ class CromwellApiServiceSpec extends FlatSpec with ScalatestRouteTest with Match it should "return with excluded metadata from the metadata route" in { Get(s"/workflows/$version/${MockApiService.ExistingWorkflowId}/metadata?excludeKey=testKey2b&excludeKey=testKey3") ~> - cromwellApiService.metadataRoute ~> + mapHttpResponse(decode(Gzip))(cromwellApiService.metadataRoute) ~> check { status should be(StatusCodes.OK) val result = responseAs[JsObject] @@ -327,7 +330,7 @@ class CromwellApiServiceSpec extends FlatSpec with ScalatestRouteTest with Match it should "return an error when included and excluded metadata requested from the metadata route" in { Get(s"/workflows/$version/${MockApiService.ExistingWorkflowId}/metadata?includeKey=testKey1&excludeKey=testKey2") ~> - cromwellApiService.metadataRoute ~> + mapHttpResponse(decode(Gzip))(cromwellApiService.metadataRoute) ~> check { assertResult(StatusCodes.BadRequest) { status diff --git a/engine/src/test/scala/cromwell/webservice/MetadataBuilderActorSpec.scala b/engine/src/test/scala/cromwell/webservice/MetadataBuilderActorSpec.scala index 9ecfeea42..9d4c7b66f 100644 --- a/engine/src/test/scala/cromwell/webservice/MetadataBuilderActorSpec.scala +++ b/engine/src/test/scala/cromwell/webservice/MetadataBuilderActorSpec.scala @@ -11,13 +11,14 @@ import cromwell.webservice.PerRequest.RequestComplete import cromwell.webservice.metadata.MetadataBuilderActor import org.scalatest.prop.TableDrivenPropertyChecks import org.scalatest.{FlatSpecLike, Matchers} +import org.specs2.mock.Mockito import spray.http.{StatusCode, StatusCodes} import spray.json._ import scala.concurrent.duration._ import scala.language.postfixOps -class MetadataBuilderActorSpec extends TestKitSuite("Metadata") with FlatSpecLike with Matchers +class MetadataBuilderActorSpec extends TestKitSuite("Metadata") with FlatSpecLike with Matchers with Mockito with TableDrivenPropertyChecks with ImplicitSender { behavior of "MetadataParser" @@ -102,14 +103,21 @@ class MetadataBuilderActorSpec extends TestKitSuite("Metadata") with FlatSpecLik type EventBuilder = (String, String, OffsetDateTime) - def makeEvent(workflow: WorkflowId)(key: String, value: MetadataValue, offsetDateTime: OffsetDateTime) = { + def makeEvent(workflow: WorkflowId)(key: String, value: MetadataValue, offsetDateTime: OffsetDateTime): MetadataEvent = { MetadataEvent(MetadataKey(workflow, None, key), Option(value), offsetDateTime) } - def assertMetadataKeyStructure(eventList: List[EventBuilder], expectedJson: String) = { - val workflow = WorkflowId.randomId() + def makeCallEvent(workflow: WorkflowId)(key: String, value: MetadataValue, offsetDateTime: OffsetDateTime) = { + val jobKey = MetadataJobKey("fqn", None, 1) + MetadataEvent(MetadataKey(workflow, Option(jobKey), key), Option(value), offsetDateTime) + } + + def assertMetadataKeyStructure(eventList: List[EventBuilder], + expectedJson: String, + workflow: WorkflowId = WorkflowId.randomId(), + eventMaker: WorkflowId => (String, MetadataValue, OffsetDateTime) => MetadataEvent = makeEvent) = { - val events = eventList map { e => (e._1, MetadataValue(e._2), e._3) } map Function.tupled(makeEvent(workflow)) + val events = eventList map { e => (e._1, MetadataValue(e._2), e._3) } map Function.tupled(eventMaker(workflow)) val expectedRes = s"""{ "calls": {}, $expectedJson, "id":"$workflow" }""" val mdQuery = MetadataQuery(workflow, None, None, None, None, expandSubWorkflows = false) @@ -117,18 +125,18 @@ class MetadataBuilderActorSpec extends TestKitSuite("Metadata") with FlatSpecLik assertMetadataResponse(queryAction, mdQuery, events, expectedRes) } - it should "keep event with later timestamp for the same key in metadata" in { + it should "assume the event list is ordered and keep last event if 2 events have same key" in { val eventBuilderList = List( ("a", "aLater", OffsetDateTime.parse("2000-01-02T12:00:00Z")), ("a", "a", OffsetDateTime.parse("2000-01-01T12:00:00Z")) ) val expectedRes = - """"a": "aLater"""".stripMargin + """"a": "a"""".stripMargin assertMetadataKeyStructure(eventBuilderList, expectedRes) } - it should "use CRDT ordering instead of timestamp for status" in { + it should "use CRDT ordering instead of timestamp for workflow state" in { val eventBuilderList = List( ("status", "Succeeded", OffsetDateTime.now), ("status", "Running", OffsetDateTime.now.plusSeconds(1)) @@ -139,6 +147,25 @@ class MetadataBuilderActorSpec extends TestKitSuite("Metadata") with FlatSpecLik assertMetadataKeyStructure(eventBuilderList, expectedRes) } + it should "use CRDT ordering instead of timestamp for call execution status" in { + val eventBuilderList = List( + ("executionStatus", "Done", OffsetDateTime.now), + ("executionStatus", "Running", OffsetDateTime.now.plusSeconds(1)) + ) + val workflowId = WorkflowId.randomId() + val expectedRes = + s""""calls": { + | "fqn": [{ + | "attempt": 1, + | "executionStatus": "Done", + | "shardIndex": -1 + | }] + | }, + | "id": "$workflowId"""".stripMargin + + assertMetadataKeyStructure(eventBuilderList, expectedRes, workflowId, makeCallEvent) + } + it should "build JSON object structure from dotted key syntax" in { val eventBuilderList = List( ("a:b:c", "abc", OffsetDateTime.now), diff --git a/project/plugins.sbt b/project/plugins.sbt index ef96d4a95..c4bfd6abc 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,4 +1,4 @@ -addSbtPlugin("se.marcuslonnberg" % "sbt-docker" % "1.4.0") +addSbtPlugin("se.marcuslonnberg" % "sbt-docker" % "1.4.1") addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.3") addSbtPlugin("io.spray" % "sbt-revolver" % "0.8.0") /* diff --git a/services/src/main/scala/cromwell/services/metadata/WorkflowQueryKey.scala b/services/src/main/scala/cromwell/services/metadata/WorkflowQueryKey.scala index e5e134ca7..14dfa7a17 100644 --- a/services/src/main/scala/cromwell/services/metadata/WorkflowQueryKey.scala +++ b/services/src/main/scala/cromwell/services/metadata/WorkflowQueryKey.scala @@ -65,7 +65,7 @@ object WorkflowQueryKey { override def validate(grouped: Map[String, Seq[(String, String)]]): ErrorOr[List[String]] = { val values = valuesFromMap(grouped).toList val nels = values map { v => - if (Try(WorkflowState.fromString(v.toLowerCase.capitalize)).isSuccess) v.validNel[String] else v.invalidNel[String] + if (Try(WorkflowState.withName(v.toLowerCase.capitalize)).isSuccess) v.validNel[String] else v.invalidNel[String] } sequenceListOfValidatedNels("Unrecognized status values", nels) } diff --git a/services/src/main/scala/cromwell/services/metadata/impl/MetadataDatabaseAccess.scala b/services/src/main/scala/cromwell/services/metadata/impl/MetadataDatabaseAccess.scala index c37122714..628d765a5 100644 --- a/services/src/main/scala/cromwell/services/metadata/impl/MetadataDatabaseAccess.scala +++ b/services/src/main/scala/cromwell/services/metadata/impl/MetadataDatabaseAccess.scala @@ -20,8 +20,8 @@ object MetadataDatabaseAccess { // Resolve the status if both `this` and `that` have defined statuses. This will evaluate to `None` // if one or both of the statuses is not defined. val resolvedStatus = for { - thisStatus <- summary1.workflowStatus map WorkflowState.fromString - thatStatus <- summary2.workflowStatus map WorkflowState.fromString + thisStatus <- summary1.workflowStatus map WorkflowState.withName + thatStatus <- summary2.workflowStatus map WorkflowState.withName } yield (thisStatus |+| thatStatus).toString WorkflowMetadataSummaryEntry( @@ -149,7 +149,7 @@ trait MetadataDatabaseAccess { def getWorkflowStatus(id: WorkflowId) (implicit ec: ExecutionContext): Future[Option[WorkflowState]] = { - databaseInterface.getWorkflowStatus(id.toString) map { _ map WorkflowState.fromString } + databaseInterface.getWorkflowStatus(id.toString) map { _ map WorkflowState.withName } } def workflowExistsWithId(possibleWorkflowId: String)(implicit ec: ExecutionContext): Future[Boolean] = { diff --git a/services/src/test/scala/cromwell/services/metadata/impl/MetadataDatabaseAccessSpec.scala b/services/src/test/scala/cromwell/services/metadata/impl/MetadataDatabaseAccessSpec.scala index f9f555abf..ccf81a3ab 100644 --- a/services/src/test/scala/cromwell/services/metadata/impl/MetadataDatabaseAccessSpec.scala +++ b/services/src/test/scala/cromwell/services/metadata/impl/MetadataDatabaseAccessSpec.scala @@ -82,6 +82,32 @@ class MetadataDatabaseAccessSpec extends FlatSpec with Matchers with ScalaFuture } } yield()).futureValue } + + it should "sort metadata events by timestamp from older to newer" taggedAs DbmsTest in { + def unorderedEvents(id: WorkflowId): Future[Vector[MetadataEvent]] = { + val workflowKey = MetadataKey(id, jobKey = None, key = null) + val now = OffsetDateTime.now() + val yesterday = now.minusDays(1) + val tomorrow = now.plusDays(1) + + val yesterdayEvent = MetadataEvent(workflowKey.copy(key = WorkflowMetadataKeys.WorkflowRoot), Option(MetadataValue("A")), yesterday) + val nowEvent = MetadataEvent(workflowKey.copy(key = WorkflowMetadataKeys.WorkflowRoot), Option(MetadataValue("B")), now) + val tomorrowEvent = MetadataEvent(workflowKey.copy(key = WorkflowMetadataKeys.WorkflowRoot), Option(MetadataValue("C")), tomorrow) + + val events = Vector(tomorrowEvent, yesterdayEvent, nowEvent) + + val expectedEvents = Vector(yesterdayEvent, nowEvent, tomorrowEvent) + + dataAccess.addMetadataEvents(events) map { _ => expectedEvents } + } + + (for { + workflow1Id <- baseWorkflowMetadata(Workflow1Name) + expected <- unorderedEvents(workflow1Id) + response <- dataAccess.queryMetadataEvents(MetadataQuery(workflow1Id, None, Option(WorkflowMetadataKeys.WorkflowRoot), None, None, expandSubWorkflows = false)) + _ = response shouldBe expected + } yield()).futureValue + } it should "create and query a workflow" taggedAs DbmsTest in { diff --git a/services/src/test/scala/cromwell/services/metadata/impl/WriteMetadataActorSpec.scala b/services/src/test/scala/cromwell/services/metadata/impl/WriteMetadataActorSpec.scala index 577013aef..2ae93db31 100644 --- a/services/src/test/scala/cromwell/services/metadata/impl/WriteMetadataActorSpec.scala +++ b/services/src/test/scala/cromwell/services/metadata/impl/WriteMetadataActorSpec.scala @@ -7,33 +7,40 @@ import cats.data.NonEmptyVector import cromwell.core.WorkflowId import cromwell.services.ServicesSpec import cromwell.services.metadata.MetadataService.PutMetadataAction -import cromwell.services.metadata.{MetadataEvent, MetadataKey, MetadataValue} import cromwell.services.metadata.impl.WriteMetadataActor.{HasEvents, NoEvents, WaitingToWrite, WritingToDb} +import cromwell.services.metadata.{MetadataEvent, MetadataKey, MetadataValue} +import org.scalatest.concurrent.Eventually import scala.concurrent.duration._ -class WriteMetadataActorSpec extends ServicesSpec("Metadata") { +class WriteMetadataActorSpec extends ServicesSpec("Metadata") with Eventually { import WriteMetadataActorSpec.Action - val actor = TestFSMRef(WriteMetadataActor(10, 1.days)) // A WMA that won't (hopefully!) perform a time based flush during this test + val actor = TestFSMRef(WriteMetadataActor(10, 1.day)) // A WMA that won't (hopefully!) perform a time based flush during this test "WriteMetadataActor" should { "start with no events and waiting to write" in { - assert(actor.stateName == WaitingToWrite) - assert(actor.stateData == NoEvents) + actor.stateName shouldBe WaitingToWrite + actor.stateData shouldBe NoEvents } "Have one event and be waiting after one event is sent" in { actor ! Action - assert(actor.stateName == WaitingToWrite) - assert(actor.stateData == HasEvents(NonEmptyVector.fromVectorUnsafe(Action.events.toVector))) + eventually { + actor.stateName shouldBe WaitingToWrite + actor.stateData shouldBe HasEvents(NonEmptyVector.fromVectorUnsafe(Action.events.toVector)) + } } "Have one event after batch size + 1 is reached" in { 1 to 10 foreach { _ => actor ! Action } - assert(actor.stateName == WritingToDb) + eventually { + actor.stateName shouldBe WritingToDb + } actor ! Action - assert(actor.stateData == HasEvents(NonEmptyVector.fromVectorUnsafe(Action.events.toVector))) + eventually { + actor.stateData shouldBe HasEvents(NonEmptyVector.fromVectorUnsafe(Action.events.toVector)) + } } } } From 63d133ada0060d2439c9a38519a5fd7817a6f6a0 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Fri, 21 Apr 2017 13:52:13 -0400 Subject: [PATCH 013/134] Publish cromwell-api-client artifact --- build.sbt | 1 + 1 file changed, 1 insertion(+) diff --git a/build.sbt b/build.sbt index f3d84863a..98789be3a 100644 --- a/build.sbt +++ b/build.sbt @@ -108,6 +108,7 @@ lazy val root = (project in file(".")) .aggregate(jesBackend) .aggregate(tesBackend) .aggregate(engine) + .aggregate(cromwellApiClient) // Next level of projects to include in the fat jar (their dependsOn will be transitively included) .dependsOn(engine) .dependsOn(jesBackend) From 1ad77169aea0be020d0efbcadafeba7b576d7b56 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Fri, 21 Apr 2017 14:54:03 -0400 Subject: [PATCH 014/134] Client publish fixup --- project/Dependencies.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index ff553b40d..6c18c81e8 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -147,7 +147,8 @@ object Dependencies { "com.typesafe.akka" %% "akka-http" % cromwellApiClientAkkaHttpV, "com.typesafe.akka" %% "akka-http-spray-json" % cromwellApiClientAkkaHttpV, "com.github.pathikrit" %% "better-files" % "3.0.0", - "org.scalatest" %% "scalatest" % "3.0.1" % Test + "org.scalatest" %% "scalatest" % "3.0.1" % Test, + "org.pegdown" % "pegdown" % "1.6.0" % Test ) val jesBackendDependencies = refinedTypeDependenciesList From 754134b3954f4ed972a03b6140e435bc41089c9c Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Mon, 24 Apr 2017 11:50:35 -0400 Subject: [PATCH 015/134] (Centaur Port) Decode responses --- .../main/scala/cromwell/api/CromwellClient.scala | 29 ++++++++++++++++------ 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/cromwellApiClient/src/main/scala/cromwell/api/CromwellClient.scala b/cromwellApiClient/src/main/scala/cromwell/api/CromwellClient.scala index 5aca78918..924bc89dc 100644 --- a/cromwellApiClient/src/main/scala/cromwell/api/CromwellClient.scala +++ b/cromwellApiClient/src/main/scala/cromwell/api/CromwellClient.scala @@ -4,7 +4,9 @@ import java.net.URL import akka.http.scaladsl.Http import akka.actor.ActorSystem +import akka.http.scaladsl.coding.{Deflate, Gzip, NoCoding} import akka.http.scaladsl.model.{HttpEntity, _} +import akka.http.scaladsl.model.headers.HttpEncodings import akka.http.scaladsl.unmarshalling.{Unmarshal, Unmarshaller} import akka.stream.ActorMaterializer import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._ @@ -23,9 +25,9 @@ class CromwellClient(val cromwellUrl: URL, val apiVersion: String)(implicit acto // Everything else is a suffix off the submit endpoint: lazy val batchSubmitEndpoint = s"$submitEndpoint/batch" private def workflowSpecificEndpoint(workflowId: WorkflowId, endpoint: String) = s"$submitEndpoint/$workflowId/$endpoint" - def abortEndpoint(workflowId: WorkflowId) = workflowSpecificEndpoint(workflowId, "abort") - def statusEndpoint(workflowId: WorkflowId) = workflowSpecificEndpoint(workflowId, "status") - def metadataEndpoint(workflowId: WorkflowId) = workflowSpecificEndpoint(workflowId, "metadata") + def abortEndpoint(workflowId: WorkflowId): String = workflowSpecificEndpoint(workflowId, "abort") + def statusEndpoint(workflowId: WorkflowId): String = workflowSpecificEndpoint(workflowId, "status") + def metadataEndpoint(workflowId: WorkflowId): String = workflowSpecificEndpoint(workflowId, "metadata") lazy val backendsEndpoint = s"$submitEndpoint/backends" import model.CromwellStatusJsonSupport._ @@ -81,7 +83,8 @@ class CromwellClient(val cromwellUrl: URL, val apiVersion: String)(implicit acto */ private def makeRequest[A](request: HttpRequest)(implicit um: Unmarshaller[ResponseEntity, A], ec: ExecutionContext): Future[A] = for { response <- Http().singleRequest(request) - entity <- Future.fromTry(response.toEntity) + decoded <- Future.fromTry(decodeResponse(response)) + entity <- Future.fromTry(decoded.toEntity) unmarshalled <- entity.to[A] } yield unmarshalled @@ -100,6 +103,18 @@ class CromwellClient(val cromwellUrl: URL, val apiVersion: String)(implicit acto options map (o => addToken(o.parseJson.asJsObject.convertTo[Map[String, JsValue]]).toJson.toString) } + + private val decoders = Map( + HttpEncodings.gzip -> Gzip, + HttpEncodings.deflate -> Deflate, + HttpEncodings.identity -> NoCoding + ) + + private def decodeResponse(response: HttpResponse): Try[HttpResponse] = { + decoders.get(response.encoding) map { decoder => + Try(decoder.decode(response)) + } getOrElse Failure(UnsuccessfulRequestException(s"No decoder for ${response.encoding}", response)) + } } object CromwellClient { @@ -107,11 +122,11 @@ object CromwellClient { def toEntity: Try[Unmarshal[ResponseEntity]] = response match { case HttpResponse(_: StatusCodes.Success, _, entity, _) => Success(Unmarshal(entity)) - case other => Failure(new UnsuccessfulRequestException(other)) + case other => Failure(UnsuccessfulRequestException("Unmarshalling error", other)) } } - final case class UnsuccessfulRequestException(httpResponse: HttpResponse) extends Exception { - override def getMessage = httpResponse.toString + final case class UnsuccessfulRequestException(message: String, httpResponse: HttpResponse) extends Exception { + override def getMessage: String = message + ": " + httpResponse.toString } } From 5b66bf5597d40c7705af6c773b4e8b0a2b219562 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Fri, 21 Apr 2017 16:35:36 -0400 Subject: [PATCH 016/134] Remove race conditions from WriteMetadataActorSpec --- .../metadata/impl/WriteMetadataActor.scala | 16 ++++-- .../metadata/impl/WriteMetadataActorSpec.scala | 58 ++++++++++++++++++++-- 2 files changed, 65 insertions(+), 9 deletions(-) diff --git a/services/src/main/scala/cromwell/services/metadata/impl/WriteMetadataActor.scala b/services/src/main/scala/cromwell/services/metadata/impl/WriteMetadataActor.scala index 99a591fff..10f8c77eb 100644 --- a/services/src/main/scala/cromwell/services/metadata/impl/WriteMetadataActor.scala +++ b/services/src/main/scala/cromwell/services/metadata/impl/WriteMetadataActor.scala @@ -9,15 +9,16 @@ import cromwell.services.metadata.MetadataService.{MetadataServiceAction, PutMet import cromwell.services.metadata.impl.WriteMetadataActor.{WriteMetadataActorData, WriteMetadataActorState} import org.slf4j.LoggerFactory +import scala.concurrent.ExecutionContext import scala.concurrent.duration._ import scala.util.{Failure, Success, Try} -final case class WriteMetadataActor(batchRate: Int, flushRate: FiniteDuration) +class WriteMetadataActor(batchRate: Int, flushRate: FiniteDuration) extends LoggingFSM[WriteMetadataActorState, WriteMetadataActorData] with ActorLogging with MetadataDatabaseAccess with SingletonServicesStore { import WriteMetadataActor._ - implicit val ec = context.dispatcher + implicit val ec: ExecutionContext = context.dispatcher override def preStart(): Unit = { context.system.scheduler.schedule(0.seconds, flushRate, self, ScheduledFlushToDb) @@ -29,8 +30,8 @@ final case class WriteMetadataActor(batchRate: Int, flushRate: FiniteDuration) when(WaitingToWrite) { case Event(PutMetadataAction(events), curData) => curData.addEvents(events) match { - case data@HasEvents(e) if e.length > batchRate => goto(WritingToDb) using data - case e => stay using e + case newData: HasEvents if newData.length > batchRate => goto(WritingToDb) using newData + case newData => stay using newData } case Event(ScheduledFlushToDb, curData) => log.debug("Initiating periodic metadata flush to DB") @@ -73,7 +74,7 @@ final case class WriteMetadataActor(batchRate: Int, flushRate: FiniteDuration) } object WriteMetadataActor { - def props(batchRate: Int, flushRate: FiniteDuration) = Props(new WriteMetadataActor(batchRate, flushRate)).withDispatcher(ServiceDispatcher) + def props(batchRate: Int, flushRate: FiniteDuration): Props = Props(new WriteMetadataActor(batchRate, flushRate)).withDispatcher(ServiceDispatcher) private lazy val logger = LoggerFactory.getLogger("WriteMetadataActor") sealed trait WriteMetadataActorMessage @@ -103,6 +104,11 @@ object WriteMetadataActor { this } } + + def length: Int = this match { + case NoEvents => 0 + case HasEvents(e) => e.length + } } case object NoEvents extends WriteMetadataActorData diff --git a/services/src/test/scala/cromwell/services/metadata/impl/WriteMetadataActorSpec.scala b/services/src/test/scala/cromwell/services/metadata/impl/WriteMetadataActorSpec.scala index 2ae93db31..583c939c6 100644 --- a/services/src/test/scala/cromwell/services/metadata/impl/WriteMetadataActorSpec.scala +++ b/services/src/test/scala/cromwell/services/metadata/impl/WriteMetadataActorSpec.scala @@ -7,16 +7,23 @@ import cats.data.NonEmptyVector import cromwell.core.WorkflowId import cromwell.services.ServicesSpec import cromwell.services.metadata.MetadataService.PutMetadataAction -import cromwell.services.metadata.impl.WriteMetadataActor.{HasEvents, NoEvents, WaitingToWrite, WritingToDb} +import cromwell.services.metadata.impl.WriteMetadataActor._ import cromwell.services.metadata.{MetadataEvent, MetadataKey, MetadataValue} +import org.scalatest.BeforeAndAfter import org.scalatest.concurrent.Eventually +import scala.concurrent.{ExecutionContext, Future, Promise} import scala.concurrent.duration._ +import scala.util.Success -class WriteMetadataActorSpec extends ServicesSpec("Metadata") with Eventually { +class WriteMetadataActorSpec extends ServicesSpec("Metadata") with Eventually with BeforeAndAfter { import WriteMetadataActorSpec.Action - val actor = TestFSMRef(WriteMetadataActor(10, 1.day)) // A WMA that won't (hopefully!) perform a time based flush during this test + var actor: TestFSMRef[WriteMetadataActorState, WriteMetadataActorData, DelayingWriteMetadataActor] = _ + + before { + actor = TestFSMRef(new DelayingWriteMetadataActor()) + } "WriteMetadataActor" should { "start with no events and waiting to write" in { @@ -33,12 +40,29 @@ class WriteMetadataActorSpec extends ServicesSpec("Metadata") with Eventually { } "Have one event after batch size + 1 is reached" in { - 1 to 10 foreach { _ => actor ! Action } + 1 to WriteMetadataActorSpec.BatchRate foreach { _ => actor ! Action } + actor.stateName shouldBe WaitingToWrite + + eventually { + actor.stateData match { + case HasEvents(e) => e.toVector.size shouldBe WriteMetadataActorSpec.BatchRate + case _ => fail("Expecting the actor to have events queued up") + } + } + actor ! Action eventually { actor.stateName shouldBe WritingToDb + actor.underlyingActor.writeToDbInProgress shouldBe true + actor.stateData shouldBe NoEvents } actor ! Action eventually { + actor.stateName shouldBe WritingToDb + actor.stateData shouldBe HasEvents(NonEmptyVector.fromVectorUnsafe(Action.events.toVector)) + } + actor.underlyingActor.completeWritePromise() + eventually { + actor.stateName shouldBe WaitingToWrite actor.stateData shouldBe HasEvents(NonEmptyVector.fromVectorUnsafe(Action.events.toVector)) } } @@ -48,4 +72,30 @@ class WriteMetadataActorSpec extends ServicesSpec("Metadata") with Eventually { object WriteMetadataActorSpec { val Event = MetadataEvent(MetadataKey(WorkflowId.randomId(), None, "key"), Option(MetadataValue("value")), OffsetDateTime.now) val Action = PutMetadataAction(Event) + + val BatchRate: Int = 10 + val FunctionallyForever: FiniteDuration = 100.days +} + +// A WMA that won't (hopefully!) perform a time based flush during this test +final class DelayingWriteMetadataActor extends WriteMetadataActor(WriteMetadataActorSpec.BatchRate, WriteMetadataActorSpec.FunctionallyForever) { + + var writeToDbInProgress: Boolean = false + var writeToDbCompletionPromise: Option[Promise[Unit]] = None + + override def addMetadataEvents(metadataEvents: Iterable[MetadataEvent])(implicit ec: ExecutionContext): Future[Unit] = { + writeToDbCompletionPromise = Option(Promise[Unit]()) + writeToDbInProgress = true + writeToDbCompletionPromise.get.future + } + + def completeWritePromise(): Unit = { + writeToDbCompletionPromise match { + case Some(promise) => + promise.complete(Success(())) + writeToDbInProgress = false + writeToDbCompletionPromise = None + case None => throw new Exception("BAD TEST! Cannot complete the actor's write future if the actor hasn't requested it yet!") + } + } } From c81343cfc6858fcf45b6b3f3721933d20f65c759 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Mon, 24 Apr 2017 16:07:28 -0400 Subject: [PATCH 017/134] Deprecation errors for DB configs --- .../scala/cromwell/services/ServicesStore.scala | 54 ++++++++++++++++------ 1 file changed, 39 insertions(+), 15 deletions(-) diff --git a/services/src/main/scala/cromwell/services/ServicesStore.scala b/services/src/main/scala/cromwell/services/ServicesStore.scala index c00aad09c..e8a8b940d 100644 --- a/services/src/main/scala/cromwell/services/ServicesStore.scala +++ b/services/src/main/scala/cromwell/services/ServicesStore.scala @@ -5,7 +5,6 @@ import cromwell.database.migration.liquibase.LiquibaseUtils import cromwell.database.slick.SlickDatabase import cromwell.database.sql.SqlDatabase import net.ceedubs.ficus.Ficus._ -import org.slf4j.LoggerFactory trait ServicesStore { def databaseInterface: SqlDatabase @@ -30,20 +29,45 @@ trait SingletonServicesStore extends ServicesStore { object SingletonServicesStore { - private lazy val log = LoggerFactory.getLogger("SingletonServicesStore") - private val databaseConfig = { - val config = ConfigFactory.load.getConfig("database") - if (config.hasPath("config")) { - log.warn( - """ - |Use of configuration path 'database.config' is deprecated. - | - |Move the configuration directly under the 'database' element, and remove the key 'database.config'. - |""".stripMargin) - config.getConfig(config.getString("config")) - } else { - config - } + private val databaseConfig = ConfigFactory.load.getConfig("database") + + if (databaseConfig.hasPath("config")) { + val msg = """ + |******************************* + |***** DEPRECATION MESSAGE ***** + |******************************* + | + |Use of configuration path 'database.config' has been deprecated. + | + |Move the configuration directly under the 'database' element, and remove the key 'database.config'. + | + |""".stripMargin + throw new Exception(msg) + } else if (databaseConfig.hasPath("driver")) { + val msg = + """ + |******************************* + |***** DEPRECATION MESSAGE ***** + |******************************* + | + |Use of configuration path 'database.driver' has been deprecated. Replace with a "profile" element instead, e.g: + | + |database { + | #driver = "slick.driver.MySQLDriver$" #old + | profile = "slick.jdbc.MySQLProfile$" #new + | db { + | driver = "com.mysql.jdbc.Driver" + | url = "jdbc:mysql://host/cromwell?rewriteBatchedStatements=true" + | user = "user" + | password = "pass" + | connectionTimeout = 5000 + | } + |} + | + |Cromwell thanks you. + |""".stripMargin + throw + new Exception(msg) } import ServicesStore.EnhancedSqlDatabase From 8f6ee4154e725b69df2a13defeaa1588b33a4bc0 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Fri, 21 Apr 2017 13:09:15 -0400 Subject: [PATCH 018/134] JES in-command write_lines fixup --- .../jes/JesAsyncBackendJobExecutionActor.scala | 15 ++++++++----- .../tes/TesAsyncBackendJobExecutionActor.scala | 4 ++-- .../scala/cromwell/backend/impl/tes/TesTask.scala | 26 +++++++++++++--------- 3 files changed, 27 insertions(+), 18 deletions(-) diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala index cc8524946..d8fd5ecfb 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala @@ -147,17 +147,19 @@ class JesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn private[jes] def generateJesInputs(jobDescriptor: BackendJobDescriptor): Set[JesInput] = { - val writeFunctionFiles = call.task.evaluateFilesFromCommand(jobDescriptor.fullyQualifiedInputs, backendEngineFunctions) map { - case (expression, file) => expression.toWdlString.md5SumShort -> Seq(file) + val fullyQualifiedPreprocessedInputs = jobDescriptor.inputDeclarations map { case (declaration, value) => declaration.fullyQualifiedName -> commandLineValueMapper(value) } + val writeFunctionFiles = call.task.evaluateFilesFromCommand(fullyQualifiedPreprocessedInputs, backendEngineFunctions) map { + case (expression, file) => expression.toWdlString.md5SumShort -> Seq(file) } /* Collect all WdlFiles from inputs to the call */ - val callInputFiles: Map[FullyQualifiedName, Seq[WdlFile]] = jobDescriptor.fullyQualifiedInputs mapValues { _.collectAsSeq { case w: WdlFile => w } } + val callInputFiles: Map[FullyQualifiedName, Seq[WdlFile]] = jobDescriptor.fullyQualifiedInputs mapValues { + _.collectAsSeq { case w: WdlFile => w } + } val inputs = (callInputFiles ++ writeFunctionFiles) flatMap { case (name, files) => jesInputsFromWdlFiles(name, files, files.map(relativeLocalizationPath), jobDescriptor) } - inputs.toSet } @@ -291,8 +293,9 @@ class JesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn // Want to force runtimeAttributes to evaluate so we can fail quickly now if we need to: def evaluateRuntimeAttributes = Future.fromTry(Try(runtimeAttributes)) - def generateJesParameters = Future.fromTry(Try { - val jesInputs: Set[JesInput] = generateJesInputs(jobDescriptor) ++ monitoringScript + cmdInput + def generateJesParameters = Future.fromTry( Try { + val generatedJesInputs = generateJesInputs(jobDescriptor) + val jesInputs: Set[JesInput] = generatedJesInputs ++ monitoringScript + cmdInput val jesOutputs: Set[JesFileOutput] = generateJesOutputs(jobDescriptor) ++ monitoringOutput standardParameters ++ gcsAuthParameter ++ jesInputs ++ jesOutputs diff --git a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesAsyncBackendJobExecutionActor.scala b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesAsyncBackendJobExecutionActor.scala index 6cdf32158..1fb547648 100644 --- a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesAsyncBackendJobExecutionActor.scala +++ b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesAsyncBackendJobExecutionActor.scala @@ -61,7 +61,7 @@ class TesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn private val tesEndpoint = workflowDescriptor.workflowOptions.getOrElse("endpoint", tesConfiguration.endpointURL) - override lazy val jobTag = jobDescriptor.key.tag + override lazy val jobTag: String = jobDescriptor.key.tag private def pipeline[T: FromResponseUnmarshaller]: HttpRequest => Future[T] = sendReceive ~> unmarshal[T] @@ -98,7 +98,7 @@ class TesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn Option(task.name), Option(task.description), Option(task.project), - Option(task.inputs), + Option(task.inputs(commandLineValueMapper)), Option(task.outputs), task.resources, task.dockerExecutor diff --git a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesTask.scala b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesTask.scala index f6645f009..a9929bce6 100644 --- a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesTask.scala +++ b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesTask.scala @@ -7,7 +7,7 @@ import cromwell.core.path.{DefaultPathBuilder, Path} import wdl4s.FullyQualifiedName import wdl4s.expression.NoFunctions import wdl4s.parser.MemoryUnit -import wdl4s.values.{WdlFile, WdlGlobFile, WdlSingleFile} +import wdl4s.values.{WdlFile, WdlGlobFile, WdlSingleFile, WdlValue} final case class TesTask(jobDescriptor: BackendJobDescriptor, configurationDescriptor: BackendConfigurationDescriptor, @@ -20,8 +20,8 @@ final case class TesTask(jobDescriptor: BackendJobDescriptor, private val workflowDescriptor = jobDescriptor.workflowDescriptor private val workflowName = workflowDescriptor.workflow.unqualifiedName private val fullyQualifiedTaskName = jobDescriptor.call.fullyQualifiedName - val name = fullyQualifiedTaskName - val description = jobDescriptor.toString + val name: String = fullyQualifiedTaskName + val description: String = jobDescriptor.toString // TODO validate "project" field of workflowOptions val project = { @@ -38,12 +38,18 @@ final case class TesTask(jobDescriptor: BackendJobDescriptor, Option(false) ) - private val writeFunctionFiles: Map[FullyQualifiedName, Seq[WdlFile]] = jobDescriptor - .call - .task - .evaluateFilesFromCommand(jobDescriptor.fullyQualifiedInputs, backendEngineFunctions) - .map { - case (expression, file) => expression.toWdlString -> Seq(file) + private def writeFunctionFiles(commandLineValueMapper: WdlValue => WdlValue): Map[FullyQualifiedName, Seq[WdlFile]] = { + val commandLineMappedInputs = jobDescriptor.inputDeclarations map { + case (declaration, value) => declaration.fullyQualifiedName -> commandLineValueMapper(value) + } + + jobDescriptor + .call + .task + .evaluateFilesFromCommand(commandLineMappedInputs, backendEngineFunctions) + .map { + case (expression, file) => expression.toWdlString -> Seq(file) + } } private val callInputFiles: Map[FullyQualifiedName, Seq[WdlFile]] = jobDescriptor @@ -52,7 +58,7 @@ final case class TesTask(jobDescriptor: BackendJobDescriptor, _.collectAsSeq { case w: WdlFile => w } } - val inputs: Seq[TaskParameter] = (callInputFiles ++ writeFunctionFiles) + def inputs(commandLineValueMapper: WdlValue => WdlValue): Seq[TaskParameter] = (callInputFiles ++ writeFunctionFiles(commandLineValueMapper)) .flatMap { case (fullyQualifiedName, files) => files.zipWithIndex.map { case (f, index) => TaskParameter( From 4a49f9374cedf70d6db53cb41c45c3bdfafeb1ca Mon Sep 17 00:00:00 2001 From: Thib Date: Tue, 25 Apr 2017 13:50:51 -0400 Subject: [PATCH 019/134] Execution Store and JobPaths optimizations (#2198) * Limit JobPaths creation and Improve execution store performance * Fix subworkflow paths * PR comments --- .../main/scala/cromwell/backend/io/JobPaths.scala | 5 +- .../cromwell/backend/io/JobPathsWithDocker.scala | 22 +++++--- .../scala/cromwell/backend/io/WorkflowPaths.scala | 10 +++- .../backend/io/WorkflowPathsWithDocker.scala | 9 ++-- .../standard/StandardCachingActorHelper.scala | 2 +- .../standard/StandardInitializationData.scala | 2 +- .../scala/cromwell/backend/io/JobPathsSpec.scala | 9 ++-- .../lifecycle/execution/ExecutionStore.scala | 60 +++++++++------------- .../execution/WorkflowExecutionActor.scala | 1 - .../execution/ExecutionStoreBenchmark.scala | 57 ++++++++++++++++++++ project/Dependencies.scala | 1 + project/Testing.scala | 14 +++-- .../jes/JesAsyncBackendJobExecutionActor.scala | 2 +- .../impl/jes/JesJobCachingActorHelper.scala | 2 +- .../cromwell/backend/impl/jes/JesJobPaths.scala | 11 ++-- .../backend/impl/jes/JesWorkflowPaths.scala | 15 +++--- .../SharedFileSystemJobExecutionActorSpec.scala | 6 +-- .../backend/impl/spark/SparkBackendFactory.scala | 4 +- .../impl/spark/SparkJobExecutionActor.scala | 2 +- .../impl/spark/SparkJobExecutionActorSpec.scala | 4 +- .../tes/TesAsyncBackendJobExecutionActor.scala | 2 +- .../cromwell/backend/impl/tes/TesJobPaths.scala | 21 +++++--- .../backend/impl/tes/TesWorkflowPaths.scala | 10 ++-- .../backend/impl/tes/TesJobPathsSpec.scala | 8 +-- 24 files changed, 176 insertions(+), 103 deletions(-) create mode 100644 engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ExecutionStoreBenchmark.scala diff --git a/backend/src/main/scala/cromwell/backend/io/JobPaths.scala b/backend/src/main/scala/cromwell/backend/io/JobPaths.scala index 2e0dfab43..72057c202 100644 --- a/backend/src/main/scala/cromwell/backend/io/JobPaths.scala +++ b/backend/src/main/scala/cromwell/backend/io/JobPaths.scala @@ -24,16 +24,17 @@ object JobPaths { } } -trait JobPaths { this: WorkflowPaths => +trait JobPaths { import JobPaths._ + def workflowPaths: WorkflowPaths def returnCodeFilename: String = "rc" def stdoutFilename: String = "stdout" def stderrFilename: String = "stderr" def scriptFilename: String = "script" def jobKey: JobKey - lazy val callRoot = callPathBuilder(workflowRoot, jobKey) + lazy val callRoot = callPathBuilder(workflowPaths.workflowRoot, jobKey) lazy val callExecutionRoot = callRoot lazy val stdout = callExecutionRoot.resolve(stdoutFilename) lazy val stderr = callExecutionRoot.resolve(stderrFilename) diff --git a/backend/src/main/scala/cromwell/backend/io/JobPathsWithDocker.scala b/backend/src/main/scala/cromwell/backend/io/JobPathsWithDocker.scala index 9524e8231..2febef209 100644 --- a/backend/src/main/scala/cromwell/backend/io/JobPathsWithDocker.scala +++ b/backend/src/main/scala/cromwell/backend/io/JobPathsWithDocker.scala @@ -4,15 +4,21 @@ import com.typesafe.config.Config import cromwell.backend.{BackendJobDescriptorKey, BackendWorkflowDescriptor} import cromwell.core.path.{Path, PathBuilder} -class JobPathsWithDocker(val jobKey: BackendJobDescriptorKey, - workflowDescriptor: BackendWorkflowDescriptor, - config: Config, - pathBuilders: List[PathBuilder] = WorkflowPaths.DefaultPathBuilders) extends WorkflowPathsWithDocker( - workflowDescriptor, config, pathBuilders) with JobPaths { +object JobPathsWithDocker { + def apply(jobKey: BackendJobDescriptorKey, + workflowDescriptor: BackendWorkflowDescriptor, + config: Config, + pathBuilders: List[PathBuilder] = WorkflowPaths.DefaultPathBuilders) = { + val workflowPaths = new WorkflowPathsWithDocker(workflowDescriptor, config, pathBuilders) + new JobPathsWithDocker(workflowPaths, jobKey) + } +} + +case class JobPathsWithDocker private[io] (override val workflowPaths: WorkflowPathsWithDocker, jobKey: BackendJobDescriptorKey) extends JobPaths { import JobPaths._ - + override lazy val callExecutionRoot = { callRoot.resolve("execution") } - val callDockerRoot = callPathBuilder(dockerWorkflowRoot, jobKey) + val callDockerRoot = callPathBuilder(workflowPaths.dockerWorkflowRoot, jobKey) val callExecutionDockerRoot = callDockerRoot.resolve("execution") val callInputsRoot = callRoot.resolve("inputs") @@ -30,7 +36,7 @@ class JobPathsWithDocker(val jobKey: BackendJobDescriptorKey, * * TODO: this assumes that p.startsWith(localExecutionRoot) */ - val subpath = p.subpath(executionRoot.getNameCount, p.getNameCount) + val subpath = p.subpath(workflowPaths.executionRoot.getNameCount, p.getNameCount) WorkflowPathsWithDocker.DockerRoot.resolve(subpath) } } diff --git a/backend/src/main/scala/cromwell/backend/io/WorkflowPaths.scala b/backend/src/main/scala/cromwell/backend/io/WorkflowPaths.scala index 8a784a1ae..fc7c1a960 100644 --- a/backend/src/main/scala/cromwell/backend/io/WorkflowPaths.scala +++ b/backend/src/main/scala/cromwell/backend/io/WorkflowPaths.scala @@ -45,5 +45,13 @@ trait WorkflowPaths extends PathFactory { * @param jobWorkflowDescriptor The workflow descriptor for the job. * @return The paths for the job. */ - def toJobPaths(jobKey: BackendJobDescriptorKey, jobWorkflowDescriptor: BackendWorkflowDescriptor): JobPaths + def toJobPaths(jobKey: BackendJobDescriptorKey, jobWorkflowDescriptor: BackendWorkflowDescriptor): JobPaths = { + // If the descriptors are the same, no need to create a new WorkflowPaths + if (workflowDescriptor == jobWorkflowDescriptor) toJobPaths(this, jobKey) + else toJobPaths(withDescriptor(jobWorkflowDescriptor), jobKey) + } + + protected def toJobPaths(workflowPaths: WorkflowPaths, jobKey: BackendJobDescriptorKey): JobPaths + + protected def withDescriptor(workflowDescriptor: BackendWorkflowDescriptor): WorkflowPaths } diff --git a/backend/src/main/scala/cromwell/backend/io/WorkflowPathsWithDocker.scala b/backend/src/main/scala/cromwell/backend/io/WorkflowPathsWithDocker.scala index 4e7811feb..1809283b4 100644 --- a/backend/src/main/scala/cromwell/backend/io/WorkflowPathsWithDocker.scala +++ b/backend/src/main/scala/cromwell/backend/io/WorkflowPathsWithDocker.scala @@ -8,11 +8,12 @@ object WorkflowPathsWithDocker { val DockerRoot: Path = DefaultPathBuilder.get("/cromwell-executions") } -class WorkflowPathsWithDocker(val workflowDescriptor: BackendWorkflowDescriptor, val config: Config, val pathBuilders: List[PathBuilder] = WorkflowPaths.DefaultPathBuilders) extends WorkflowPaths { +final case class WorkflowPathsWithDocker(workflowDescriptor: BackendWorkflowDescriptor, config: Config, pathBuilders: List[PathBuilder] = WorkflowPaths.DefaultPathBuilders) extends WorkflowPaths { val dockerWorkflowRoot: Path = workflowPathBuilder(WorkflowPathsWithDocker.DockerRoot) - override def toJobPaths(jobKey: BackendJobDescriptorKey, - jobWorkflowDescriptor: BackendWorkflowDescriptor): JobPathsWithDocker = { - new JobPathsWithDocker(jobKey, jobWorkflowDescriptor, config, pathBuilders) + override def toJobPaths(workflowPaths: WorkflowPaths, jobKey: BackendJobDescriptorKey): JobPathsWithDocker = { + new JobPathsWithDocker(workflowPaths.asInstanceOf[WorkflowPathsWithDocker], jobKey) } + + override protected def withDescriptor(workflowDescriptor: BackendWorkflowDescriptor): WorkflowPaths = this.copy(workflowDescriptor = workflowDescriptor) } diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardCachingActorHelper.scala b/backend/src/main/scala/cromwell/backend/standard/StandardCachingActorHelper.scala index cb76541a1..7b259ccdc 100644 --- a/backend/src/main/scala/cromwell/backend/standard/StandardCachingActorHelper.scala +++ b/backend/src/main/scala/cromwell/backend/standard/StandardCachingActorHelper.scala @@ -36,7 +36,7 @@ trait StandardCachingActorHelper extends JobCachingActorHelper { def serviceRegistryActor: ActorRef // So... JobPaths doesn't extend WorkflowPaths, but does contain a self-type - lazy val workflowPaths: WorkflowPaths = jobPaths.asInstanceOf[WorkflowPaths] + lazy val workflowPaths: WorkflowPaths = jobPaths.workflowPaths def getPath(str: String): Try[Path] = workflowPaths.getPath(str) diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardInitializationData.scala b/backend/src/main/scala/cromwell/backend/standard/StandardInitializationData.scala index e4dae0e0b..734377dd5 100644 --- a/backend/src/main/scala/cromwell/backend/standard/StandardInitializationData.scala +++ b/backend/src/main/scala/cromwell/backend/standard/StandardInitializationData.scala @@ -15,7 +15,7 @@ class StandardInitializationData standardExpressionFunctionsClass.getConstructor(classOf[StandardExpressionFunctionsParams]) def expressionFunctions(jobPaths: JobPaths): StandardExpressionFunctions = { - val pathBuilders = jobPaths.asInstanceOf[WorkflowPaths].pathBuilders + val pathBuilders = jobPaths.workflowPaths.pathBuilders val callContext = jobPaths.callContext val standardParams = DefaultStandardExpressionFunctionsParams(pathBuilders, callContext) standardExpressionFunctionsConstructor.newInstance(standardParams) diff --git a/backend/src/test/scala/cromwell/backend/io/JobPathsSpec.scala b/backend/src/test/scala/cromwell/backend/io/JobPathsSpec.scala index 2b38535ad..793680b6c 100644 --- a/backend/src/test/scala/cromwell/backend/io/JobPathsSpec.scala +++ b/backend/src/test/scala/cromwell/backend/io/JobPathsSpec.scala @@ -31,7 +31,8 @@ class JobPathsSpec extends FlatSpec with Matchers with BackendSpec { val wd = buildWorkflowDescriptor(TestWorkflows.HelloWorld) val call: TaskCall = wd.workflow.taskCalls.head val jobKey = BackendJobDescriptorKey(call, None, 1) - val jobPaths = new JobPathsWithDocker(jobKey, wd, backendConfig) + val workflowPaths = new WorkflowPathsWithDocker(wd, backendConfig) + val jobPaths = new JobPathsWithDocker(workflowPaths, jobKey) val id = wd.id jobPaths.callRoot.pathAsString shouldBe fullPath(s"local-cromwell-executions/wf_hello/$id/call-hello") @@ -58,17 +59,17 @@ class JobPathsSpec extends FlatSpec with Matchers with BackendSpec { fullPath("/cromwell-executions/dock/path") val jobKeySharded = BackendJobDescriptorKey(call, Option(0), 1) - val jobPathsSharded = new JobPathsWithDocker(jobKeySharded, wd, backendConfig) + val jobPathsSharded = new JobPathsWithDocker(workflowPaths, jobKeySharded) jobPathsSharded.callExecutionRoot.pathAsString shouldBe fullPath(s"local-cromwell-executions/wf_hello/$id/call-hello/shard-0/execution") val jobKeyAttempt = BackendJobDescriptorKey(call, None, 2) - val jobPathsAttempt = new JobPathsWithDocker(jobKeyAttempt, wd, backendConfig) + val jobPathsAttempt = new JobPathsWithDocker(workflowPaths, jobKeyAttempt) jobPathsAttempt.callExecutionRoot.pathAsString shouldBe fullPath(s"local-cromwell-executions/wf_hello/$id/call-hello/attempt-2/execution") val jobKeyShardedAttempt = BackendJobDescriptorKey(call, Option(0), 2) - val jobPathsShardedAttempt = new JobPathsWithDocker(jobKeyShardedAttempt, wd, backendConfig) + val jobPathsShardedAttempt = new JobPathsWithDocker(workflowPaths, jobKeyShardedAttempt) jobPathsShardedAttempt.callExecutionRoot.pathAsString shouldBe fullPath(s"local-cromwell-executions/wf_hello/$id/call-hello/shard-0/attempt-2/execution") } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/ExecutionStore.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/ExecutionStore.scala index 8a60ce44d..36d4c0b56 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/ExecutionStore.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/ExecutionStore.scala @@ -3,11 +3,15 @@ package cromwell.engine.workflow.lifecycle.execution import cromwell.backend.BackendJobDescriptorKey import cromwell.core.ExecutionStatus._ import cromwell.core.{CallKey, JobKey} +import cromwell.engine.workflow.lifecycle.execution.ExecutionStore.FqnIndex import cromwell.engine.workflow.lifecycle.execution.WorkflowExecutionActor.{apply => _, _} import wdl4s._ +import scala.language.postfixOps object ExecutionStore { + type FqnIndex = (String, Option[Int]) + def empty = ExecutionStore(Map.empty[JobKey, ExecutionStatus], hasNewRunnables = false) def apply(workflow: Workflow, workflowCoercedInputs: WorkflowCoercedInputs) = { @@ -29,7 +33,12 @@ final case class ExecutionStore(private val statusStore: Map[JobKey, ExecutionSt // View of the statusStore more suited for lookup based on status lazy val store: Map[ExecutionStatus, List[JobKey]] = statusStore.groupBy(_._2).mapValues(_.keys.toList) - lazy val doneKeys = store.filterKeys(_.isDoneOrBypassed).values.toList.flatten + // Takes only keys that are done, and creates a map such that they're indexed by fqn and index + // This allows for quicker lookup (by hash) instead of traversing the whole list and yields + // significant improvements at large scale (run ExecutionStoreBenchmark) + lazy val doneKeys: Map[FqnIndex, JobKey] = store.filterKeys(_.isDoneOrBypassed).values.flatten.map { key => + (key.scope.fullyQualifiedName, key.index) -> key + } toMap private def keysWithStatus(status: ExecutionStatus) = store.getOrElse(status, List.empty) @@ -71,48 +80,35 @@ final case class ExecutionStore(private val statusStore: Map[JobKey, ExecutionSt this.copy(statusStore = statusStore ++ values, hasNewRunnables = hasNewRunnables || values.values.exists(_.isTerminal)) } - def runnableScopes = keysWithStatus(NotStarted) filter arePrerequisitesDone(doneKeys) + def runnableScopes = keysWithStatus(NotStarted) filter arePrerequisitesDone - def findCompletedShardsForOutput(key: CollectorKey): List[JobKey] = doneKeys collect { + def findCompletedShardsForOutput(key: CollectorKey): List[JobKey] = doneKeys.values.toList collect { case k @ (_: CallKey | _:DynamicDeclarationKey) if k.scope == key.scope && k.isShard => k } - // Just used to decide whether a collector can be run. In case the shard entries haven't been populated into the - // execution store yet. - private final case class TempJobKey(scope: Scope with GraphNode, index: Option[Int]) extends JobKey { - // If these are ever used, we've done something wrong... - override def attempt: Int = throw new NotImplementedError("We've done something wrong.") - override def tag: String = throw new NotImplementedError("We've done something wrong.") - } - - private def emulateShardEntries(key: CollectorKey): List[JobKey] = { - (0 until key.scatterWidth).toList flatMap { i => key.scope match { - case c: Call => List(TempJobKey(c, Option(i))) - case d: Declaration => List(TempJobKey(d, Option(i))) + private def emulateShardEntries(key: CollectorKey): Set[FqnIndex] = { + (0 until key.scatterWidth).toSet map { i: Int => key.scope match { + case c: Call => c.fullyQualifiedName -> Option(i) + case d: Declaration => d.fullyQualifiedName -> Option(i) case _ => throw new RuntimeException("Don't collect that.") }} } - private def arePrerequisitesDone(doneKeys: List[JobKey])(key: JobKey): Boolean = { - val upstreamAreDone = key.scope.upstream forall { - case n @ (_: Call | _: Scatter | _: Declaration) => upstreamIsDone(key, n, doneKeys) + private def arePrerequisitesDone(key: JobKey): Boolean = { + lazy val upstreamAreDone = key.scope.upstream forall { + case n @ (_: Call | _: Scatter | _: Declaration) => upstreamIsDone(key, n) case _ => true } - lazy val shardEntriesForCollectorAreDone: Boolean = key match { - case collector: CollectorKey => - emulateShardEntries(collector) - .map(shard => (shard.scope.fullyQualifiedName, shard.index)) - .diff( - doneKeys.map(key => (key.scope.fullyQualifiedName, key.index)) - ).isEmpty + val shardEntriesForCollectorAreDone: Boolean = key match { + case collector: CollectorKey => emulateShardEntries(collector).diff(doneKeys.keys.toSet).isEmpty case _ => true } - upstreamAreDone && shardEntriesForCollectorAreDone + shardEntriesForCollectorAreDone && upstreamAreDone } - private def upstreamIsDone(entry: JobKey, prerequisiteScope: Scope, doneKeys: List[JobKey]): Boolean = { + private def upstreamIsDone(entry: JobKey, prerequisiteScope: Scope): Boolean = { prerequisiteScope.closestCommonAncestor(entry.scope) match { /* * If this entry refers to a Scope which has a common ancestor with prerequisiteScope @@ -123,19 +119,13 @@ final case class ExecutionStore(private val statusStore: Map[JobKey, ExecutionSt * NOTE: this algorithm was designed for ONE-LEVEL of scattering and probably does not * work as-is for nested scatter blocks */ - case Some(ancestor: Scatter) => - doneKeys exists { k => - k.scope == prerequisiteScope && k.index == entry.index - } + case Some(ancestor: Scatter) => doneKeys.contains(prerequisiteScope.fullyQualifiedName -> entry.index) /* * Otherwise, simply refer to the collector entry. This means that 'entry' depends * on every shard of the pre-requisite scope to finish. */ - case _ => - doneKeys exists { k => - k.scope == prerequisiteScope && k.index.isEmpty - } + case _ => doneKeys.contains(prerequisiteScope.fullyQualifiedName -> None) } } } diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala index 254a7a1b0..883d3c653 100644 --- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala +++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/execution/WorkflowExecutionActor.scala @@ -19,7 +19,6 @@ import cromwell.util.StopAndLogSupervisor import cromwell.webservice.EngineStatsActor import lenthall.exception.ThrowableAggregation import lenthall.util.TryUtil -import wdl4s.values.{WdlArray, WdlBoolean, WdlOptionalValue, WdlValue, WdlString} import org.apache.commons.lang3.StringUtils import wdl4s.WdlExpression.ScopedLookupFunction import wdl4s.expression.WdlFunctions diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ExecutionStoreBenchmark.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ExecutionStoreBenchmark.scala new file mode 100644 index 000000000..091bd3c88 --- /dev/null +++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/ExecutionStoreBenchmark.scala @@ -0,0 +1,57 @@ +package cromwell.engine.workflow.lifecycle.execution + +import cromwell.backend.BackendJobDescriptorKey +import cromwell.core.ExecutionStatus.{apply => _, _} +import cromwell.core.{ExecutionStatus, JobKey} +import cromwell.util.SampleWdl +import org.scalameter.api._ +import wdl4s.{TaskCall, WdlNamespaceWithWorkflow} +import org.scalameter.picklers.Implicits._ + +/** + * Benchmarks the performance of the execution store using ScalaMeter (http://scalameter.github.io/) + * This is not run automatically by "sbt test". To run this test specifically, either use intellij integration, or run + * sbt "project engine" "benchmark:test-only cromwell.engine.workflow.lifecycle.execution.ExecutionStoreBenchmark" + * sbt benchmark:test will run all ScalaMeter tests + */ +object ExecutionStoreBenchmark extends Bench[Double] { + + /* Benchmark configuration */ + lazy val measurer = new Measurer.Default + lazy val executor = SeparateJvmsExecutor(new Executor.Warmer.Default, Aggregator.average, measurer) + lazy val reporter = new LoggingReporter[Double] + lazy val persistor = Persistor.None + + val wdl = WdlNamespaceWithWorkflow.load(SampleWdl.PrepareScatterGatherWdl().wdlSource(), Seq.empty).get + val prepareCall: TaskCall = wdl.workflow.findCallByName("do_prepare").get.asInstanceOf[TaskCall] + val scatterCall: TaskCall = wdl.workflow.findCallByName("do_scatter").get.asInstanceOf[TaskCall] + + def makeKey(call: TaskCall, executionStatus: ExecutionStatus)(index: Int) = { + BackendJobDescriptorKey(call, Option(index), 1) -> executionStatus + } + + // Generates numbers from 1000 to 10000 with 1000 gap: + // 1000, 2000, ..., 10000 + val sizes: Gen[Int] = Gen.range("size")(1000, 10000, 1000) + + // Generates executionStores using the given above sizes + // Each execution store contains X simulated shards of "prepareCall" in status Done and X simulated shards of "scatterCall" in status NotStarted + // This provides a good starting point to evaluate the speed of "runnableCalls", as it needs to iterate over all "NotStarted" keys, and for each one + // look for their upstreams keys in status "Done" + val executionStores: Gen[ExecutionStore] = for { + size <- sizes + doneMap = (0 until size map makeKey(prepareCall, ExecutionStatus.Done)).toMap + notStartedMap = (0 until size map makeKey(scatterCall, ExecutionStatus.NotStarted)).toMap + finalMap: Map[JobKey, ExecutionStatus] = doneMap ++ notStartedMap + } yield new ExecutionStore(finalMap, true) + + performance of "ExecutionStore" in { + // Measures how fast the execution store can find runnable calls with lots of "Done" calls and "NotStarted" calls. + // Other "shapes" would be valuable to get a better sense of how this method behaves in various situations (with Collector Keys etc...) + measure method "runnableCalls" in { + using(executionStores) in { es => + es.runnableScopes + } + } + } +} diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 6c18c81e8..68c48a0db 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -164,6 +164,7 @@ object Dependencies { val engineDependencies = List( "commons-codec" % "commons-codec" % "1.10", "commons-io" % "commons-io" % "2.5", + "com.storm-enroute" %% "scalameter" % "0.8.2", "io.swagger" % "swagger-parser" % "1.0.22" % Test, "org.yaml" % "snakeyaml" % "1.17" % Test ) ++ sprayServerDependencies diff --git a/project/Testing.scala b/project/Testing.scala index 46ca8f155..157e6bc98 100644 --- a/project/Testing.scala +++ b/project/Testing.scala @@ -6,6 +6,7 @@ object Testing { lazy val DockerTest = config("docker") extend Test lazy val NoDockerTest = config("nodocker") extend Test lazy val CromwellIntegrationTest = config("integration") extend Test + lazy val CromwellBenchmarkTest = config("benchmark") extend Test lazy val CromwellNoIntegrationTest = config("nointegration") extend Test lazy val DbmsTest = config("dbms") extend Test @@ -14,10 +15,8 @@ object Testing { lazy val DontUseDockerTaggedTests = Tests.Argument(TestFrameworks.ScalaTest, "-l", DockerTestTag) lazy val CromwellIntegrationTestTag = "CromwellIntegrationTest" - lazy val UseCromwellIntegrationTaggedTests = - Tests.Argument(TestFrameworks.ScalaTest, "-n", CromwellIntegrationTestTag) - lazy val DontUseCromwellIntegrationTaggedTests = - Tests.Argument(TestFrameworks.ScalaTest, "-l", CromwellIntegrationTestTag) + lazy val UseCromwellIntegrationTaggedTests = Tests.Argument(TestFrameworks.ScalaTest, "-n", CromwellIntegrationTestTag) + lazy val DontUseCromwellIntegrationTaggedTests = Tests.Argument(TestFrameworks.ScalaTest, "-l", CromwellIntegrationTestTag) lazy val GcsIntegrationTestTag = "GcsIntegrationTest" lazy val UseGcsIntegrationTaggedTests = Tests.Argument(TestFrameworks.ScalaTest, "-n", GcsIntegrationTestTag) @@ -54,7 +53,11 @@ object Testing { // `nointegration:test` - Run all tests, except integration testOptions in CromwellNoIntegrationTest := (testOptions in AllTests).value ++ Seq(DontUseCromwellIntegrationTaggedTests, DontUseGcsIntegrationTaggedTests, DontUsePostMVPTaggedTests), // `dbms:test` - Run database management tests. - testOptions in DbmsTest := (testOptions in AllTests).value ++ Seq(UseDbmsTaggedTests) + testOptions in DbmsTest := (testOptions in AllTests).value ++ Seq(UseDbmsTaggedTests), + // Add scalameter as a test framework in the CromwellBenchmarkTest scope + testFrameworks in CromwellBenchmarkTest += new TestFramework("org.scalameter.ScalaMeterFramework"), + // Don't execute benchmarks in parallel + parallelExecution in CromwellBenchmarkTest := false ) /* TODO: This syntax of test in (NoTests, assembly) isn't correct @@ -84,6 +87,7 @@ object Testing { .configs(DockerTest).settings(inConfig(DockerTest)(Defaults.testTasks): _*) .configs(NoDockerTest).settings(inConfig(NoDockerTest)(Defaults.testTasks): _*) .configs(CromwellIntegrationTest).settings(inConfig(CromwellIntegrationTest)(Defaults.testTasks): _*) + .configs(CromwellBenchmarkTest).settings(inConfig(CromwellBenchmarkTest)(Defaults.testTasks): _*) .configs(CromwellNoIntegrationTest).settings(inConfig(CromwellNoIntegrationTest)(Defaults.testTasks): _*) .configs(DbmsTest).settings(inConfig(DbmsTest)(Defaults.testTasks): _*) } diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala index cc8524946..19b9b609d 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesAsyncBackendJobExecutionActor.scala @@ -115,7 +115,7 @@ class JesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn private def gcsAuthParameter: Option[JesInput] = { if (jesAttributes.auths.gcs.requiresAuthFile || dockerConfiguration.isDefined) - Option(JesLiteralInput(ExtraConfigParamName, jesCallPaths.gcsAuthFilePath.pathAsString)) + Option(JesLiteralInput(ExtraConfigParamName, jesCallPaths.workflowPaths.gcsAuthFilePath.pathAsString)) else None } diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobCachingActorHelper.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobCachingActorHelper.scala index acdca18e2..99bde5142 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobCachingActorHelper.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobCachingActorHelper.scala @@ -40,7 +40,7 @@ trait JesJobCachingActorHelper extends StandardCachingActorHelper { lazy val jesAttributes: JesAttributes = jesConfiguration.jesAttributes lazy val monitoringScript: Option[JesInput] = { - jesCallPaths.monitoringPath map { path => + jesCallPaths.workflowPaths.monitoringPath map { path => JesFileInput(s"$MonitoringParamName-in", path.pathAsString, JesWorkingDisk.MountPoint.resolve(JesMonitoringScript), workingDisk) } diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobPaths.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobPaths.scala index 3dbf11b91..cd24aa3a8 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobPaths.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesJobPaths.scala @@ -1,24 +1,23 @@ package cromwell.backend.impl.jes import akka.actor.ActorSystem -import cromwell.backend.io.JobPaths import cromwell.backend.{BackendJobDescriptorKey, BackendWorkflowDescriptor} +import cromwell.backend.io.JobPaths import cromwell.core.path.Path import cromwell.services.metadata.CallMetadataKeys object JesJobPaths { def apply(jobKey: BackendJobDescriptorKey, workflowDescriptor: BackendWorkflowDescriptor, jesConfiguration: JesConfiguration)(implicit actorSystem: ActorSystem): JesJobPaths = { - new JesJobPaths(jobKey, workflowDescriptor, jesConfiguration) + val workflowPath = new JesWorkflowPaths(workflowDescriptor, jesConfiguration) + new JesJobPaths(workflowPath, jobKey) } val JesLogPathKey = "jesLog" val GcsExecPathKey = "gcsExec" } -class JesJobPaths(val jobKey: BackendJobDescriptorKey, workflowDescriptor: BackendWorkflowDescriptor, - jesConfiguration: JesConfiguration)(implicit actorSystem: ActorSystem) extends - JesWorkflowPaths(workflowDescriptor, jesConfiguration)(actorSystem) with JobPaths { +case class JesJobPaths(override val workflowPaths: JesWorkflowPaths, jobKey: BackendJobDescriptorKey)(implicit actorSystem: ActorSystem) extends JobPaths { val jesLogBasename = { val index = jobKey.index.map(s => s"-$s").getOrElse("") @@ -46,7 +45,7 @@ class JesJobPaths(val jobKey: BackendJobDescriptorKey, workflowDescriptor: Backe override lazy val customMetadataPaths = Map( CallMetadataKeys.BackendLogsPrefix + ":log" -> jesLogPath ) ++ ( - monitoringPath map { p => Map(JesMetadataKeys.MonitoringLog -> p) } getOrElse Map.empty + workflowPaths.monitoringPath map { p => Map(JesMetadataKeys.MonitoringLog -> p) } getOrElse Map.empty ) override lazy val customDetritusPaths: Map[String, Path] = Map( diff --git a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesWorkflowPaths.scala b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesWorkflowPaths.scala index b0d0eb3ed..454c06a7d 100644 --- a/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesWorkflowPaths.scala +++ b/supportedBackends/jes/src/main/scala/cromwell/backend/impl/jes/JesWorkflowPaths.scala @@ -14,14 +14,9 @@ import scala.language.postfixOps object JesWorkflowPaths { private val GcsRootOptionKey = "jes_gcs_root" private val AuthFilePathOptionKey = "auth_bucket" - - def apply(workflowDescriptor: BackendWorkflowDescriptor, - jesConfiguration: JesConfiguration)(implicit actorSystem: ActorSystem): JesWorkflowPaths = { - new JesWorkflowPaths(workflowDescriptor, jesConfiguration) - } } -class JesWorkflowPaths(val workflowDescriptor: BackendWorkflowDescriptor, +case class JesWorkflowPaths(workflowDescriptor: BackendWorkflowDescriptor, jesConfiguration: JesConfiguration)(implicit actorSystem: ActorSystem) extends WorkflowPaths { override lazy val executionRootString: String = @@ -52,10 +47,12 @@ class JesWorkflowPaths(val workflowDescriptor: BackendWorkflowDescriptor, getPath(path).get } - override def toJobPaths(jobKey: BackendJobDescriptorKey, - jobWorkflowDescriptor: BackendWorkflowDescriptor): JesJobPaths = { - JesJobPaths(jobKey, jobWorkflowDescriptor, jesConfiguration) + override def toJobPaths(workflowPaths: WorkflowPaths, jobKey: BackendJobDescriptorKey): JesJobPaths = { + new JesJobPaths(workflowPaths.asInstanceOf[JesWorkflowPaths], jobKey) } + + override protected def withDescriptor(workflowDescriptor: BackendWorkflowDescriptor): WorkflowPaths = this.copy(workflowDescriptor = workflowDescriptor) + override def config: Config = jesConfiguration.configurationDescriptor.backendConfig override def pathBuilders: List[PathBuilder] = List(gcsPathBuilder) } diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala index bb2e8211b..7332acabf 100644 --- a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala +++ b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala @@ -113,7 +113,7 @@ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSyst val jobDescriptor: BackendJobDescriptor = jobDescriptorFromSingleCallWorkflow(workflowDescriptor, inputs, WorkflowOptions.empty, runtimeAttributeDefinitions) val expectedResponse = JobSucceededResponse(jobDescriptor.key, Some(0), expectedOutputs, None, Seq.empty) - val jobPaths = new JobPathsWithDocker(jobDescriptor.key, workflowDescriptor, conf.backendConfig) + val jobPaths = JobPathsWithDocker(jobDescriptor.key, workflowDescriptor, conf.backendConfig) whenReady(backend.execute) { executionResponse => assertResponse(executionResponse, expectedResponse) @@ -161,7 +161,7 @@ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSyst val backendRef = createBackendRef(jobDescriptor, TestConfig.backendRuntimeConfigDescriptor) val backend = backendRef.underlyingActor - val jobPaths = new JobPathsWithDocker(jobDescriptor.key, workflowDescriptor, ConfigFactory.empty) + val jobPaths = JobPathsWithDocker(jobDescriptor.key, workflowDescriptor, ConfigFactory.empty) jobPaths.callExecutionRoot.createPermissionedDirectories() jobPaths.stdout.write("Hello stubby ! ") jobPaths.stderr.touch() @@ -252,7 +252,7 @@ class SharedFileSystemJobExecutionActorSpec extends TestKitSuite("SharedFileSyst val workflowDescriptor = buildWorkflowDescriptor(OutputProcess, inputs) val jobDescriptor: BackendJobDescriptor = jobDescriptorFromSingleCallWorkflow(workflowDescriptor, inputs, WorkflowOptions.empty, runtimeAttributeDefinitions) val backend = createBackend(jobDescriptor, TestConfig.backendRuntimeConfigDescriptor) - val jobPaths = new JobPathsWithDocker(jobDescriptor.key, workflowDescriptor, TestConfig.backendRuntimeConfigDescriptor.backendConfig) + val jobPaths = JobPathsWithDocker(jobDescriptor.key, workflowDescriptor, TestConfig.backendRuntimeConfigDescriptor.backendConfig) val expectedA = WdlFile(jobPaths.callExecutionRoot.resolve("a").toAbsolutePath.pathAsString) val expectedB = WdlFile(jobPaths.callExecutionRoot.resolve("dir").toAbsolutePath.resolve("b").pathAsString) val expectedOutputs = Map( diff --git a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkBackendFactory.scala b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkBackendFactory.scala index 40d625953..58df00f37 100644 --- a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkBackendFactory.scala +++ b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkBackendFactory.scala @@ -23,8 +23,8 @@ case class SparkBackendFactory(name: String, configurationDescriptor: BackendCon override def expressionLanguageFunctions(workflowDescriptor: BackendWorkflowDescriptor, jobKey: BackendJobDescriptorKey, initializationData: Option[BackendInitializationData]): WdlStandardLibraryFunctions = { - val jobPaths = new JobPathsWithDocker(jobKey, workflowDescriptor, configurationDescriptor.backendConfig) - val callContext = new CallContext( + val jobPaths = JobPathsWithDocker(jobKey, workflowDescriptor, configurationDescriptor.backendConfig) + val callContext = CallContext( jobPaths.callExecutionRoot, jobPaths.stdout.toAbsolutePath.toString, jobPaths.stderr.toAbsolutePath.toString diff --git a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkJobExecutionActor.scala b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkJobExecutionActor.scala index 80c73baa3..987e85f9e 100644 --- a/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkJobExecutionActor.scala +++ b/supportedBackends/spark/src/main/scala/cromwell/backend/impl/spark/SparkJobExecutionActor.scala @@ -44,7 +44,7 @@ class SparkJobExecutionActor(override val jobDescriptor: BackendJobDescriptor, private val sparkDeployMode = configurationDescriptor.backendConfig.getString("deployMode").toLowerCase override val sharedFileSystemConfig = fileSystemsConfig.getConfig("local") private val workflowDescriptor = jobDescriptor.workflowDescriptor - private val jobPaths = new JobPathsWithDocker(jobDescriptor.key, workflowDescriptor, configurationDescriptor.backendConfig) + private val jobPaths = JobPathsWithDocker(jobDescriptor.key, workflowDescriptor, configurationDescriptor.backendConfig) // Files private val executionDir = jobPaths.callExecutionRoot diff --git a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkJobExecutionActorSpec.scala b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkJobExecutionActorSpec.scala index 0020977f8..0c61145ea 100644 --- a/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkJobExecutionActorSpec.scala +++ b/supportedBackends/spark/src/test/scala/cromwell/backend/impl/spark/SparkJobExecutionActorSpec.scala @@ -438,7 +438,7 @@ class SparkJobExecutionActorSpec extends TestKitSuite("SparkJobExecutionActor") } private def cleanUpJob(jobPaths: JobPathsWithDocker): Unit = { - File(jobPaths.workflowRoot).delete(true) + File(jobPaths.workflowPaths.workflowRoot).delete(true) () } @@ -446,7 +446,7 @@ class SparkJobExecutionActorSpec extends TestKitSuite("SparkJobExecutionActor") val backendWorkflowDescriptor = buildWorkflowDescriptor(wdl = wdlSource, inputs = inputFiles.getOrElse(Map.empty), runtime = runtimeString) val backendConfigurationDescriptor = if (isCluster) BackendConfigurationDescriptor(backendClusterConfig, ConfigFactory.load) else BackendConfigurationDescriptor(backendClientConfig, ConfigFactory.load) val jobDesc = jobDescriptorFromSingleCallWorkflow(backendWorkflowDescriptor, inputFiles.getOrElse(Map.empty), WorkflowOptions.empty, Set.empty) - val jobPaths = if (isCluster) new JobPathsWithDocker(jobDesc.key, backendWorkflowDescriptor, backendClusterConfig) else new JobPathsWithDocker(jobDesc.key, backendWorkflowDescriptor, backendClientConfig) + val jobPaths = if (isCluster) JobPathsWithDocker(jobDesc.key, backendWorkflowDescriptor, backendClusterConfig) else JobPathsWithDocker(jobDesc.key, backendWorkflowDescriptor, backendClientConfig) val executionDir = jobPaths.callExecutionRoot val stdout = File(executionDir.toString, "stdout") stdout.createIfNotExists(asDirectory = false, createParents = true) diff --git a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesAsyncBackendJobExecutionActor.scala b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesAsyncBackendJobExecutionActor.scala index 6cdf32158..4bff6948d 100644 --- a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesAsyncBackendJobExecutionActor.scala +++ b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesAsyncBackendJobExecutionActor.scala @@ -70,7 +70,7 @@ class TesAsyncBackendJobExecutionActor(override val standardParams: StandardAsyn override def mapCommandLineWdlFile(wdlFile: WdlFile): WdlFile = { val localPath = DefaultPathBuilder.get(wdlFile.valueString).toAbsolutePath localPath match { - case p if p.startsWith(tesJobPaths.DockerRoot) => + case p if p.startsWith(tesJobPaths.workflowPaths.DockerRoot) => val containerPath = p.pathAsString WdlFile(containerPath) case p if p.startsWith(tesJobPaths.callExecutionRoot) => diff --git a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesJobPaths.scala b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesJobPaths.scala index 928fd0fa8..2091c85c0 100644 --- a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesJobPaths.scala +++ b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesJobPaths.scala @@ -1,22 +1,29 @@ package cromwell.backend.impl.tes import com.typesafe.config.Config -import cromwell.backend.io.{JobPaths, WorkflowPaths} import cromwell.backend.{BackendJobDescriptorKey, BackendWorkflowDescriptor} +import cromwell.backend.io.{JobPaths, WorkflowPaths} import cromwell.core.path.{DefaultPathBuilder, Path, PathBuilder} -class TesJobPaths(val jobKey: BackendJobDescriptorKey, - workflowDescriptor: BackendWorkflowDescriptor, - config: Config, - pathBuilders: List[PathBuilder] = WorkflowPaths.DefaultPathBuilders) extends TesWorkflowPaths( - workflowDescriptor, config, pathBuilders) with JobPaths { +object TesJobPaths { + def apply(jobKey: BackendJobDescriptorKey, + workflowDescriptor: BackendWorkflowDescriptor, + config: Config, + pathBuilders: List[PathBuilder] = WorkflowPaths.DefaultPathBuilders) = { + val workflowPaths = TesWorkflowPaths(workflowDescriptor, config, pathBuilders) + new TesJobPaths(workflowPaths, jobKey) + } +} + +case class TesJobPaths private[tes] (override val workflowPaths: TesWorkflowPaths, + jobKey: BackendJobDescriptorKey) extends JobPaths { import JobPaths._ override lazy val callExecutionRoot = { callRoot.resolve("execution") } - val callDockerRoot = callPathBuilder(dockerWorkflowRoot, jobKey) + val callDockerRoot = callPathBuilder(workflowPaths.dockerWorkflowRoot, jobKey) val callExecutionDockerRoot = callDockerRoot.resolve("execution") val callInputsDockerRoot = callDockerRoot.resolve("inputs") val callInputsRoot = callRoot.resolve("inputs") diff --git a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesWorkflowPaths.scala b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesWorkflowPaths.scala index 0fcd9c4f9..c85cee1a0 100644 --- a/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesWorkflowPaths.scala +++ b/supportedBackends/tes/src/main/scala/cromwell/backend/impl/tes/TesWorkflowPaths.scala @@ -6,7 +6,7 @@ import cromwell.backend.{BackendJobDescriptorKey, BackendWorkflowDescriptor} import cromwell.core.path.{PathBuilder, PathFactory} import net.ceedubs.ficus.Ficus._ -class TesWorkflowPaths(override val workflowDescriptor: BackendWorkflowDescriptor, +case class TesWorkflowPaths(override val workflowDescriptor: BackendWorkflowDescriptor, override val config: Config, override val pathBuilders: List[PathBuilder] = WorkflowPaths.DefaultPathBuilders) extends WorkflowPaths { @@ -17,8 +17,10 @@ class TesWorkflowPaths(override val workflowDescriptor: BackendWorkflowDescripto } val dockerWorkflowRoot = workflowPathBuilder(DockerRoot) - override def toJobPaths(jobKey: BackendJobDescriptorKey, - jobWorkflowDescriptor: BackendWorkflowDescriptor): TesJobPaths = { - new TesJobPaths(jobKey, jobWorkflowDescriptor, config, pathBuilders) + override def toJobPaths(workflowPaths: WorkflowPaths, + jobKey: BackendJobDescriptorKey): TesJobPaths = { + new TesJobPaths(workflowPaths.asInstanceOf[TesWorkflowPaths], jobKey) } + + override protected def withDescriptor(workflowDescriptor: BackendWorkflowDescriptor): WorkflowPaths = this.copy(workflowDescriptor = workflowDescriptor) } diff --git a/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesJobPathsSpec.scala b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesJobPathsSpec.scala index 1d2ed7db6..ccfe67dd8 100644 --- a/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesJobPathsSpec.scala +++ b/supportedBackends/tes/src/test/scala/cromwell/backend/impl/tes/TesJobPathsSpec.scala @@ -12,7 +12,7 @@ class TesJobPathsSpec extends FlatSpec with Matchers with BackendSpec { val wd = buildWorkflowDescriptor(TestWorkflows.HelloWorld) val call: TaskCall = wd.workflow.taskCalls.head val jobKey = BackendJobDescriptorKey(call, None, 1) - val jobPaths = new TesJobPaths(jobKey, wd, TesTestConfig.backendConfig) + val jobPaths = TesJobPaths(jobKey, wd, TesTestConfig.backendConfig) val id = wd.id jobPaths.callRoot.toString shouldBe File(s"local-cromwell-executions/wf_hello/$id/call-hello").pathAsString @@ -34,17 +34,17 @@ class TesJobPathsSpec extends FlatSpec with Matchers with BackendSpec { File(s"/cromwell-executions/wf_hello/$id/call-hello/execution").pathAsString val jobKeySharded = BackendJobDescriptorKey(call, Option(0), 1) - val jobPathsSharded = new TesJobPaths(jobKeySharded, wd, TesTestConfig.backendConfig) + val jobPathsSharded = TesJobPaths(jobKeySharded, wd, TesTestConfig.backendConfig) jobPathsSharded.callExecutionRoot.toString shouldBe File(s"local-cromwell-executions/wf_hello/$id/call-hello/shard-0/execution").pathAsString val jobKeyAttempt = BackendJobDescriptorKey(call, None, 2) - val jobPathsAttempt = new TesJobPaths(jobKeyAttempt, wd, TesTestConfig.backendConfig) + val jobPathsAttempt = TesJobPaths(jobKeyAttempt, wd, TesTestConfig.backendConfig) jobPathsAttempt.callExecutionRoot.toString shouldBe File(s"local-cromwell-executions/wf_hello/$id/call-hello/attempt-2/execution").pathAsString val jobKeyShardedAttempt = BackendJobDescriptorKey(call, Option(0), 2) - val jobPathsShardedAttempt = new TesJobPaths(jobKeyShardedAttempt, wd, TesTestConfig.backendConfig) + val jobPathsShardedAttempt = TesJobPaths(jobKeyShardedAttempt, wd, TesTestConfig.backendConfig) jobPathsShardedAttempt.callExecutionRoot.toString shouldBe File(s"local-cromwell-executions/wf_hello/$id/call-hello/shard-0/attempt-2/execution").pathAsString } From 50375c78ae7f96a6768ef950794ca34c5b0b1a31 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Tue, 25 Apr 2017 18:07:33 -0400 Subject: [PATCH 020/134] Cromwell's failure is now complete --- .../migration/src/main/resources/changelog.xml | 1 + .../resources/changesets/failure_metadata_2.xml | 10 ++++ .../migration/custom/BatchedTaskChange.scala | 22 +++---- .../DeduplicateFailureMessageIds.scala | 5 +- .../ExpandSingleFailureStrings.scala | 70 ++++++++++++++++++++++ .../table/symbol/SymbolTableMigration.scala | 7 ++- .../RenameWorkflowOptionsInMetadata.scala | 5 +- .../workflowoptions/WorkflowOptionsChange.scala | 5 +- 8 files changed, 105 insertions(+), 20 deletions(-) create mode 100644 database/migration/src/main/resources/changesets/failure_metadata_2.xml create mode 100644 database/migration/src/main/scala/cromwell/database/migration/failuremetadata/ExpandSingleFailureStrings.scala diff --git a/database/migration/src/main/resources/changelog.xml b/database/migration/src/main/resources/changelog.xml index c0a72170c..f4a2432a8 100644 --- a/database/migration/src/main/resources/changelog.xml +++ b/database/migration/src/main/resources/changelog.xml @@ -61,6 +61,7 @@ +