From 16d7816660dd4bc8cb4d2091edeeb655e1722d76 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Tue, 28 Jun 2016 13:29:27 -0400 Subject: [PATCH 1/3] Correcting documenting for backend selection on the command line --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9b26feb19..5ed4ed8fb 100644 --- a/README.md +++ b/README.md @@ -317,7 +317,7 @@ A backend represents a way to run the user's command specified in the `task` sec * Platform Load Sharing Facility - Use `bsub` and job monitoring to run scripts. * Google JES - Launch jobs on Google Compute Engine through the Job Execution Service (JES). -Backends are specified via the configuration option `backend.backend` which can accept the values: `sge`, `lsf`, `local`, and `jes` (e.g. `java -Dbackend.backend=sge`). +Backends are specified via the configuration option `backend.defaultBackend` which can accept the values: `sge`, `lsf`, `local`, and `jes` (e.g. `java -Dbackend.defaultBackend=sge`). ## Backend Filesystems From 13f6ff1e1c8d056d1d4a873a799c931734a819e3 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Tue, 28 Jun 2016 14:40:58 -0400 Subject: [PATCH 2/3] PR fixup --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5ed4ed8fb..e7e12f3ab 100644 --- a/README.md +++ b/README.md @@ -317,7 +317,8 @@ A backend represents a way to run the user's command specified in the `task` sec * Platform Load Sharing Facility - Use `bsub` and job monitoring to run scripts. * Google JES - Launch jobs on Google Compute Engine through the Job Execution Service (JES). -Backends are specified via the configuration option `backend.defaultBackend` which can accept the values: `sge`, `lsf`, `local`, and `jes` (e.g. `java -Dbackend.defaultBackend=sge`). +The default backend for a workflow is specified via the configuration option `backend.defaultBackend` which can accept the values: `sge`, `lsf`, `local`, and `jes` (e.g. `java -Dbackend.defaultBackend=sge`). +This can be overridden by adding a `backend` entry to a submission's workflow options file. ## Backend Filesystems From 42b47c18b12c32ca3af06f6d18026d8eb067e198 Mon Sep 17 00:00:00 2001 From: Ruchi Date: Thu, 18 Aug 2016 16:17:30 -0400 Subject: [PATCH 3/3] Keep relevant Jes metadata events (#1304) filter jes metadata events by specific events --- .../scala/cromwell/engine/backend/jes/Run.scala | 7 +++-- .../cromwell/engine/backend/jes/RunSpec.scala | 35 +++++++++++++++++++--- 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/engine/src/main/scala/cromwell/engine/backend/jes/Run.scala b/engine/src/main/scala/cromwell/engine/backend/jes/Run.scala index 5175572d8..5663f71ec 100644 --- a/engine/src/main/scala/cromwell/engine/backend/jes/Run.scala +++ b/engine/src/main/scala/cromwell/engine/backend/jes/Run.scala @@ -1,8 +1,8 @@ package cromwell.engine.backend.jes import com.google.api.client.util.ArrayMap +import com.google.api.services.genomics.model.{CancelOperationRequest, LoggingOptions, RunPipelineArgs, RunPipelineRequest, ServiceAccount, _} import com.google.api.services.genomics.{Genomics, model} -import com.google.api.services.genomics.model.{CancelOperationRequest, LoggingOptions, Pipeline, RunPipelineArgs, RunPipelineRequest, ServiceAccount, _} import com.typesafe.config.ConfigFactory import cromwell.core.WorkflowId import cromwell.engine.backend.BackendCallJobDescriptor @@ -26,6 +26,7 @@ object Run { lazy val MaximumPollingInterval = Duration(ConfigFactory.load.getConfig("backend").getConfig("jes").getInt("maximumPollingInterval"), "seconds") val InitialPollingInterval = 5 seconds val PollingBackoffFactor = 1.1 + val AcceptableEvents = Set("start", "pulling-image", "localizing-files", "running-docker", "delocalizing-files", "ok", "fail", "start-shutdown") def apply(runIdForResumption: Option[String], jesJobDescriptor: JesJobDescriptor, @@ -119,13 +120,15 @@ object Run { } toSeq } else Seq.empty + val filteredEventsList: Seq[EventStartTime] = eventsList filter { i => AcceptableEvents.contains(i.name) } + // The final event is only used as the book-end for the final pairing (see below) so the name is never actually used... // ... which is rather a pity actually - it's a jolly good name. val finaleEvents = eventIfExists("endTime", op, "cromwell poll interval") ++ Seq( EventStartTime("The Queen flying around with a jet-pack, with Winston Churchill cheering and waving a huge Union Jack in the background", DateTime.now)) // Join the Seqs together, pair up consecutive elements then make events with start and end times. - ((starterEvents ++ eventsList ++ finaleEvents).sliding(2) toSeq) map { case Seq(a, b) => ExecutionEventEntry(a.name, a.timestamp, b.timestamp) } + ((starterEvents ++ filteredEventsList ++ finaleEvents).sliding(2) toSeq) map { case Seq(a, b) => ExecutionEventEntry(a.name, a.timestamp, b.timestamp) } } private def eventIfExists(name: String, op: Operation, eventName: String): Seq[EventStartTime] = { diff --git a/engine/src/test/scala/cromwell/engine/backend/jes/RunSpec.scala b/engine/src/test/scala/cromwell/engine/backend/jes/RunSpec.scala index 7576deac1..9b77d4c60 100644 --- a/engine/src/test/scala/cromwell/engine/backend/jes/RunSpec.scala +++ b/engine/src/test/scala/cromwell/engine/backend/jes/RunSpec.scala @@ -14,11 +14,11 @@ class RunSpec extends FlatSpec with Matchers { val op: Operation = new Operation() val event1: ArrayMap[String, String] = ArrayMap.create(2) - event1.add("description", "blah") + event1.add("description", "pulling-image") event1.add("startTime", "2015-12-05T00:00:01+00:00") val event2: ArrayMap[String, String] = ArrayMap.create(2) - event2.add("description", "blah2") + event2.add("description", "start") event2.add("startTime", "2015-12-05T00:01:00+00:00") val events = new util.ArrayList(Seq(event1, event2).asJava) @@ -30,14 +30,41 @@ class RunSpec extends FlatSpec with Matchers { "events" -> events ) - op.setMetadata(metadata.asJava) Run.getEventList(op) should have size 5 - Run.getEventList(op) filter { x => x.description == "blah" } foreach { x => + Run.getEventList(op) filter { x => x.description == "pulling-image" } foreach { x => x.startTime.getMillis should be (new DateTime("2015-12-05T00:00:01.000Z").getMillis) x.endTime.getMillis should be (new DateTime("2015-12-05T00:01:00.000Z").getMillis) } + } + + "JES Run" should "not parse unwated events from Operation metadata" in { + val op: Operation = new Operation() + + val event1: ArrayMap[String, String] = ArrayMap.create(2) + event1.add("description", "blah") + event1.add("startTime", "2015-12-05T00:00:01+00:00") + + val event2: ArrayMap[String, String] = ArrayMap.create(2) + event2.add("description", "blah2") + event2.add("startTime", "2015-12-05T00:01:00+00:00") + + val events = new util.ArrayList(Seq(event1, event2).asJava) + + val metadata: Map[String, AnyRef] = Map( + "createTime" -> "2015-12-05T00:00:00+00:00", + "startTime" -> "2015-12-05T00:00:01+00:00", + "endTime" -> "2015-12-05T11:00:00+00:00", + "events" -> events + ) + op.setMetadata(metadata.asJava) + + Run.getEventList(op) should have size 3 + Run.getEventList(op) filter { x => x.description == "blah" } foreach { x => + x.startTime.getMillis should be (None) + x.endTime.getMillis should be (None) + } } }