…d_line

Correcting documenting for backend selection on the command line

README.md

@@ -317,7 +317,8 @@ A backend represents a way to run the user's command specified in the `task` sec
 * Platform Load Sharing Facility - Use `bsub` and job monitoring to run scripts.
 * Google JES - Launch jobs on Google Compute Engine through the Job Execution Service (JES).
 
-Backends are specified via the configuration option `backend.backend` which can accept the values: `sge`, `lsf`, `local`, and `jes` (e.g. `java -Dbackend.backend=sge`).
+The default backend for a workflow is specified via the configuration option `backend.defaultBackend` which can accept the values: `sge`, `lsf`, `local`, and `jes` (e.g. `java -Dbackend.defaultBackend=sge`).
+This can be overridden by adding a `backend` entry to a submission's workflow options file.
 
 ## Backend Filesystems
 

engine/src/main/scala/cromwell/engine/backend/jes/Run.scala

@@ -1,8 +1,8 @@
 package cromwell.engine.backend.jes
 
 import com.google.api.client.util.ArrayMap
+import com.google.api.services.genomics.model.{CancelOperationRequest, LoggingOptions, RunPipelineArgs, RunPipelineRequest, ServiceAccount, _}
 import com.google.api.services.genomics.{Genomics, model}
-import com.google.api.services.genomics.model.{CancelOperationRequest, LoggingOptions, Pipeline, RunPipelineArgs, RunPipelineRequest, ServiceAccount, _}
 import com.typesafe.config.ConfigFactory
 import cromwell.core.WorkflowId
 import cromwell.engine.backend.BackendCallJobDescriptor
@@ -26,6 +26,7 @@ object Run  {
   lazy val MaximumPollingInterval = Duration(ConfigFactory.load.getConfig("backend").getConfig("jes").getInt("maximumPollingInterval"), "seconds")
   val InitialPollingInterval = 5 seconds
   val PollingBackoffFactor = 1.1
+  val AcceptableEvents = Set("start", "pulling-image", "localizing-files", "running-docker", "delocalizing-files", "ok", "fail", "start-shutdown")
 
   def apply(runIdForResumption: Option[String],
             jesJobDescriptor: JesJobDescriptor,
@@ -119,13 +120,15 @@ object Run  {
       } toSeq
     } else Seq.empty
 
+    val filteredEventsList: Seq[EventStartTime] = eventsList filter { i => AcceptableEvents.contains(i.name) }
+
     // The final event is only used as the book-end for the final pairing (see below) so the name is never actually used...
     // ... which is rather a pity actually - it's a jolly good name.
     val finaleEvents = eventIfExists("endTime", op, "cromwell poll interval") ++ Seq(
       EventStartTime("The Queen flying around with a jet-pack, with Winston Churchill cheering and waving a huge Union Jack in the background", DateTime.now))
 
     // Join the Seqs together, pair up consecutive elements then make events with start and end times.
-    ((starterEvents ++ eventsList ++ finaleEvents).sliding(2) toSeq) map { case Seq(a, b) => ExecutionEventEntry(a.name, a.timestamp, b.timestamp) }
+    ((starterEvents ++ filteredEventsList ++ finaleEvents).sliding(2) toSeq) map { case Seq(a, b) => ExecutionEventEntry(a.name, a.timestamp, b.timestamp) }
   }
 
   private def eventIfExists(name: String, op: Operation, eventName: String): Seq[EventStartTime] = {

engine/src/test/scala/cromwell/engine/backend/jes/RunSpec.scala

@@ -14,11 +14,11 @@ class RunSpec extends FlatSpec with Matchers {
     val op: Operation = new Operation()
 
     val event1: ArrayMap[String, String] = ArrayMap.create(2)
-    event1.add("description", "blah")
+    event1.add("description", "pulling-image")
     event1.add("startTime", "2015-12-05T00:00:01+00:00")
 
     val event2: ArrayMap[String, String] = ArrayMap.create(2)
-    event2.add("description", "blah2")
+    event2.add("description", "start")
     event2.add("startTime", "2015-12-05T00:01:00+00:00")
 
     val events = new util.ArrayList(Seq(event1, event2).asJava)
@@ -30,14 +30,41 @@ class RunSpec extends FlatSpec with Matchers {
       "events" -> events
     )
 
-
     op.setMetadata(metadata.asJava)
 
     Run.getEventList(op) should have size 5
-    Run.getEventList(op) filter { x => x.description == "blah" } foreach { x =>
+    Run.getEventList(op) filter { x => x.description == "pulling-image" } foreach { x =>
       x.startTime.getMillis should be (new DateTime("2015-12-05T00:00:01.000Z").getMillis)
       x.endTime.getMillis should be (new DateTime("2015-12-05T00:01:00.000Z").getMillis)
     }
+  }
+
+  "JES Run" should "not parse unwated events from Operation metadata" in {
+    val op: Operation = new Operation()
+
+    val event1: ArrayMap[String, String] = ArrayMap.create(2)
+    event1.add("description", "blah")
+    event1.add("startTime", "2015-12-05T00:00:01+00:00")
+
+    val event2: ArrayMap[String, String] = ArrayMap.create(2)
+    event2.add("description", "blah2")
+    event2.add("startTime", "2015-12-05T00:01:00+00:00")
+
+    val events = new util.ArrayList(Seq(event1, event2).asJava)
+
+    val metadata: Map[String, AnyRef] = Map(
+      "createTime" -> "2015-12-05T00:00:00+00:00",
+      "startTime" -> "2015-12-05T00:00:01+00:00",
+      "endTime" -> "2015-12-05T11:00:00+00:00",
+      "events" -> events
+    )
 
+    op.setMetadata(metadata.asJava)
+
+    Run.getEventList(op) should have size 3
+    Run.getEventList(op) filter { x => x.description == "blah" } foreach { x =>
+      x.startTime.getMillis should be (None)
+      x.endTime.getMillis should be (None)
+    }
   }
 }