Skip to content

Commit

Permalink
Validate that a docker image referenced in WDL validates.
Browse files Browse the repository at this point in the history 
Validate that if WDL contains a docker reference and it is properly formatted,
that the docker image exists.  Dealt with both 'official' and 'personal'
docker hub images and optional tags.  Added integration tests.
  • Loading branch information
@gbggrant
gbggrant committed Aug 5, 2015
1 parent 9160eef commit 5484f5d
Show file tree
Hide file tree
Showing 10 changed files with 458 additions and 11 deletions.
39 changes: 35 additions & 4 deletions src/main/scala/org/broadinstitute/dsde/agora/server/business/AgoraBusiness.scala
View file
@@ -1,5 +1,8 @@
package org.broadinstitute.dsde.agora.server.business

import cromwell.parser.WdlParser.SyntaxError
import org.broadinstitute.dsde.agora.server.webservice.util.{DockerImageReference, DockerHubClient}

import cromwell.binding._
import org.broadinstitute.dsde.agora.server.dataaccess.AgoraDao
import org.broadinstitute.dsde.agora.server.dataaccess.acls.AgoraPermissions._
Expand Down Expand Up @@ -52,14 +55,42 @@ class AgoraBusiness(authorizationProvider: AuthorizationProvider) {
private def validatePayload(agoraEntity: AgoraEntity, username: String): Unit = {
agoraEntity.entityType.get match {
case AgoraEntityType.Task =>
WdlNamespace.load(agoraEntity.payload.get)

val namespace = WdlNamespace.load(agoraEntity.payload.get)
// Passed basic validation. Now check if (any) docker images that are referenced exist
namespace.tasks.foreach { validateDockerImage }
case AgoraEntityType.Workflow =>
val resolver = MethodImportResolver(username, this, authorizationProvider)
WdlNamespace.load(agoraEntity.payload.get, resolver.importResolver _)

val namespace = WdlNamespace.load(agoraEntity.payload.get, resolver.importResolver _)
// Passed basic validation. Now check if (any) docker images that are referenced exist
namespace.tasks.foreach { validateDockerImage }
case AgoraEntityType.Configuration =>
//add config validation here
}
}

private def validateDockerImage(task: Task) = {
if (task.runtimeAttributes.docker.isDefined) {
DockerHubClient.doesDockerImageExist(parseDockerString(task.runtimeAttributes.docker.get))
}
}

/**
* Parses out user/image:tag from a docker string.
*
* @param imageId docker imageId string. Looks like ubuntu:latest ggrant/joust:latest
*/
private def parseDockerString(imageId: String) = {
val splitUser = imageId.split('/')
if (splitUser.length > 2) {
throw new SyntaxError("Docker image string '" + imageId + "' is malformed")
}
val user = if (splitUser.length == 1) None else Option(splitUser(0))
val splitTag = splitUser(splitUser.length - 1).split(':')
if (splitTag.length > 2) {
throw new SyntaxError("Docker image string '" + imageId + "' is malformed")
}
val repo = splitTag(0)
val tag = if (splitTag.length == 1) "latest" else splitTag(1)
DockerImageReference(user, repo, tag)
}
}
4 changes: 3 additions & 1 deletion src/main/scala/org/broadinstitute/dsde/agora/server/webservice/AddHandler.scala
View file
Expand Up @@ -7,6 +7,7 @@ import org.broadinstitute.dsde.agora.server.dataaccess.acls.AuthorizationProvide
import org.broadinstitute.dsde.agora.server.model.AgoraApiJsonSupport._
import org.broadinstitute.dsde.agora.server.model.{AgoraEntity, AgoraError}
import org.broadinstitute.dsde.agora.server.webservice.PerRequest.RequestComplete
import org.broadinstitute.dsde.agora.server.webservice.util.DockerHubClient.DockerImageNotFoundException
import org.broadinstitute.dsde.agora.server.webservice.util.ServiceMessages
import spray.http.StatusCodes._
import spray.httpx.SprayJsonSupport._
Expand All @@ -27,7 +28,8 @@ class AddHandler(authorizationProvider: AuthorizationProvider) extends Actor {
try {
add(requestContext, agoraAddRequest, username)
} catch {
case e: SyntaxError => context.parent ! RequestComplete(BadRequest, AgoraError("Syntax error in payload: " + e.getMessage))
case se: SyntaxError => context.parent ! RequestComplete(BadRequest, AgoraError("Syntax error in payload: " + se.getMessage))
case de: DockerImageNotFoundException => context.parent ! RequestComplete(BadRequest, AgoraError("Invalid Docker Referenced in payload: " + de.getMessage))
}
context.stop(self)
}
Expand Down
62 changes: 62 additions & 0 deletions src/main/scala/org/broadinstitute/dsde/agora/server/webservice/util/DockerHubClient.scala
View file
@@ -0,0 +1,62 @@
package org.broadinstitute.dsde.agora.server.webservice.util

import akka.actor.ActorSystem
import spray.client.pipelining._
import spray.http.HttpResponse
import spray.http.StatusCodes._
import org.broadinstitute.dsde.agora.server.webservice.util.DockerHubJsonSupport._

import scala.concurrent.duration._
import scala.concurrent.{Await, Future}

case class DockerImageReference(user: Option[String], repo: String, tag: String)

case class DockerTagInfo(pk: Int, id: String)

object DockerHubClient {
case class DockerConnectionException(text: String = "There was a problem connecting to Docker Hub APIs.") extends Exception(text)

case class DockerImageNotFoundException(dockerImage: DockerImageReference) extends Exception {
override def getMessage: String = {
s"Docker Image for User ${dockerImage.user.getOrElse("'Official'")}/" + s"${dockerImage.repo}" + s":${dockerImage.tag} not found."
}
}

implicit val actorSystem = ActorSystem("agora")

import actorSystem.dispatcher

val timeout = 5.seconds

def pipeline = sendReceive

def dcGET(url: String): Future[HttpResponse] = pipeline(Get(url))

def doesDockerImageExist(dockerImage: DockerImageReference): Boolean = {
val dockerImageInfo = dcGET(dockerImageTagUrl(dockerImage))

val dockerTagExists = dockerImageInfo map { response: HttpResponse =>
response.status match {
case OK => unmarshal[List[DockerTagInfo]].apply(response).nonEmpty
case NotFound => throw new DockerImageNotFoundException(dockerImage)
case _ => throw new DockerImageNotFoundException(dockerImage)
}
}

dockerImageInfo.onFailure { case _ => throw DockerImageNotFoundException(dockerImage) }
val isDockerImageInfoFound = Await.result(dockerTagExists, timeout)
isDockerImageInfoFound
}

val dockerImageRepositoryBaseUrl = "https://index.docker.io/v1/repositories/"
def dockerImageTagUrl(dockerImage: DockerImageReference) = {
var url = dockerImageRepositoryBaseUrl
if (dockerImage.user.isDefined) {
url += dockerImage.user.get + "/"
}
url += dockerImage.repo + "/tags/" + dockerImage.tag
url
}

}

8 changes: 8 additions & 0 deletions ...ain/scala/org/broadinstitute/dsde/agora/server/webservice/util/DockerHubJsonSupport.scala
View file
@@ -0,0 +1,8 @@
package org.broadinstitute.dsde.agora.server.webservice.util

import spray.httpx.SprayJsonSupport
import spray.json.DefaultJsonProtocol

object DockerHubJsonSupport extends DefaultJsonProtocol with SprayJsonSupport {
implicit val DockerTagInfoFormat = jsonFormat2(DockerTagInfo)
}
60 changes: 60 additions & 0 deletions src/test/scala/org/broadinstitute/dsde/agora/server/AgoraIntegrationTestData.scala
View file
Expand Up @@ -23,6 +23,66 @@ object AgoraIntegrationTestData {
payload = payload1,
entityType = Option(AgoraEntityType.Workflow))

val testAgoraEntityWithValidOfficialDockerImageInWdl = new AgoraEntity(namespace = Option("___docker_test" + System.currentTimeMillis()),
name = name1,
synopsis = synopsis1,
documentation = documentation1,
owner = Option(owner1),
payload = payloadWithValidOfficialDockerImageInWdl,
entityType = Option(AgoraEntityType.Task)
)

val testAgoraEntityWithInvalidOfficialDockerRepoNameInWdl = new AgoraEntity(namespace = Option("___docker_test" + System.currentTimeMillis()),
name = name1,
synopsis = synopsis1,
documentation = documentation1,
owner = Option(owner1),
payload = payloadWithInvalidOfficialDockerRepoNameInWdl,
entityType = Option(AgoraEntityType.Task)
)

val testAgoraEntityWithInvalidOfficialDockerTagNameInWdl = new AgoraEntity(namespace = Option("___docker_test" + System.currentTimeMillis()),
name = name1,
synopsis = synopsis1,
documentation = documentation1,
owner = Option(owner1),
payload = payloadWithInvalidOfficialDockerTagNameInWdl,
entityType = Option(AgoraEntityType.Task)
)

val testAgoraEntityWithValidPersonalDockerInWdl = new AgoraEntity(namespace = Option("___docker_test" + System.currentTimeMillis()),
name = name1,
synopsis = synopsis1,
documentation = documentation1,
owner = Option(owner1),
payload = payloadWithValidPersonalDockerImageInWdl,
entityType = Option(AgoraEntityType.Task)
)

val testAgoraEntityWithInvalidPersonalDockerUserNameInWdl = new AgoraEntity(namespace = Option("___docker_test" + System.currentTimeMillis()),
name = name1,
synopsis = synopsis1,
documentation = documentation1,
owner = Option(owner1),
payload = payloadWithInvalidPersonalDockerUserNameInWdl,
entityType = Option(AgoraEntityType.Task)
)

val testAgoraEntityWithInvalidPersonalDockerRepoNameInWdl = new AgoraEntity(namespace = Option("___docker_test" + System.currentTimeMillis()),
name = name1,
synopsis = synopsis1,
documentation = documentation1,
owner = Option(owner1),
payload = payloadWithInvalidPersonalDockerRepoNameInWdl,
entityType = Option(AgoraEntityType.Task)
)

val testAgoraEntityWithInvalidPersonalDockerTagNameInWdl = new AgoraEntity(namespace = Option("___docker_test" + System.currentTimeMillis()),
name = name1,
synopsis = synopsis1,
documentation = documentation1,
owner = Option(owner1),
payload = payloadWithInvalidPersonalDockerTagNameInWdl,
entityType = Option(AgoraEntityType.Task)
)
}
12 changes: 8 additions & 4 deletions src/test/scala/org/broadinstitute/dsde/agora/server/AgoraIntegrationTestSuite.scala
View file
@@ -1,17 +1,21 @@

package org.broadinstitute.dsde.agora.server

import org.broadinstitute.dsde.agora.server.business.AgoraBusinessIntegrationSpec
import org.broadinstitute.dsde.agora.server.dataaccess.acls.GcsAuthorizationSpec
import org.broadinstitute.dsde.agora.server.dataaccess.acls.gcs.GcsAuthorizationProvider
import org.broadinstitute.dsde.agora.server.dataaccess.mongo.EmbeddedMongo
import org.broadinstitute.dsde.agora.server.webservice.AclIntegrationSpec
import org.broadinstitute.dsde.agora.server.webservice.AgoraImportIntegrationSpec
import org.scalatest.{BeforeAndAfterAll, Suites}

class AgoraIntegrationTestSuite extends Suites(
new GcsAuthorizationSpec,
new AclIntegrationSpec) with BeforeAndAfterAll {
val agora = new Agora(AgoraConfig.DevEnvironment)

new AclIntegrationSpec,
new AgoraBusinessIntegrationSpec,
new AgoraImportIntegrationSpec
) with BeforeAndAfterAll {
val agora = new Agora(AgoraConfig.DevEnvironment) // Integration Tests use the dev environment settings
// AgoraOpenAMDirectives, GcsAuthorizationProvider, non-embedded Mongo
override def beforeAll() {
println(s"Starting Agora web services ($suiteName)")
agora.start()
Expand Down
110 changes: 109 additions & 1 deletion src/test/scala/org/broadinstitute/dsde/agora/server/AgoraTestData.scala
View file
Expand Up @@ -173,7 +173,106 @@ object AgoraTestData {
| },
| "namespace": "ns"
|}""".stripMargin)

val payloadWithValidOfficialDockerImageInWdl = Option( """
|task wc {
| command {
| wc -l ${sep=' ' File files+} | tail -1 | cut -d' ' -f 2
| }
| output {
| Int count = read_int("stdout")
| }
| runtime {
| docker: "ubuntu:latest"
| }
|}
| """.stripMargin)
val payloadWithInvalidOfficialDockerRepoNameInWdl = Option( """
|task wc {
| command {
| wc -l ${sep=' ' File files+} | tail -1 | cut -d' ' -f 2
| }
| output {
| Int count = read_int("stdout")
| }
| runtime {
| docker: "ubuntu_doesnotexist:latest"
| }
|}
| """.stripMargin)
val payloadWithInvalidOfficialDockerTagNameInWdl = Option( """
|task wc {
| command {
| wc -l ${sep=' ' File files+} | tail -1 | cut -d' ' -f 2
| }
| output {
| Int count = read_int("stdout")
| }
| runtime {
| docker: "ubuntu:ggrant_latest"
| }
|}
| """.stripMargin)
val payloadWithValidPersonalDockerImageInWdl = Option( """
|task wc {
| command {
| wc -l ${sep=' ' File files+} | tail -1 | cut -d' ' -f 2
| }
| output {
| Int count = read_int("stdout")
| }
| runtime {
| docker: "broadinstitute/scala-baseimage"
| }
|}
| """.stripMargin)
val payloadWithInvalidPersonalDockerUserNameInWdl = Option( """
|task wc {
| command {
| wc -l ${sep=' ' File files+} | tail -1 | cut -d' ' -f 2
| }
| output {
| Int count = read_int("stdout")
| }
| runtime {
| docker: "broadinstitute_doesnotexist/scala-baseimage:latest"
| memory: "2MB"
| cores: 1
| disk: "3MB"
| }
|}
| """.stripMargin)
val payloadWithInvalidPersonalDockerRepoNameInWdl = Option( """
|task wc {
| command {
| wc -l ${sep=' ' File files+} | tail -1 | cut -d' ' -f 2
| }
| output {
| Int count = read_int("stdout")
| }
| runtime {
| docker: "broadinstitute/scala-baseimage_doesnotexist:latest"
| memory: "2MB"
| cores: 1
| disk: "3MB"
| }
|}
| """.stripMargin)
val payloadWithInvalidPersonalDockerTagNameInWdl = Option( """
|task wc {
| command {
| wc -l ${sep=' ' File files+} | tail -1 | cut -d' ' -f 2
| }
| output {
| Int count = read_int("stdout")
| }
| runtime {
| docker: "broadinstitute/scala-baseimage:latest_doesnotexist"
| memory: "2MB"
| cores: 1
| disk: "3MB"
| }
|}
| """.stripMargin)
val testEntity1 = AgoraEntity(namespace = namespace1,
name = name1,
synopsis = synopsis1,
Expand Down Expand Up @@ -279,6 +378,15 @@ object AgoraTestData {
entityType = Option(AgoraEntityType.Workflow)
)

val testAgoraEntityWithInvalidOfficialDockerImageInWdl = new AgoraEntity(namespace = namespace1,
name = name1,
synopsis = synopsis1,
documentation = documentation1,
owner = owner1,
payload = payloadWithInvalidOfficialDockerTagNameInWdl,
entityType = Option(AgoraEntityType.Task)
)

val testEntityWorkflowWithExistentWdlImport = new AgoraEntity(
namespace = namespace1,
name = name1,
Expand Down

0 comments on commit 5484f5d

Please sign in to comment.