Poll official firecloud google groups for admins DSDEEPB-1388

Firecloud is containting their official list of admins in google groups for each environment.
If such a group is defined in config, poll for it's members, and synchronize our admin-user statuses with this list

build.sbt

@@ -21,7 +21,7 @@ libraryDependencies ++= Seq(
   "com.gettyimages" %% "spray-swagger" % "0.5.0",
   "com.github.simplyscala" %% "scalatest-embedmongo" % "0.2.2",
   "com.google.api-client" % "google-api-client" % "1.20.0" excludeAll ExclusionRule(organization = "com.google.guava"),
-  "com.google.apis" % "google-api-services-storage" % "v1-rev35-1.20.0",
+  "com.google.apis" % "google-api-services-admin-directory" % "directory_v1-rev53-1.20.0",
   "com.h2database" % "h2" % "1.3.175",
   "com.typesafe.akka" %% "akka-actor" % "2.3.4",
   "com.typesafe.akka" %% "akka-slf4j" % "2.3.11",

src/main/scala/org/broadinstitute/dsde/agora/server/AgoraConfig.scala

@@ -72,6 +72,7 @@ object AgoraConfig {
   lazy val webserviceInterface = config.as[Option[String]]("webservice.interface").getOrElse("0.0.0.0")
   lazy val supervisorLogging = config.as[Option[Boolean]]("supervisor.logging").getOrElse(true)
   lazy val kamonInstrumentation = config.as[Option[Boolean]]("kamon.instrumentation").getOrElse(true)
+  lazy val adminSweepInterval = config.as[Option[Int]]("admin.sweep.interval").getOrElse(15)
 
   // Mongo
   lazy val mongoDbHosts = config.as[List[String]]("mongodb.hosts")
@@ -85,8 +86,10 @@ object AgoraConfig {
 
   // Google Credentials
   lazy val gcsProjectId = config.as[String]("gcs.project.id")
-  lazy val gcsServiceAccountUserEmail = config.as[String]("gcs.service.account.email")
-  lazy val gcServiceAccountP12KeyFile = config.as[String]("gcs.service.account.p12.key.file")
+  lazy val gcsServiceAccountEmail = config.as[String]("gcs.service.account.email")
+  lazy val gcsServiceAccountP12KeyFile = config.as[String]("gcs.service.account.p12.key.file")
+  lazy val gcsUserEmail = config.as[String]("gcs.user.email")
+  lazy val adminGoogleGroup = config.as[Option[String]]("admin.google.group")
 
   //Config Settings
   object SwaggerConfig {

src/main/scala/org/broadinstitute/dsde/agora/server/dataaccess/permissions/AdminPermissionsClient.scala

@@ -0,0 +1,11 @@
+package org.broadinstitute.dsde.agora.server.dataaccess.permissions
+
+import org.broadinstitute.dsde.agora.server.model.AgoraEntity
+
+object AdminPermissionsClient extends PermissionsClient {
+
+  def listAdminUsers = listAdmins
+
+  def alias(entity: AgoraEntity) =
+    entity.namespace.get + "." + entity.name.get + "." + entity.snapshotId.get
+}

src/main/scala/org/broadinstitute/dsde/agora/server/dataaccess/permissions/AdminSweeper.scala

@@ -0,0 +1,60 @@
+package org.broadinstitute.dsde.agora.server.dataaccess.permissions
+
+import akka.actor.{Actor, Props}
+import org.broadinstitute.dsde.agora.server.AgoraConfig
+import org.broadinstitute.dsde.agora.server.webservice.util.GoogleApiUtils
+import com.google.api.services.admin.directory.model.Member
+import scala.collection.JavaConversions._
+
+object AdminSweeper {
+  def props(pollFunction: () => List[String]): Props = Props(classOf[AdminSweeper], pollFunction)
+
+  case class Sweep()
+
+  /**
+   * Function to poll for the member-emails of a config-defined google group.
+   * Google group is assumed to contain an up-to-date list of admins as members.
+   */
+  val adminsGoogleGroupPoller: () => List[String] = { () =>
+    GoogleApiUtils.getGroupDirectory
+      .members
+      .list(AgoraConfig.adminGoogleGroup.get)
+      .execute
+      .getMembers
+      .toList
+      .map(_.getEmail)
+  }
+}
+
+/**
+ * Poll for an updated list of admins, and update our user table to reflect this list.
+ * Intended to be run via a scheduler from the parent actor
+ * TODO- Implement bulk transactions for better scalability. Currently runs a DB transaction for each user whose admin status needs changing.
+ */
+class AdminSweeper(pollAdmins: () => List[String]) extends Actor {
+  import AdminSweeper.Sweep
+  def receive = {
+    case Sweep => synchronizeAdmins
+  }
+  def synchronizeAdmins: Unit = {
+    // get expected and observed admins lists
+    val trueAdmins: List[String] = pollAdmins()
+    val currentAdmins = AdminPermissionsClient.listAdminUsers
+
+    // Difference the lists
+    val newAdmins = trueAdmins.filterNot(currentAdmins.toSet)
+    val adminsToDelete = currentAdmins.filterNot(trueAdmins.toSet)
+
+    // Update our user table to reflect list differences
+    for (newAdmin <- newAdmins) {
+      println(newAdmin)
+      AdminPermissionsClient.updateAdmin(newAdmin, true)
+    }
+
+    for (adminToDelete <- adminsToDelete) {
+      AdminPermissionsClient.updateAdmin(adminToDelete, false)
+    }
+  }
+
+
+}

src/main/scala/org/broadinstitute/dsde/agora/server/dataaccess/permissions/PermissionsClient.scala

@@ -32,6 +32,44 @@ trait PermissionsClient {
     user.isAdmin
   }
 
+  def listAdmins(): Seq[String] = {
+    val adminsQuery = for {
+      user <- users if user.is_admin === true
+    } yield user.email
+
+    try {
+      Await.result(db.run(adminsQuery.result), timeout)
+    } catch {
+      case ex: Throwable =>
+        throw new PermissionNotFoundException(ex.getMessage, ex)
+    }
+  }
+
+  def updateAdmin(userEmail: String, adminStatus: Boolean) = {
+    addUserIfNotInDatabase(userEmail)
+
+    // construct update action
+    val adminsUpdateAction = for {
+      user <- users
+        .filter(_.email === userEmail)
+        .map(_.is_admin)
+        .update(adminStatus)
+    } yield user
+
+    // run update action
+    try {
+      val rowsEdited = Await.result(db.run(adminsUpdateAction), timeout)
+
+      if (rowsEdited == 0)
+        throw new Exception("No rows were edited.")
+      else
+        rowsEdited
+
+    } catch {
+      case ex: Throwable => throw new PermissionNotFoundException(s"Could not make user ${userEmail} admin", ex)
+    }
+  }
+
   // Entities
   def addEntity(entity: AgoraEntity): Future[Int] =
     Await.ready(db.run(entities += EntityDao(alias(entity))), timeout)

src/main/scala/org/broadinstitute/dsde/agora/server/webservice/ApiServiceActor.scala

@@ -1,15 +1,20 @@
 package org.broadinstitute.dsde.agora.server.webservice
 
-import akka.actor.Props
+import akka.actor.SupervisorStrategy.Restart
+import akka.actor.{OneForOneStrategy, Props}
 import com.gettyimages.spray.swagger.SwaggerHttpService
 import com.typesafe.scalalogging.slf4j.LazyLogging
 import com.wordnik.swagger.model.ApiInfo
 import org.broadinstitute.dsde.agora.server.AgoraConfig
+import org.broadinstitute.dsde.agora.server.dataaccess.permissions.AdminSweeper
+import AdminSweeper.Sweep
+import org.broadinstitute.dsde.agora.server.dataaccess.permissions.AdminSweeper
 import org.broadinstitute.dsde.agora.server.webservice.configurations.ConfigurationsService
 import org.broadinstitute.dsde.agora.server.webservice.methods.MethodsService
 import spray.http.StatusCodes._
 import spray.routing._
 import spray.util.LoggingContext
+import scala.concurrent.duration._
 
 import scala.reflect.runtime.universe._
 
@@ -25,6 +30,28 @@ class ApiServiceActor extends HttpServiceActor with LazyLogging {
     def actorRefFactory = context
   }
 
+  override val supervisorStrategy =
+    OneForOneStrategy(loggingEnabled = AgoraConfig.supervisorLogging) {
+      case e: Throwable =>
+        logger.error("ApiServiceActor child threw exception. Child will be restarted\n" + e.getMessage)
+        Restart
+    }
+
+  /**
+   * Firecloud system maintains it's set of admins as a google group.
+   *
+   * If such a group is specified in config, poll it at regular intervals
+   *   to synchronize the admins defined in our users table
+   */
+  AgoraConfig.adminGoogleGroup match {
+    case Some(group) =>
+      import context.dispatcher
+      val adminSweeper = actorRefFactory.actorOf(AdminSweeper.props(AdminSweeper.adminsGoogleGroupPoller))
+      val adminScheduler =
+        context.system.scheduler.schedule(5 seconds, AgoraConfig.adminSweepInterval minutes, adminSweeper, Sweep)
+    case None =>
+  }
+
   val methodsService = new MethodsService() with ActorRefFactoryContext
   val configurationsService = new ConfigurationsService() with ActorRefFactoryContext
 

src/main/scala/org/broadinstitute/dsde/agora/server/webservice/util/GoogleApiUtils.scala

@@ -0,0 +1,35 @@
+package org.broadinstitute.dsde.agora.server.webservice.util
+
+import java.io.File
+import java.util.Collections
+
+import com.google.api.client.auth.oauth2.Credential
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport
+import com.google.api.client.json.jackson2.JacksonFactory
+import com.google.api.services.admin.directory.{DirectoryScopes, Directory}
+import org.broadinstitute.dsde.agora.server.AgoraConfig
+
+import scala.concurrent.duration._
+
+object GoogleApiUtils {
+  val emailAddress = AgoraConfig.gcsServiceAccountEmail
+  val JSON_FACTORY = JacksonFactory.getDefaultInstance
+  val httpTransport = GoogleNetHttpTransport.newTrustedTransport
+  val directoryScopes = Collections.singleton(DirectoryScopes.ADMIN_DIRECTORY_GROUP)
+
+  private def getGroupServiceAccountCredential: Credential = {
+    new GoogleCredential.Builder()
+      .setTransport(httpTransport)
+      .setJsonFactory(JSON_FACTORY)
+      .setServiceAccountId(emailAddress)
+      .setServiceAccountScopes(directoryScopes)
+      .setServiceAccountUser(AgoraConfig.gcsUserEmail)
+      .setServiceAccountPrivateKeyFromP12File(new File(AgoraConfig.gcsServiceAccountP12KeyFile))
+      .build()
+  }
+
+  def getGroupDirectory = {
+    new Directory.Builder(httpTransport, JSON_FACTORY, getGroupServiceAccountCredential).setApplicationName("firecloud:agora").build()
+  }
+}

src/test/scala/org/broadinstitute/dsde/agora/server/AgoraTestSuite.scala

@@ -1,5 +1,6 @@
 package org.broadinstitute.dsde.agora.server
 
+import akka.actor.ActorSystem
 import org.broadinstitute.dsde.agora.server.business.{AgoraBusinessIntegrationSpec, AgoraBusinessTest}
 import org.broadinstitute.dsde.agora.server.dataaccess.mongo.MethodsDbTest
 import org.broadinstitute.dsde.agora.server.dataaccess.permissions._
@@ -21,7 +22,8 @@ class AgoraTestSuite extends Suites(
   new NamespacePermissionsClientSpec,
   new AgoraBusinessIntegrationSpec,
   new AgoraImportIntegrationSpec,
-  new PermissionIntegrationSpec) with BeforeAndAfterAll with AgoraTestFixture {
+  new PermissionIntegrationSpec,
+  new AdminSweeperSpec(ActorSystem("test"))) with BeforeAndAfterAll with AgoraTestFixture {
 
   val agora = new Agora()
 

src/test/scala/org/broadinstitute/dsde/agora/server/dataaccess/permissions/AdminSweeperSpec.scala

@@ -0,0 +1,40 @@
+package org.broadinstitute.dsde.agora.server.dataaccess.permissions
+
+import akka.actor.ActorSystem
+import akka.testkit.{ImplicitSender, TestKit, TestActorRef}
+import org.broadinstitute.dsde.agora.server.dataaccess.permissions.AdminSweeper.Sweep
+import org.broadinstitute.dsde.agora.server.{AgoraTestData, AgoraTestFixture}
+import org.scalatest.{DoNotDiscover, WordSpecLike, Matchers, BeforeAndAfterAll}
+import scala.concurrent.duration._
+
+object AdminSweeperSpec {
+  def getMockAdminsList:() =>  List[String] = { () =>
+    List("fake@broadinstitute.org", AgoraTestData.owner1.get, AgoraTestData.owner2.get)
+  }
+}
+
+@DoNotDiscover
+class AdminSweeperSpec(_system: ActorSystem) extends TestKit(_system) with WordSpecLike with Matchers with BeforeAndAfterAll with AgoraTestFixture with ImplicitSender {
+
+  override protected def beforeAll() = {
+  ensureDatabasesAreRunning()
+  }
+
+  override protected def afterAll() = {
+  clearDatabases()
+  }
+
+  "Agora" should {
+    "be able to synchronize it's list of admins via the AdminSweeper" in {
+    addAdminUser()
+
+    val adminSweeper = TestActorRef(AdminSweeper.props(AdminSweeperSpec.getMockAdminsList))
+    within(30 seconds) {
+      adminSweeper ! Sweep
+      awaitAssert(AdminPermissionsClient.listAdminUsers === AdminSweeperSpec.getMockAdminsList)
+    }
+  }
+
+  }
+
+}

src/test/scala/org/broadinstitute/dsde/agora/server/dataaccess/permissions/AgoraPermissionsSpec.scala

@@ -1,7 +1,7 @@
 
 package org.broadinstitute.dsde.agora.server.dataaccess.permissions
 
-import org.broadinstitute.dsde.agora.server.AgoraTestFixture
+import org.broadinstitute.dsde.agora.server.{AgoraTestData, AgoraTestFixture}
 import org.broadinstitute.dsde.agora.server.dataaccess.permissions.AgoraPermissions._
 import org.broadinstitute.dsde.agora.server.webservice.ApiServiceSpec
 import org.scalatest.{BeforeAndAfterAll, DoNotDiscover}
@@ -69,4 +69,25 @@ class AgoraPermissionsSpec extends ApiServiceSpec with BeforeAndAfterAll with Ag
     assert(newAuthorization.canCreate === true)
     assert(newAuthorization.canManage === false)
   }
+
+  "Agora" should "be able to list admins" in {
+    addAdminUser()
+    val adminUsers = AdminPermissionsClient.listAdminUsers
+    assert(adminUsers.length === 1)
+    assert(adminUsers.head === AgoraTestData.adminUser.get)
+  }
+
+  "Agora" should "be able to update the admin status of a user" in {
+    val adminUser = AgoraTestData.adminUser.get
+
+    // Set adminUsers's admin status false
+    AdminPermissionsClient.updateAdmin(adminUser, false)
+    var adminUsers = AdminPermissionsClient.listAdminUsers
+    assert(adminUsers.length === 0)
+
+    // Set adminUsers's admin status true
+    AdminPermissionsClient.updateAdmin(adminUser, true)
+    adminUsers = AdminPermissionsClient.listAdminUsers
+    assert(adminUsers.length === 1)
+  }
 }