-
Notifications
You must be signed in to change notification settings - Fork 4
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
IA-Add google2 code for disk management (#305)
* add new DiskService and begin DiskInterpreter * rest of interpreter methods * rm disk operations from GoogleComputeInterpreter * rm unused method * version bump, readme and changelog * fix listDisks * version bump consistency * rm duplicate ZoneName * println in listDisks for debug, ManualTestImports object for testing (TEMP) * check if nextPageToken is empty before making another request * fix listDisks and remove manualTestImports object * add traceId to testing locally readme * change version to 0.10 * change version in settings * uncomment line * update listDisks to only use .iterateAll, rm unused imports, rm protected from KubernetesServiceKindName Co-authored-by: Gabriela <gcarrill@broadinstitute.org>
- Loading branch information
Showing
9 changed files
with
163 additions
and
26 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
70 changes: 70 additions & 0 deletions
70
google2/src/main/scala/org/broadinstitute/dsde/workbench/google2/GoogleDiskInterpreter.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,70 @@ | ||
| package org.broadinstitute.dsde.workbench.google2 | ||
|
|
||
| import cats.effect.concurrent.Semaphore | ||
| import cats.effect.{Async, Blocker, ContextShift, Timer} | ||
| import cats.mtl.ApplicativeAsk | ||
| import com.google.cloud.compute.v1.{Disk, DiskClient, DisksResizeRequest, Operation, ProjectZoneDiskName, ProjectZoneName} | ||
| import fs2.Stream | ||
| import io.chrisdavenport.log4cats.StructuredLogger | ||
| import org.broadinstitute.dsde.workbench.RetryConfig | ||
| import org.broadinstitute.dsde.workbench.model.TraceId | ||
| import org.broadinstitute.dsde.workbench.model.google.GoogleProject | ||
|
|
||
| import scala.collection.JavaConverters._ | ||
|
|
||
| private[google2] class GoogleDiskInterpreter[F[_]: Async: StructuredLogger: Timer: ContextShift] ( | ||
| diskClient: DiskClient, | ||
| retryConfig: RetryConfig, | ||
| blocker: Blocker, | ||
| blockerBound: Semaphore[F] | ||
| ) extends GoogleDiskService[F] { | ||
|
|
||
| override def createDisk(project: GoogleProject, zone: ZoneName, disk: Disk) | ||
| (implicit ev: ApplicativeAsk[F, TraceId] | ||
| ): F[Operation] = { | ||
| val projectZone = ProjectZoneName.of(project.value, zone.value) | ||
| retryF( | ||
| Async[F].delay(diskClient.insertDisk(projectZone, disk)), | ||
| s"com.google.cloud.compute.v1DiskClient.insertDisk(${projectZone.toString}, ${disk.getName})" | ||
| ).compile.lastOrError | ||
| } | ||
|
|
||
| override def deleteDisk(project: GoogleProject, zone: ZoneName, diskName: DiskName)( | ||
| implicit ev: ApplicativeAsk[F, TraceId] | ||
| ): F[Operation] = { | ||
| val projectZoneDiskName = ProjectZoneDiskName.of(diskName.value, project.value, zone.value) | ||
| retryF( | ||
| Async[F].delay(diskClient.deleteDisk(projectZoneDiskName)), | ||
| s"com.google.cloud.compute.v1.DiskClient.deleteDisk(${projectZoneDiskName.toString})" | ||
| ).compile.lastOrError | ||
| } | ||
|
|
||
| override def listDisks(project: GoogleProject, zone: ZoneName)( | ||
| implicit ev: ApplicativeAsk[F, TraceId] | ||
| ): Stream[F, Disk] = { | ||
| val projectZone = ProjectZoneName.of(project.value, zone.value) | ||
| for { | ||
| pagedResults <- retryF( | ||
| Async[F].delay(diskClient.listDisks(projectZone)), | ||
| s"com.google.cloud.compute.v1.DiskClient.listDisks(${projectZone.toString})" | ||
| ) | ||
|
|
||
| res <- Stream.fromIterator[F](pagedResults.iterateAll().iterator().asScala) | ||
| } yield res | ||
| } | ||
|
|
||
| override def resizeDisk(project: GoogleProject, zone: ZoneName, diskName: DiskName, newSizeGb: Int)( | ||
| implicit ev: ApplicativeAsk[F, TraceId] | ||
| ): F[Operation] = { | ||
| val projectZoneDiskName = ProjectZoneDiskName.of(diskName.value, project.value, zone.value) | ||
| val request = DisksResizeRequest.newBuilder().setSizeGb(newSizeGb.toString).build() | ||
| retryF( | ||
| Async[F].delay(diskClient.resizeDisk(projectZoneDiskName, request)), | ||
| s"com.google.cloud.compute.v1.DiskClient.resizeDisk(${projectZoneDiskName.toString}, $newSizeGb)" | ||
| ).compile.lastOrError | ||
| } | ||
|
|
||
| private def retryF[A](fa: F[A], loggingMsg: String)(implicit ev: ApplicativeAsk[F, TraceId]): Stream[F, A] = | ||
| tracedRetryGoogleF(retryConfig)(blockerBound.withPermit(blocker.blockOn(fa)), loggingMsg) | ||
|
|
||
| } |
76 changes: 76 additions & 0 deletions
76
google2/src/main/scala/org/broadinstitute/dsde/workbench/google2/GoogleDiskService.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,76 @@ | ||
| package org.broadinstitute.dsde.workbench.google2 | ||
|
|
||
| import cats.effect.concurrent.Semaphore | ||
| import cats.effect.{Async, Blocker, ContextShift, Resource, Timer} | ||
| import cats.mtl.ApplicativeAsk | ||
| import com.google.api.gax.core.FixedCredentialsProvider | ||
| import com.google.api.services.compute.ComputeScopes | ||
| import com.google.auth.oauth2.GoogleCredentials | ||
| import org.broadinstitute.dsde.workbench.model.google.GoogleProject | ||
| import com.google.cloud.compute.v1._ | ||
| import fs2.Stream | ||
| import io.chrisdavenport.log4cats.StructuredLogger | ||
| import org.broadinstitute.dsde.workbench.RetryConfig | ||
| import org.broadinstitute.dsde.workbench.google2.util.RetryPredicates | ||
| import org.broadinstitute.dsde.workbench.model.TraceId | ||
|
|
||
| import scala.collection.JavaConverters._ | ||
|
|
||
| /** | ||
| * Algebra for Google Disk access. | ||
| */ | ||
| trait GoogleDiskService[F[_]] { | ||
| def createDisk(project: GoogleProject, zone: ZoneName, disk: Disk)( | ||
| implicit ev: ApplicativeAsk[F, TraceId] | ||
| ): F[Operation] | ||
|
|
||
| def deleteDisk(project: GoogleProject, zone: ZoneName, diskName: DiskName)( | ||
| implicit ev: ApplicativeAsk[F, TraceId] | ||
| ): F[Operation] | ||
|
|
||
| def listDisks(project: GoogleProject, zone: ZoneName)( | ||
| implicit ev: ApplicativeAsk[F, TraceId] | ||
| ): Stream[F, Disk] | ||
|
|
||
| def resizeDisk(project: GoogleProject, zone: ZoneName, diskName: DiskName, newSizeGb: Int)( | ||
| implicit ev: ApplicativeAsk[F, TraceId] | ||
| ): F[Operation] | ||
| } | ||
|
|
||
| object GoogleDiskService { | ||
| def resource[F[_]: StructuredLogger: Async: Timer: ContextShift]( | ||
| pathToCredential: String, | ||
| blocker: Blocker, | ||
| blockerBound: Semaphore[F], | ||
| retryConfig: RetryConfig = RetryPredicates.standardRetryConfig): Resource[F, GoogleDiskService[F]] = | ||
| for { | ||
| credential <- credentialResource(pathToCredential) | ||
| scopedCredential = credential.createScoped(Seq(ComputeScopes.COMPUTE).asJava) | ||
| interpreter <- fromCredential(scopedCredential, blocker, blockerBound, retryConfig) | ||
| } yield interpreter | ||
|
|
||
| private def fromCredential[F[_]: StructuredLogger: Async: Timer: ContextShift]( | ||
| googleCredentials: GoogleCredentials, | ||
| blocker: Blocker, | ||
| blockerBound: Semaphore[F], | ||
| retryConfig: RetryConfig | ||
| ): Resource[F, GoogleDiskService[F]] = { | ||
| val credentialsProvider = FixedCredentialsProvider.create(googleCredentials) | ||
|
|
||
| val diskSettings = DiskSettings | ||
| .newBuilder() | ||
| .setCredentialsProvider(credentialsProvider) | ||
| .build() | ||
|
|
||
| for { | ||
| diskClient <- backgroundResourceF(DiskClient.create(diskSettings)) | ||
| } yield new GoogleDiskInterpreter[F]( | ||
| diskClient, | ||
| retryConfig, | ||
| blocker, | ||
| blockerBound | ||
| ) | ||
| } | ||
| } | ||
|
|
||
| final case class DiskName(value: String) extends AnyVal |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters