Skip to content

Commit

Permalink
Support Fargate service for head job in AWS Batch compute env (#336)
Browse files Browse the repository at this point in the history 
* update CE addition flags, support fargate

* update reflection files

* fix tests, new forge options in request

* restore fusionv1 flag, handle with deprecation error + UTs

* remove fusionv1 from base forge config creation and AWS batch tests

* add fusion v2 and wave flags to manual mode

* validate fusion v2 requirements in CLI side

* fix tests
  • Loading branch information
@JaimeSeqLabs
JaimeSeqLabs committed Oct 13, 2023
1 parent 654deaa commit d1b62c9
Show file tree
Hide file tree
Showing 5 changed files with 95 additions and 27 deletions.
37 changes: 25 additions & 12 deletions conf/reflect-config.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,9 @@
"name":"com.sun.crypto.provider.TlsPrfGenerator$V12",
"methods":[{"name":"<init>","parameterTypes":[] }]
},
{
"name":"com.sun.jndi.url.java.javaURLContextFactory"
},
{
"name":"groovy.lang.Closure"
},
Expand Down Expand Up @@ -1551,7 +1554,8 @@
"name":"io.seqera.tower.cli.exceptions.ApiExceptionMessage",
"allDeclaredFields":true,
"allDeclaredMethods":true,
"allDeclaredConstructors":true
"allDeclaredConstructors":true,
"methods":[{"name":"<init>","parameterTypes":[] }, {"name":"setMessage","parameterTypes":["java.lang.String"] }]
},
{
"name":"io.seqera.tower.cli.responses.ComputeEnvAdded",
Expand Down Expand Up @@ -2144,7 +2148,8 @@
"name":"io.seqera.tower.model.AwsSecurityKeys",
"allDeclaredFields":true,
"allDeclaredMethods":true,
"allDeclaredConstructors":true
"allDeclaredConstructors":true,
"methods":[{"name":"<init>","parameterTypes":[] }, {"name":"setAccessKey","parameterTypes":["java.lang.String"] }, {"name":"setAssumeRoleArn","parameterTypes":["java.lang.String"] }, {"name":"setSecretKey","parameterTypes":["java.lang.String"] }]
},
{
"name":"io.seqera.tower.model.AzBatchConfig",
Expand Down Expand Up @@ -2190,12 +2195,14 @@
"name":"io.seqera.tower.model.ComputeEnv",
"allDeclaredFields":true,
"allDeclaredMethods":true,
"allDeclaredConstructors":true
"allDeclaredConstructors":true,
"methods":[{"name":"getConfig","parameterTypes":[] }, {"name":"getCredentialsId","parameterTypes":[] }, {"name":"getDateCreated","parameterTypes":[] }, {"name":"getDeleted","parameterTypes":[] }, {"name":"getDescription","parameterTypes":[] }, {"name":"getId","parameterTypes":[] }, {"name":"getLastUpdated","parameterTypes":[] }, {"name":"getLastUsed","parameterTypes":[] }, {"name":"getMessage","parameterTypes":[] }, {"name":"getName","parameterTypes":[] }, {"name":"getOrgId","parameterTypes":[] }, {"name":"getPlatform","parameterTypes":[] }, {"name":"getPrimary","parameterTypes":[] }, {"name":"getStatus","parameterTypes":[] }, {"name":"getWorkspaceId","parameterTypes":[] }]
},
{
"name":"io.seqera.tower.model.ComputeEnv$PlatformEnum",
"allDeclaredFields":true,
"allDeclaredMethods":true
"allDeclaredMethods":true,
"methods":[{"name":"getValue","parameterTypes":[] }]
},
{
"name":"io.seqera.tower.model.ComputeEnvDbDto",
Expand Down Expand Up @@ -2257,13 +2264,15 @@
"name":"io.seqera.tower.model.CreateComputeEnvRequest",
"allDeclaredFields":true,
"allDeclaredMethods":true,
"allDeclaredConstructors":true
"allDeclaredConstructors":true,
"methods":[{"name":"getComputeEnv","parameterTypes":[] }, {"name":"getLabelIds","parameterTypes":[] }]
},
{
"name":"io.seqera.tower.model.CreateComputeEnvResponse",
"allDeclaredFields":true,
"allDeclaredMethods":true,
"allDeclaredConstructors":true
"allDeclaredConstructors":true,
"methods":[{"name":"<init>","parameterTypes":[] }, {"name":"setComputeEnvId","parameterTypes":["java.lang.String"] }]
},
{
"name":"io.seqera.tower.model.CreateCredentialsRequest",
Expand Down Expand Up @@ -2377,12 +2386,14 @@
"name":"io.seqera.tower.model.Credentials",
"allDeclaredFields":true,
"allDeclaredMethods":true,
"allDeclaredConstructors":true
"allDeclaredConstructors":true,
"methods":[{"name":"<init>","parameterTypes":[] }, {"name":"setBaseUrl","parameterTypes":["java.lang.String"] }, {"name":"setCategory","parameterTypes":["java.lang.String"] }, {"name":"setDescription","parameterTypes":["java.lang.String"] }, {"name":"setId","parameterTypes":["java.lang.String"] }, {"name":"setKeys","parameterTypes":["io.seqera.tower.model.SecurityKeys"] }, {"name":"setName","parameterTypes":["java.lang.String"] }, {"name":"setProvider","parameterTypes":["io.seqera.tower.model.Credentials$ProviderEnum"] }]
},
{
"name":"io.seqera.tower.model.Credentials$ProviderEnum",
"allDeclaredFields":true,
"allDeclaredMethods":true
"allDeclaredMethods":true,
"methods":[{"name":"fromValue","parameterTypes":["java.lang.String"] }]
},
{
"name":"io.seqera.tower.model.Dataset",
Expand Down Expand Up @@ -2649,7 +2660,8 @@
"name":"io.seqera.tower.model.ListCredentialsResponse",
"allDeclaredFields":true,
"allDeclaredMethods":true,
"allDeclaredConstructors":true
"allDeclaredConstructors":true,
"methods":[{"name":"<init>","parameterTypes":[] }, {"name":"setCredentials","parameterTypes":["java.util.List"] }]
},
{
"name":"io.seqera.tower.model.ListDatasetVersionsResponse",
Expand Down Expand Up @@ -2766,7 +2778,7 @@
"allDeclaredFields":true,
"allDeclaredMethods":true,
"allDeclaredConstructors":true,
"methods":[{"name":"<init>","parameterTypes":[] }, {"name":"setOrgId","parameterTypes":["java.lang.Long"] }, {"name":"setOrgLogoUrl","parameterTypes":["java.lang.String"] }, {"name":"setOrgName","parameterTypes":["java.lang.String"] }, {"name":"setWorkspaceFullName","parameterTypes":["java.lang.String"] }, {"name":"setWorkspaceId","parameterTypes":["java.lang.Long"] }, {"name":"setWorkspaceName","parameterTypes":["java.lang.String"] }]
"methods":[{"name":"<init>","parameterTypes":[] }, {"name":"setOrgId","parameterTypes":["java.lang.Long"] }, {"name":"setOrgLogoUrl","parameterTypes":["java.lang.String"] }, {"name":"setOrgName","parameterTypes":["java.lang.String"] }, {"name":"setVisibility","parameterTypes":["io.seqera.tower.model.Visibility"] }, {"name":"setWorkspaceFullName","parameterTypes":["java.lang.String"] }, {"name":"setWorkspaceId","parameterTypes":["java.lang.Long"] }, {"name":"setWorkspaceName","parameterTypes":["java.lang.String"] }]
},
{
"name":"io.seqera.tower.model.OrgRole",
Expand Down Expand Up @@ -3011,7 +3023,7 @@
"allDeclaredFields":true,
"allDeclaredMethods":true,
"allDeclaredConstructors":true,
"methods":[{"name":"<init>","parameterTypes":[] }, {"name":"setAvatar","parameterTypes":["java.lang.String"] }, {"name":"setDescription","parameterTypes":["java.lang.String"] }, {"name":"setFirstName","parameterTypes":["java.lang.String"] }, {"name":"setId_JsonNullable","parameterTypes":["org.openapitools.jackson.nullable.JsonNullable"] }, {"name":"setLastName","parameterTypes":["java.lang.String"] }, {"name":"setNotification","parameterTypes":["java.lang.Boolean"] }, {"name":"setOrganization","parameterTypes":["java.lang.String"] }, {"name":"setUserName","parameterTypes":["java.lang.String"] }]
"methods":[{"name":"<init>","parameterTypes":[] }, {"name":"setAvatar","parameterTypes":["java.lang.String"] }, {"name":"setAvatarId","parameterTypes":["java.lang.String"] }, {"name":"setDateCreated","parameterTypes":["java.time.OffsetDateTime"] }, {"name":"setDeleted","parameterTypes":["java.lang.Boolean"] }, {"name":"setDescription","parameterTypes":["java.lang.String"] }, {"name":"setEmail","parameterTypes":["java.lang.String"] }, {"name":"setFirstName","parameterTypes":["java.lang.String"] }, {"name":"setId","parameterTypes":["java.lang.Long"] }, {"name":"setId_JsonNullable","parameterTypes":["org.openapitools.jackson.nullable.JsonNullable"] }, {"name":"setLastAccess","parameterTypes":["java.time.OffsetDateTime"] }, {"name":"setLastName","parameterTypes":["java.lang.String"] }, {"name":"setLastUpdated","parameterTypes":["java.time.OffsetDateTime"] }, {"name":"setMarketingConsent","parameterTypes":["java.lang.Boolean"] }, {"name":"setNotification","parameterTypes":["java.lang.Boolean"] }, {"name":"setOrganization","parameterTypes":["java.lang.String"] }, {"name":"setTermsOfUseConsent","parameterTypes":["java.lang.Boolean"] }, {"name":"setUserName","parameterTypes":["java.lang.String"] }]
},
{
"name":"io.seqera.tower.model.UserOptions",
Expand All @@ -3023,7 +3035,8 @@
{
"name":"io.seqera.tower.model.Visibility",
"allDeclaredFields":true,
"allDeclaredMethods":true
"allDeclaredMethods":true,
"methods":[{"name":"fromValue","parameterTypes":["java.lang.String"] }]
},
{
"name":"io.seqera.tower.model.WfManifest",
Expand Down
32 changes: 28 additions & 4 deletions src/main/java/io/seqera/tower/cli/commands/computeenvs/platforms/AwsBatchForgePlatform.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
package io.seqera.tower.cli.commands.computeenvs.platforms;

import io.seqera.tower.ApiException;
import io.seqera.tower.cli.exceptions.TowerException;
import io.seqera.tower.model.AwsBatchConfig;
import io.seqera.tower.model.ComputeEnv.PlatformEnum;
import io.seqera.tower.model.ForgeConfig;
Expand All @@ -37,9 +38,18 @@ public class AwsBatchForgePlatform extends AbstractPlatform<AwsBatchConfig> {
@Option(names = {"--no-ebs-auto-scale"}, description = "Disable the provisioning of EBS auto-expandable disk.")
public boolean noEbsAutoScale;

@Option(names = {"--fusion"}, description = "With Fusion enabled, S3 buckets specified in the Pipeline work directory and Allowed S3 Buckets fields will be accessible in the compute nodes storage using the file path /fusion/s3/BUCKET_NAME.")
@Option(names = {"--fusion"}, description = "DEPRECATED - Use '--fusion-v2' instead.")
public boolean fusion;

@Option(names = {"--fusion-v2"}, description = "With Fusion v2 enabled, S3 buckets specified in the Pipeline work directory and Allowed S3 Buckets fields will be accessible in the compute nodes storage (requires Wave containers service).")
public boolean fusionV2;

@Option(names = {"--wave"}, description = "Allow access to private container repositories and the provisioning of containers in your Nextflow pipelines via the Wave containers service.")
public boolean wave;

@Option(names = {"--fargate"}, description = "Run the Nextflow head job using the Fargate container service (requires Fusion v2 and Spot provisioning model).")
public boolean fargate;

@Option(names = {"--gpu"}, description = "Deploys GPU enabled Ec2 instances.")
public boolean gpu;

Expand Down Expand Up @@ -81,6 +91,8 @@ public AwsBatchConfig computeConfig() throws ApiException, IOException {
.postRunScript(postRunScriptString())
.environment(environmentVariables())
.region(region)
.fusion2Enabled(isFusionV2Enabled())
.waveEnabled(wave)

// Forge
.forge(buildForge())
Expand All @@ -94,12 +106,23 @@ public AwsBatchConfig computeConfig() throws ApiException, IOException {
.headJobRole(adv().headJobRole);
}

private ForgeConfig buildForge() {
private Boolean isFusionV2Enabled() throws TowerException {
// TODO: delete this validation once wave is no longer a requirement for Fusion V2
if (fusionV2 && !wave) throw new TowerException("Fusion v2 requires Wave service");
return fusionV2;
}

private ForgeConfig buildForge() throws TowerException {

// TODO: delete this once fusion v1 is completely removed
if (fusion) {
throw new TowerException("Fusion v1 is deprecated, please use '--fusion-v2' instead");
}

ForgeConfig forge = new ForgeConfig()
.type(provisioningModel)
.maxCpus(maxCpus)
.ebsAutoScale(!noEbsAutoScale)
.fusionEnabled(fusion)
.gpuEnabled(gpu)
.allowBuckets(allowBuckets)
.disposeOnDeletion(!preserveResources)
Expand All @@ -112,7 +135,8 @@ private ForgeConfig buildForge() {
.ec2KeyPair(adv().keyPair)
.minCpus(adv().minCpus == null ? 0 : adv().minCpus)
.ebsBlockSize(adv().ebsBlockSize)
.bidPercentage(adv().bidPercentage);
.bidPercentage(adv().bidPercentage)
.fargateHeadEnabled(fargate);


if (efs != null) {
Expand Down
8 changes: 8 additions & 0 deletions src/main/java/io/seqera/tower/cli/commands/computeenvs/platforms/AwsBatchManualPlatform.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,12 @@ public class AwsBatchManualPlatform extends AbstractPlatform<AwsBatchConfig> {
@Option(names = {"--compute-queue"}, description = "The default Batch queue to which Nextflow will submit jobs. This can be overwritten via Nextflow config.", required = true)
public String computeQueue;

@Option(names = {"--fusion-v2"}, description = "With Fusion v2 enabled, S3 buckets specified in the Pipeline work directory and Allowed S3 Buckets fields will be accessible in the compute nodes storage (requires Wave containers service).")
public boolean fusionV2;

@Option(names = {"--wave"}, description = "Allow access to private container repositories and the provisioning of containers in your Nextflow pipelines via the Wave containers service.")
public boolean wave;

@ArgGroup(heading = "%nAdvanced options:%n", validate = false)
public AdvancedOptions adv;

Expand All @@ -52,6 +58,8 @@ public AwsBatchConfig computeConfig() throws IOException {
.preRunScript(preRunScriptString())
.postRunScript(postRunScriptString())
.environment(environmentVariables())
.fusion2Enabled(fusionV2)
.waveEnabled(wave)
.region(region)

// Queues
Expand Down

0 comments on commit d1b62c9

Please sign in to comment.