Skip to content

Commit

Permalink
Allow clients to specify full additional file upload descriptors.
Browse files Browse the repository at this point in the history
  • Loading branch information
@baconmania
baconmania committed Jan 8, 2018
1 parent c96cb6e commit f7766f4
Show file tree
Hide file tree
Showing 9 changed files with 49 additions and 43 deletions.
16 changes: 8 additions & 8 deletions SingularityBase/src/main/java/com/hubspot/singularity/SingularityPendingRequest.java
View file
Expand Up @@ -28,19 +28,19 @@ public enum PendingType {
private final Optional<String> message;
private final Optional<String> actionId;
private final Optional<Resources> resources;
private final Optional<String> s3UploaderKeyPatternOverride;
private final List<SingularityS3UploaderFile> s3UploaderAdditionalFiles;
private final Optional<String> runAsUserOverride;
private final Map<String, String> envOverrides;
private final List<SingularityMesosArtifact> extraArtifacts;
private final Optional<Long> runAt;

public SingularityPendingRequest(String requestId, String deployId, long timestamp, Optional<String> user, PendingType pendingType, Optional<Boolean> skipHealthchecks, Optional<String> message) {
this(requestId, deployId, timestamp, user, pendingType, Optional.<List<String>> absent(), Optional.<String> absent(), skipHealthchecks, message, Optional.<String> absent(), Optional.<Resources>absent(), Optional.<String>absent(), Optional.absent(), null, null, Optional.<Long> absent());
this(requestId, deployId, timestamp, user, pendingType, Optional.<List<String>> absent(), Optional.<String> absent(), skipHealthchecks, message, Optional.<String> absent(), Optional.<Resources>absent(), Collections.emptyList(), Optional.absent(), null, null, Optional.<Long> absent());
}

public SingularityPendingRequest(String requestId, String deployId, long timestamp, Optional<String> user, PendingType pendingType, Optional<List<String>> cmdLineArgsList,
Optional<String> runId, Optional<Boolean> skipHealthchecks, Optional<String> message, Optional<String> actionId) {
this(requestId, deployId, timestamp, user, pendingType, cmdLineArgsList, runId, skipHealthchecks, message, actionId, Optional.<Resources>absent(), Optional.<String>absent(), Optional.absent(), null, null, Optional.<Long>absent());
this(requestId, deployId, timestamp, user, pendingType, cmdLineArgsList, runId, skipHealthchecks, message, actionId, Optional.<Resources>absent(), Collections.emptyList(), Optional.absent(), null, null, Optional.<Long>absent());
}

@JsonCreator
Expand All @@ -55,7 +55,7 @@ public SingularityPendingRequest(@JsonProperty("requestId") String requestId,
@JsonProperty("message") Optional<String> message,
@JsonProperty("actionId") Optional<String> actionId,
@JsonProperty("resources") Optional<Resources> resources,
@JsonProperty("s3UploaderKeyPatternOverride") Optional<String> s3UploaderKeyPatternOverride,
@JsonProperty("s3UploaderAdditionalFiles") List<SingularityS3UploaderFile> s3UploaderAdditionalFiles,
@JsonProperty("runAsUserOverride") Optional<String> runAsUserOverride,
@JsonProperty("envOverrides") Map<String, String> envOverrides,
@JsonProperty("extraArtifacts") List<SingularityMesosArtifact> extraArtifacts,
Expand All @@ -71,7 +71,7 @@ public SingularityPendingRequest(@JsonProperty("requestId") String requestId,
this.message = message;
this.actionId = actionId;
this.resources = resources;
this.s3UploaderKeyPatternOverride = s3UploaderKeyPatternOverride;
this.s3UploaderAdditionalFiles = s3UploaderAdditionalFiles;
this.runAsUserOverride = runAsUserOverride;

if (Objects.nonNull(envOverrides)) {
Expand Down Expand Up @@ -133,8 +133,8 @@ public Optional<Resources> getResources() {
return resources;
}

public Optional<String> getS3UploaderKeyPatternOverride() {
return s3UploaderKeyPatternOverride;
public List<SingularityS3UploaderFile> getS3UploaderAdditionalFiles() {
return s3UploaderAdditionalFiles;
}

public Map<String, String> getEnvOverrides() {
Expand Down Expand Up @@ -168,7 +168,7 @@ public String toString() {
", message=" + message +
", actionId=" + actionId +
", resources=" + resources +
", s3UploaderKeyPatternOverride=" + s3UploaderKeyPatternOverride +
", s3UploaderAdditionalFiles=" + s3UploaderAdditionalFiles +
", runAsUserOverride=" + runAsUserOverride +
", envOverrides=" + envOverrides +
", extraArtifacts=" + extraArtifacts +
Expand Down
10 changes: 5 additions & 5 deletions SingularityBase/src/main/java/com/hubspot/singularity/SingularityPendingRequestBuilder.java
View file
Expand Up @@ -22,7 +22,7 @@ public class SingularityPendingRequestBuilder {
private Optional<String> message;
private Optional<String> actionId;
private Optional<Resources> resources;
private Optional<String> s3UploaderKeyPatternOverride;
private List<SingularityS3UploaderFile> s3UploaderAdditionalFiles;
private Optional<String> runAsUserOverride;
private Map<String, String> envOverrides;
private List<SingularityMesosArtifact> extraArtifacts;
Expand Down Expand Up @@ -127,8 +127,8 @@ public SingularityPendingRequestBuilder setResources(Optional<Resources> resourc
return this;
}

public SingularityPendingRequestBuilder setS3UploaderKeyPatternOverride(Optional<String> s3UploaderKeyPatternOverride) {
this.s3UploaderKeyPatternOverride = s3UploaderKeyPatternOverride;
public SingularityPendingRequestBuilder setS3UploaderAdditionalFiles(List<SingularityS3UploaderFile> s3UploaderAdditionalFiles) {
this.s3UploaderAdditionalFiles = s3UploaderAdditionalFiles;
return this;
}

Expand Down Expand Up @@ -170,7 +170,7 @@ public SingularityPendingRequest build() {
message,
actionId,
resources,
s3UploaderKeyPatternOverride,
s3UploaderAdditionalFiles,
runAsUserOverride,
envOverrides,
extraArtifacts,
Expand All @@ -192,7 +192,7 @@ public String toString() {
", message" + message +
", actionId" + actionId +
", resources" + resources +
", s3UploaderKeyPatternOverride=" + s3UploaderKeyPatternOverride +
", s3UploaderAdditionalFiles=" + s3UploaderAdditionalFiles +
", runAsUserOverride" + runAsUserOverride +
", envOverrides" + envOverrides +
", extraArtifacts" + extraArtifacts +
Expand Down
12 changes: 6 additions & 6 deletions SingularityBase/src/main/java/com/hubspot/singularity/SingularityPendingTask.java
View file
Expand Up @@ -23,7 +23,7 @@ public class SingularityPendingTask {
private final Optional<Boolean> skipHealthchecks;
private final Optional<String> message;
private final Optional<Resources> resources;
private final Optional<String> s3UploaderKeyPatternOverride;
private final List<SingularityS3UploaderFile> s3UploaderAdditionalFiles;
private final Optional<String> runAsUserOverride;
private final Map<String, String> envOverrides;
private final List<SingularityMesosArtifact> extraArtifacts;
Expand Down Expand Up @@ -59,7 +59,7 @@ public SingularityPendingTask(@JsonProperty("pendingTaskId") SingularityPendingT
@JsonProperty("skipHealthchecks") Optional<Boolean> skipHealthchecks,
@JsonProperty("message") Optional<String> message,
@JsonProperty("resources") Optional<Resources> resources,
@JsonProperty("s3UploaderKeyPatternOverride") Optional<String> s3UploaderKeyPatternOverride,
@JsonProperty("s3UploaderAdditionalFiles") List<SingularityS3UploaderFile> s3UploaderAdditionalFiles,
@JsonProperty("runAsUserOverride") Optional<String> runAsUserOverride,
@JsonProperty("envOverrides") Map<String, String> envOverrides,
@JsonProperty("extraArtifacts") List<SingularityMesosArtifact> extraArtifacts,
Expand All @@ -71,7 +71,7 @@ public SingularityPendingTask(@JsonProperty("pendingTaskId") SingularityPendingT
this.runId = runId;
this.skipHealthchecks = skipHealthchecks;
this.resources = resources;
this.s3UploaderKeyPatternOverride = s3UploaderKeyPatternOverride;
this.s3UploaderAdditionalFiles = s3UploaderAdditionalFiles;
this.runAsUserOverride = runAsUserOverride;

if (Objects.nonNull(envOverrides)) {
Expand Down Expand Up @@ -134,8 +134,8 @@ public Optional<Resources> getResources() {
return resources;
}

public Optional<String> getS3UploaderKeyPatternOverride() {
return s3UploaderKeyPatternOverride;
public List<SingularityS3UploaderFile> getS3UploaderAdditionalFiles() {
return s3UploaderAdditionalFiles;
}

public Optional<String> getRunAsUserOverride() {
Expand All @@ -162,7 +162,7 @@ public String toString() {
", skipHealthchecks=" + skipHealthchecks +
", message=" + message +
", resources=" + resources +
", s3UploaderKeyPatternOverride=" + s3UploaderKeyPatternOverride +
", s3UploaderAdditionalFiles=" + s3UploaderAdditionalFiles +
", runAsUserOverride=" + runAsUserOverride +
", envOverrides=" + envOverrides +
", extraArtifacts" + extraArtifacts +
Expand Down
12 changes: 6 additions & 6 deletions SingularityBase/src/main/java/com/hubspot/singularity/SingularityPendingTaskBuilder.java
View file
Expand Up @@ -17,7 +17,7 @@ public class SingularityPendingTaskBuilder {
private Optional<Boolean> skipHealthchecks;
private Optional<String> message;
private Optional<Resources> resources;
private Optional<String> s3UploaderKeyPatternOverride;
private List<SingularityS3UploaderFile> s3UploaderAdditionalFiles;
private Optional<String> runAsUserOverride;
private Map<String, String> envOverrides;
private List<SingularityMesosArtifact> extraArtifacts;
Expand All @@ -31,7 +31,7 @@ public SingularityPendingTaskBuilder() {
this.skipHealthchecks = Optional.absent();
this.message = Optional.absent();
this.resources = Optional.absent();
this.s3UploaderKeyPatternOverride = Optional.absent();
this.s3UploaderAdditionalFiles = Collections.emptyList();
this.runAsUserOverride = Optional.absent();
this.envOverrides = Collections.emptyMap();
this.extraArtifacts = Collections.emptyList();
Expand Down Expand Up @@ -84,8 +84,8 @@ public SingularityPendingTaskBuilder setResources(Resources resources) {
return this;
}

public SingularityPendingTaskBuilder setS3UploaderKeyPatternOverride(Optional<String> s3UploaderKeyPatternOverride) {
this.s3UploaderKeyPatternOverride = s3UploaderKeyPatternOverride;
public SingularityPendingTaskBuilder setS3UploaderAdditionalFiles(List<SingularityS3UploaderFile> s3UploaderAdditionalFiles) {
this.s3UploaderAdditionalFiles = s3UploaderAdditionalFiles;
return this;
}

Expand All @@ -111,7 +111,7 @@ public SingularityPendingTaskBuilder setActionId(String actionId) {

public SingularityPendingTask build() {
return new SingularityPendingTask(
pendingTaskId, cmdLineArgsList, user, runId, skipHealthchecks, message, resources, s3UploaderKeyPatternOverride, runAsUserOverride, envOverrides, extraArtifacts, actionId
pendingTaskId, cmdLineArgsList, user, runId, skipHealthchecks, message, resources, s3UploaderAdditionalFiles, runAsUserOverride, envOverrides, extraArtifacts, actionId
);
}

Expand All @@ -125,7 +125,7 @@ public String toString() {
", skipHealthchecks=" + skipHealthchecks +
", message=" + message +
", resources=" + resources +
", s3UploaderKeyPatternOverride=" + s3UploaderKeyPatternOverride +
", s3UploaderAdditionalFiles=" + s3UploaderAdditionalFiles +
", runAsUserOverride=" + runAsUserOverride +
", envOverrides=" + envOverrides +
", extraArtifacts=" + extraArtifacts +
Expand Down
10 changes: 5 additions & 5 deletions SingularityBase/src/main/java/com/hubspot/singularity/SingularityRunNowRequestBuilder.java
View file
Expand Up @@ -15,7 +15,7 @@ public class SingularityRunNowRequestBuilder {
private Optional<List<String>> commandLineArgs;
private Optional<Boolean> skipHealthchecks;
private Optional<Resources> resources;
private Optional<String> s3UploaderKeyPatternOverride;
private List<SingularityS3UploaderFile> s3UploaderAdditionalFiles;
private Optional<String> runAsUserOverride;
private Map<String, String> envOverrides;
private List<SingularityMesosArtifact> extraArtifacts;
Expand Down Expand Up @@ -59,8 +59,8 @@ public SingularityRunNowRequestBuilder setResources(Resources resources) {
return this;
}

public SingularityRunNowRequestBuilder setS3UploaderKeyPatternOverride(Optional<String> s3UploaderKeyPatternOverride) {
this.s3UploaderKeyPatternOverride = s3UploaderKeyPatternOverride;
public SingularityRunNowRequestBuilder setS3UploaderAdditionalFiles(List<SingularityS3UploaderFile> s3UploaderAdditionalFiles) {
this.s3UploaderAdditionalFiles = s3UploaderAdditionalFiles;
return this;
}

Expand All @@ -86,7 +86,7 @@ public SingularityRunNowRequestBuilder setRunAt(Long runAt) {

public SingularityRunNowRequest build() {
return new SingularityRunNowRequest(
message, skipHealthchecks, runId, commandLineArgs, resources, s3UploaderKeyPatternOverride, runAsUserOverride, envOverrides, extraArtifacts, runAt);
message, skipHealthchecks, runId, commandLineArgs, resources, s3UploaderAdditionalFiles, runAsUserOverride, envOverrides, extraArtifacts, runAt);
}

@Override
Expand All @@ -97,7 +97,7 @@ public String toString() {
", commandLineArgs=" + commandLineArgs +
", skipHealthchecks=" + skipHealthchecks +
", resources=" + resources +
", s3UploaderKeyPatternOverride=" + s3UploaderKeyPatternOverride +
", s3UploaderAdditionalFiles=" + s3UploaderAdditionalFiles +
", runNowUserOverride=" + runAsUserOverride +
", envOverrides=" + envOverrides +
", extraArtifacts=" + extraArtifacts +
Expand Down
17 changes: 9 additions & 8 deletions SingularityBase/src/main/java/com/hubspot/singularity/api/SingularityRunNowRequest.java
View file
Expand Up @@ -10,6 +10,7 @@
import com.google.common.base.Optional;
import com.hubspot.mesos.Resources;
import com.hubspot.mesos.SingularityMesosArtifact;
import com.hubspot.singularity.SingularityS3UploaderFile;
import com.wordnik.swagger.annotations.ApiModelProperty;

public class SingularityRunNowRequest {
Expand All @@ -19,7 +20,7 @@ public class SingularityRunNowRequest {
private final Optional<List<String>> commandLineArgs;
private final Optional<Boolean> skipHealthchecks;
private final Optional<Resources> resources;
private final Optional<String> s3UploaderKeyPatternOverride;
private final List<SingularityS3UploaderFile> s3UploaderAdditionalFiles;
private final Optional<String> runAsUserOverride;
private final Map<String, String> envOverrides;
private final List<SingularityMesosArtifact> extraArtifacts;
Expand All @@ -32,7 +33,7 @@ public SingularityRunNowRequest(
Optional<List<String>> commandLineArgs,
Optional<Resources> resources
) {
this(message, skipHealthchecks, runId, commandLineArgs, resources, Optional.absent(), Optional.absent(), null, null, Optional.absent());
this(message, skipHealthchecks, runId, commandLineArgs, resources, Collections.emptyList(), Optional.absent(), null, null, Optional.absent());
}

@JsonCreator
Expand All @@ -41,7 +42,7 @@ public SingularityRunNowRequest(@JsonProperty("message") Optional<String> messag
@JsonProperty("runId") Optional<String> runId,
@JsonProperty("commandLineArgs") Optional<List<String>> commandLineArgs,
@JsonProperty("resources") Optional<Resources> resources,
@JsonProperty("s3UploaderKeyPatternOverride") Optional<String> s3UploaderKeyPatternOverride,
@JsonProperty("s3UploaderAdditionalFiles") List<SingularityS3UploaderFile> s3UploaderAdditionalFiles,
@JsonProperty("runAsUserOverride") Optional<String> runAsUserOverride,
@JsonProperty("envOverrides") Map<String, String> envOverrides,
@JsonProperty("extraArtifacts") List<SingularityMesosArtifact> extraArtifacts,
Expand All @@ -51,7 +52,7 @@ public SingularityRunNowRequest(@JsonProperty("message") Optional<String> messag
this.runId = runId;
this.skipHealthchecks = skipHealthchecks;
this.resources = resources;
this.s3UploaderKeyPatternOverride = s3UploaderKeyPatternOverride;
this.s3UploaderAdditionalFiles = s3UploaderAdditionalFiles;
this.runAsUserOverride = runAsUserOverride;

if (Objects.nonNull(envOverrides)) {
Expand Down Expand Up @@ -94,9 +95,9 @@ public Optional<Resources> getResources() {
return resources;
}

@ApiModelProperty(required=false, value="Override the S3 upload path prefix for logs generated by this task")
public Optional<String> getS3UploaderKeyPatternOverride() {
return s3UploaderKeyPatternOverride;
@ApiModelProperty(required=false, value="Specify additional sandbox files to upload to S3 for this run")
public List<SingularityS3UploaderFile> getS3UploaderAdditionalFiles() {
return s3UploaderAdditionalFiles;
}

@ApiModelProperty(required=false, value="Override the user under which this task's command will be launched.")
Expand Down Expand Up @@ -127,7 +128,7 @@ public String toString() {
", commandLineArgs=" + commandLineArgs +
", skipHealthchecks=" + skipHealthchecks +
", resources=" + resources +
", s3UploaderKeyPatternOverride=" + s3UploaderKeyPatternOverride +
", s3UploaderAdditionalFiles=" + s3UploaderAdditionalFiles +
", runAsUserOverride=" + runAsUserOverride +
", envOverrides=" + envOverrides +
", extraArtifacts=" + extraArtifacts +
Expand Down
4 changes: 2 additions & 2 deletions SingularityService/src/main/java/com/hubspot/singularity/data/SingularityValidator.java
View file
Expand Up @@ -434,7 +434,7 @@ public SingularityPendingRequest checkRunNowRequest(String deployId,
runNowRequest.getMessage(),
Optional.absent(),
runNowRequest.getResources(),
runNowRequest.getS3UploaderKeyPatternOverride(),
runNowRequest.getS3UploaderAdditionalFiles(),
runNowRequest.getRunAsUserOverride(),
runNowRequest.getEnvOverrides(),
runNowRequest.getExtraArtifacts(),
Expand All @@ -451,7 +451,7 @@ private SingularityRunNowRequest fillRunNowRequest(Optional<SingularityRunNowReq
Optional.of(getRunId(request.getRunId())),
request.getCommandLineArgs(),
request.getResources(),
request.getS3UploaderKeyPatternOverride(),
request.getS3UploaderAdditionalFiles(),
request.getRunAsUserOverride(),
request.getEnvOverrides(),
request.getExtraArtifacts(),
Expand Down
9 changes: 7 additions & 2 deletions ...arityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosTaskBuilder.java
View file
Expand Up @@ -404,7 +404,7 @@ private void prepareCustomExecutor(final TaskInfo.Builder bldr, final Singularit
} else {
defaultS3Bucket = configuration.getS3ConfigurationOptional().get().getS3Bucket();
}
s3UploaderKeyPattern = task.getPendingTask().getS3UploaderKeyPatternOverride().or(configuration.getS3ConfigurationOptional().get().getS3KeyFormat());
s3UploaderKeyPattern = configuration.getS3ConfigurationOptional().get().getS3KeyFormat();
}

if (task.getPendingTask().getCmdLineArgsList().isPresent() && !task.getPendingTask().getCmdLineArgsList().get().isEmpty()) {
Expand All @@ -418,7 +418,12 @@ private void prepareCustomExecutor(final TaskInfo.Builder bldr, final Singularit
executorDataBldr.setExtraCmdLineArgs(extraCmdLineArgsBuilder.build());
}

List<SingularityS3UploaderFile> uploaderAdditionalFiles = configuration.getS3ConfigurationOptional().isPresent() ? configuration.getS3ConfigurationOptional().get().getS3UploaderAdditionalFiles() : Collections.<SingularityS3UploaderFile>emptyList();
List<SingularityS3UploaderFile> uploaderAdditionalFiles = new ArrayList<>();
if (configuration.getS3ConfigurationOptional().isPresent()) {
uploaderAdditionalFiles.addAll(configuration.getS3ConfigurationOptional().get().getS3UploaderAdditionalFiles());
}
uploaderAdditionalFiles.addAll(task.getPendingTask().getS3UploaderAdditionalFiles());

Optional<String> maybeS3StorageClass = configuration.getS3ConfigurationOptional().isPresent() ? configuration.getS3ConfigurationOptional().get().getS3StorageClass() : Optional.<String>absent();
Optional<Long> maybeApplyAfterBytes = configuration.getS3ConfigurationOptional().isPresent() ? configuration.getS3ConfigurationOptional().get().getApplyS3StorageClassAfterBytes() : Optional.<Long>absent();

Expand Down
2 changes: 1 addition & 1 deletion SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityScheduler.java
View file
Expand Up @@ -798,7 +798,7 @@ private List<SingularityPendingTask> getScheduledTaskIds(int numMissingInstances
pendingRequest.getSkipHealthchecks(),
pendingRequest.getMessage(),
pendingRequest.getResources(),
pendingRequest.getS3UploaderKeyPatternOverride(),
pendingRequest.getS3UploaderAdditionalFiles(),
pendingRequest.getRunAsUserOverride(),
pendingRequest.getEnvOverrides(),
pendingRequest.getExtraArtifacts(),
Expand Down

0 comments on commit f7766f4

Please sign in to comment.