allow artifact list

SingularityBase/src/main/java/com/hubspot/deploy/Artifact.java

@@ -4,9 +4,18 @@
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonSubTypes;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
 import com.google.common.base.Optional;
 
 @JsonIgnoreProperties(ignoreUnknown = true)
+@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY)
+@JsonSubTypes({
+    @JsonSubTypes.Type(value = EmbeddedArtifact.class, name = "EmbeddedArtifact"),
+    @JsonSubTypes.Type(value = S3Artifact.class, name = "S3Artifact"),
+    @JsonSubTypes.Type(value = S3ArtifactSignature.class, name = "S3ArtifactSignature"),
+    @JsonSubTypes.Type(value = ExternalArtifact.class, name = "ExternalArtifact")
+})
 public abstract class Artifact {
 
   private final String name;

SingularityBase/src/main/java/com/hubspot/deploy/ExecutorData.java

@@ -1,9 +1,11 @@
 package com.hubspot.deploy;
 
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.common.base.Optional;
@@ -144,6 +146,11 @@ public Optional<List<S3ArtifactSignature>> getS3ArtifactSignatures() {
     return s3ArtifactSignatures;
   }
 
+  @JsonIgnore
+  public List<S3ArtifactSignature> getS3ArtifactSignaturesOrEmpty() {
+    return s3ArtifactSignatures.or(Collections.<S3ArtifactSignature> emptyList());
+  }
+
   @ApiModelProperty(required=false, value="Run logrotate this often. Can be HOURLY, DAILY, WEEKLY, MONTHLY")
   public Optional<SingularityExecutorLogrotateFrequency> getLogrotateFrequency() {
     return logrotateFrequency;

SingularityBase/src/main/java/com/hubspot/deploy/ExternalArtifact.java

@@ -12,8 +12,9 @@ public class ExternalArtifact extends RemoteArtifact {
 
   @JsonCreator
   public ExternalArtifact(@JsonProperty("name") String name, @JsonProperty("filename") String filename, @JsonProperty("md5sum") Optional<String> md5sum,
-      @JsonProperty("url") String url, @JsonProperty("filesize") Optional<Long> filesize, @JsonProperty("targetFolderRelativeToTask") Optional<String> targetFolderRelativeToTask) {
-    super(name, filename, md5sum, filesize, targetFolderRelativeToTask);
+      @JsonProperty("url") String url, @JsonProperty("filesize") Optional<Long> filesize, @JsonProperty("targetFolderRelativeToTask") Optional<String> targetFolderRelativeToTask,
+      @JsonProperty("isArtifactList") Optional<Boolean> isArtifactList) {
+    super(name, filename, md5sum, filesize, targetFolderRelativeToTask, isArtifactList);
     this.url = url;
   }
 

SingularityBase/src/main/java/com/hubspot/deploy/RemoteArtifact.java

@@ -2,21 +2,33 @@
 
 import java.util.Objects;
 
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.google.common.base.Optional;
 
 public abstract class RemoteArtifact extends Artifact {
 
   private final Optional<Long> filesize;
+  private final Optional<Boolean> isArtifactList;
 
-  public RemoteArtifact(String name, String filename, Optional<String> md5sum, Optional<Long> filesize, Optional<String> targetFolderRelativeToTask) {
+  public RemoteArtifact(String name, String filename, Optional<String> md5sum, Optional<Long> filesize, Optional<String> targetFolderRelativeToTask, Optional<Boolean> isArtifactList) {
     super(name, filename, md5sum, targetFolderRelativeToTask);
     this.filesize = filesize;
+    this.isArtifactList = isArtifactList;
   }
 
   public Optional<Long> getFilesize() {
     return filesize;
   }
 
+  public Optional<Boolean> getIsArtifactList() {
+    return isArtifactList;
+  }
+
+  @JsonIgnore
+  public boolean isArtifactList() {
+    return isArtifactList.or(Boolean.FALSE).booleanValue();
+  }
+
   @Override
   public boolean equals(Object o) {
     if (this == o) {
@@ -29,18 +41,20 @@ public boolean equals(Object o) {
       return false;
     }
     RemoteArtifact that = (RemoteArtifact) o;
-    return Objects.equals(filesize, that.filesize);
+    return Objects.equals(filesize, that.filesize) &&
+        Objects.equals(isArtifactList, that.isArtifactList);
   }
 
   @Override
   public int hashCode() {
-    return Objects.hash(super.hashCode(), filesize);
+    return Objects.hash(super.hashCode(), filesize, isArtifactList);
   }
 
   @Override
   public String toString() {
     return "RemoteArtifact{" +
         "filesize=" + filesize +
+        "isArtifactList" + isArtifactList +
         "} " + super.toString();
   }
 }

SingularityBase/src/main/java/com/hubspot/deploy/S3Artifact.java

@@ -13,8 +13,9 @@ public class S3Artifact extends RemoteArtifact {
 
   @JsonCreator
   public S3Artifact(@JsonProperty("name") String name, @JsonProperty("filename") String filename, @JsonProperty("md5sum") Optional<String> md5sum, @JsonProperty("filesize") Optional<Long> filesize,
-      @JsonProperty("s3Bucket") String s3Bucket, @JsonProperty("s3ObjectKey") String s3ObjectKey, @JsonProperty("targetFolderRelativeToTask") Optional<String> targetFolderRelativeToTask) {
-    super(name, filename, md5sum, filesize, targetFolderRelativeToTask);
+      @JsonProperty("s3Bucket") String s3Bucket, @JsonProperty("s3ObjectKey") String s3ObjectKey, @JsonProperty("targetFolderRelativeToTask") Optional<String> targetFolderRelativeToTask,
+      @JsonProperty("isArtifactList") Optional<Boolean> isArtifactList) {
+    super(name, filename, md5sum, filesize, targetFolderRelativeToTask, isArtifactList);
 
     this.s3Bucket = s3Bucket;
     this.s3ObjectKey = s3ObjectKey;

SingularityBase/src/main/java/com/hubspot/deploy/S3ArtifactSignature.java

@@ -15,8 +15,9 @@ public class S3ArtifactSignature extends S3Artifact {
   @JsonCreator
   public S3ArtifactSignature(@JsonProperty("name") String name, @JsonProperty("filename") String filename, @JsonProperty("md5sum") Optional<String> md5sum,
       @JsonProperty("filesize") Optional<Long> filesize, @JsonProperty("s3Bucket") String s3Bucket, @JsonProperty("s3ObjectKey") String s3ObjectKey,
-      @JsonProperty("artifactFilename") String artifactFilename, @JsonProperty("targetFolderRelativeToTask") Optional<String> targetFolderRelativeToTask) {
-    super(name, filename, md5sum, filesize, s3Bucket, s3ObjectKey, targetFolderRelativeToTask);
+      @JsonProperty("artifactFilename") String artifactFilename, @JsonProperty("targetFolderRelativeToTask") Optional<String> targetFolderRelativeToTask,
+      @JsonProperty("isArtifactList") Optional<Boolean> isArtifactList) {
+    super(name, filename, md5sum, filesize, s3Bucket, s3ObjectKey, targetFolderRelativeToTask, isArtifactList);
 
     this.artifactFilename = artifactFilename;
   }

SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorTaskBuilder.java

@@ -80,7 +80,7 @@ public SingularityExecutorTask buildTask(String taskId, ExecutorDriver driver, T
 
     jsonObjectFileHelper.writeObject(taskDefinition, executorConfiguration.getTaskDefinitionPath(taskId), log);
 
-    return new SingularityExecutorTask(driver, executorUtils, baseConfiguration, executorConfiguration, taskDefinition, executorPid, artifactFetcher, taskInfo, templateManager, log, jsonObjectFileHelper, dockerUtils, s3Configuration);
+    return new SingularityExecutorTask(driver, executorUtils, baseConfiguration, executorConfiguration, taskDefinition, executorPid, artifactFetcher, taskInfo, templateManager, log, jsonObjectFileHelper, dockerUtils, s3Configuration, jsonObjectMapper);
   }
 
   private SingularityTaskExecutorData readExecutorData(ObjectMapper objectMapper, Protos.TaskInfo taskInfo) {

SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorArtifactFetcher.java

@@ -2,7 +2,6 @@
 
 import java.io.IOException;
 import java.nio.file.Path;
-import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 import java.util.concurrent.ExecutionException;
@@ -17,6 +16,7 @@
 import com.google.inject.name.Named;
 import com.hubspot.deploy.EmbeddedArtifact;
 import com.hubspot.deploy.ExecutorData;
+import com.hubspot.deploy.ExternalArtifact;
 import com.hubspot.deploy.RemoteArtifact;
 import com.hubspot.deploy.S3Artifact;
 import com.hubspot.deploy.S3ArtifactSignature;
@@ -61,18 +61,16 @@ public SingularityExecutorArtifactFetcher(@Named(SingularityExecutorModule.LOCAL
   public SingularityExecutorTaskArtifactFetcher buildTaskFetcher(ExecutorData executorData, SingularityExecutorTask task) {
     ArtifactManager artifactManager = new ArtifactManager(runnerBaseConfiguration, s3Configuration, task.getLog(), exceptionNotifier);
 
-    return new SingularityExecutorTaskArtifactFetcher(artifactManager, executorData, task);
+    return new SingularityExecutorTaskArtifactFetcher(artifactManager, task);
   }
 
   public class SingularityExecutorTaskArtifactFetcher {
 
     private final ArtifactManager artifactManager;
-    private final ExecutorData executorData;
     private final SingularityExecutorTask task;
 
-    private SingularityExecutorTaskArtifactFetcher(ArtifactManager artifactManager, ExecutorData executorData, SingularityExecutorTask task) {
+    private SingularityExecutorTaskArtifactFetcher(ArtifactManager artifactManager, SingularityExecutorTask task) {
       this.artifactManager = artifactManager;
-      this.executorData = executorData;
       this.task = task;
     }
 
@@ -81,29 +79,30 @@ public void cancel() {
       artifactManager.signalKillToProcessIfActive();
     }
 
-    public void fetchFiles() throws InterruptedException {
-      extractFiles(task, artifactManager, executorData);
+    public void fetchFiles(List<EmbeddedArtifact> embeddedArtifacts, List<S3Artifact> s3Artifacts, List<S3ArtifactSignature> s3ArtifactsWithSignature,
+        List<ExternalArtifact> remoteArtifacts)
+        throws InterruptedException {
+      extractFiles(task, artifactManager, embeddedArtifacts);
 
       boolean fetchS3ArtifactsLocally = true;
 
       final ImmutableList<S3Artifact> allS3Artifacts = ImmutableList.<S3Artifact>builder()
-          .addAll(executorData.getS3Artifacts())
-          .addAll(executorData.getS3ArtifactSignatures().or(Collections.<S3ArtifactSignature>emptyList()))
+          .addAll(s3Artifacts)
+          .addAll(s3ArtifactsWithSignature)
           .build();
 
       if (executorConfiguration.isUseLocalDownloadService() && !allS3Artifacts.isEmpty()) {
         final long start = System.currentTimeMillis();
 
-        task.getLog().info("Fetching {} (S3) artifacts and {} (S3) artifact signatures from {}", executorData.getS3Artifacts().size(),
-            executorData.getS3ArtifactSignatures().isPresent() ? executorData.getS3ArtifactSignatures().get().size() : 0, localDownloadUri);
+        task.getLog().info("Fetching {} (S3) artifacts and {} (S3) artifact signatures from {}", s3Artifacts.size(), s3ArtifactsWithSignature.size(), localDownloadUri);
 
         try {
           downloadFilesFromLocalDownloadService(allS3Artifacts, task);
 
           fetchS3ArtifactsLocally = false;
 
-          task.getLog().info("Fetched {} (S3) artifacts and {} (S3) artifact signatures from local download service in {}", executorData.getS3Artifacts().size(),
-              executorData.getS3ArtifactSignatures().isPresent() ? executorData.getS3ArtifactSignatures().get().size() : 0, JavaUtils.duration(start));
+          task.getLog().info("Fetched {} (S3) artifacts and {} (S3) artifact signatures from local download service in {}", s3Artifacts.size(), s3ArtifactsWithSignature.size(),
+              JavaUtils.duration(start));
         } catch (InterruptedException ie) {
           task.getLog().warn("Interrupted while downloading S3 artifacts from local download service");
           throw ie;
@@ -118,13 +117,13 @@ public void fetchFiles() throws InterruptedException {
         }
       }
 
-      for (RemoteArtifact externalArtifact : executorData.getExternalArtifacts()) {
+      for (RemoteArtifact externalArtifact : remoteArtifacts) {
         downloadRemoteArtifact(externalArtifact, artifactManager, task);
       }
     }
 
-    private void extractFiles(SingularityExecutorTask task, ArtifactManager artifactManager, ExecutorData executorData) {
-      for (EmbeddedArtifact artifact : executorData.getEmbeddedArtifacts()) {
+    private void extractFiles(SingularityExecutorTask task, ArtifactManager artifactManager, List<EmbeddedArtifact> embeddedArtifacts) {
+      for (EmbeddedArtifact artifact : embeddedArtifacts) {
         artifactManager.extract(artifact, task.getArtifactPath(artifact, task.getTaskDefinition().getTaskDirectoryPath()));
       }
     }

SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorArtifactVerifier.java

@@ -27,13 +27,13 @@ public SingularityExecutorArtifactVerifier(SingularityExecutorTaskDefinition tas
     this.taskDefinition = taskDefinition;
   }
 
-  public void checkSignatures() {
-    if (!taskDefinition.getExecutorData().getS3ArtifactSignatures().isPresent() || taskDefinition.getExecutorData().getS3ArtifactSignatures().get().isEmpty()) {
+  public void checkSignatures(List<S3ArtifactSignature> s3ArtifactsWithSignatures) {
+    if (s3ArtifactsWithSignatures.isEmpty()) {
       log.info("No files containing artifact signatures specified, skipping verification.");
       return;
     }
 
-    for (S3ArtifactSignature s3ArtifactSignature : taskDefinition.getExecutorData().getS3ArtifactSignatures().get()) {
+    for (S3ArtifactSignature s3ArtifactSignature : s3ArtifactsWithSignatures) {
       checkArtifactSignature(s3ArtifactSignature);
     }
   }

SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTask.java

@@ -11,6 +11,7 @@
 import org.apache.mesos.Protos;
 import org.apache.mesos.Protos.TaskState;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.base.Optional;
 import com.hubspot.deploy.Artifact;
 import com.hubspot.deploy.ExecutorData;
@@ -44,7 +45,7 @@ public class SingularityExecutorTask {
   private final SingularityExecutorArtifactVerifier artifactVerifier;
 
   public SingularityExecutorTask(ExecutorDriver driver, ExecutorUtils executorUtils, SingularityRunnerBaseConfiguration baseConfiguration, SingularityExecutorConfiguration executorConfiguration, SingularityExecutorTaskDefinition taskDefinition, String executorPid,
-      SingularityExecutorArtifactFetcher artifactFetcher, Protos.TaskInfo taskInfo, TemplateManager templateManager, Logger log, JsonObjectFileHelper jsonObjectFileHelper, DockerUtils dockerUtils, SingularityS3Configuration s3Configuration) {
+      SingularityExecutorArtifactFetcher artifactFetcher, Protos.TaskInfo taskInfo, TemplateManager templateManager, Logger log, JsonObjectFileHelper jsonObjectFileHelper, DockerUtils dockerUtils, SingularityS3Configuration s3Configuration, ObjectMapper objectMapper) {
     this.driver = driver;
     this.taskInfo = taskInfo;
     this.log = log;
@@ -61,7 +62,7 @@ public SingularityExecutorTask(ExecutorDriver driver, ExecutorUtils executorUtil
 
     this.taskLogManager = new SingularityExecutorTaskLogManager(taskDefinition, templateManager, baseConfiguration, executorConfiguration, log, jsonObjectFileHelper);
     this.taskCleanup = new SingularityExecutorTaskCleanup(taskLogManager, executorConfiguration, taskDefinition, log, dockerUtils);
-    this.processBuilder = new SingularityExecutorTaskProcessBuilder(this, executorUtils, artifactFetcher, templateManager, executorConfiguration, taskDefinition.getExecutorData(), executorPid, dockerUtils);
+    this.processBuilder = new SingularityExecutorTaskProcessBuilder(this, executorUtils, artifactFetcher, templateManager, executorConfiguration, taskDefinition.getExecutorData(), executorPid, dockerUtils, objectMapper);
     this.artifactVerifier = new SingularityExecutorArtifactVerifier(taskDefinition, log, executorConfiguration, s3Configuration);
   }