make extra s3 uploader files relative to task directory and with opti…

…onal custom directory
HubSpot · Apr 27, 2016 · 7ada27e · 7ada27e · tpetr · May 5, 2016
1 parent 5c2a7c6
commit 7ada27e
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 4 deletions.
diff --git a/.../com/hubspot/singularity/executor/config/SingularityExecutorS3UploaderAdditionalFile.java b/.../com/hubspot/singularity/executor/config/SingularityExecutorS3UploaderAdditionalFile.java
@@ -9,21 +9,24 @@ public class SingularityExecutorS3UploaderAdditionalFile {
     private final Optional<String> s3UploaderBucket;
     private final Optional<String> s3UploaderKeyPattern;
     private final Optional<String> s3UploaderFilenameHint;
+    private final Optional<String> directory;
 
     @JsonCreator
     public static SingularityExecutorS3UploaderAdditionalFile fromString(String value) {
-        return new SingularityExecutorS3UploaderAdditionalFile(value, Optional.<String>absent(), Optional.<String>absent(), Optional.<String>absent());
+        return new SingularityExecutorS3UploaderAdditionalFile(value, Optional.<String>absent(), Optional.<String>absent(), Optional.<String>absent(), Optional.<String>absent());
     }
 
     @JsonCreator
     public SingularityExecutorS3UploaderAdditionalFile(@JsonProperty("filename") String filename,
         @JsonProperty("s3UploaderBucket") Optional<String> s3UploaderBucket,
         @JsonProperty("s3UploaderKeyPattern") Optional<String> s3UploaderKeyPattern,
-        @JsonProperty("s3UploaderFilenameHint") Optional<String> s3UploaderFilenameHint) {
+        @JsonProperty("s3UploaderFilenameHint") Optional<String> s3UploaderFilenameHint,
+        @JsonProperty("directory") Optional<String> directory) {
         this.filename = filename;
         this.s3UploaderBucket = s3UploaderBucket;
         this.s3UploaderKeyPattern = s3UploaderKeyPattern;
         this.s3UploaderFilenameHint = s3UploaderFilenameHint;
+        this.directory = directory;
     }
 
     public String getFilename() {
@@ -42,13 +45,18 @@ public Optional<String> getS3UploaderFilenameHint() {
         return s3UploaderFilenameHint;
     }
 
+    public Optional<String> getDirectory() {
+        return directory;
+    }
+
     @Override
     public String toString() {
         return "SingularityExecutorS3UploaderAdditionalFile[" +
             "filename='" + filename + '\'' +
             ", s3UploaderBucket=" + s3UploaderBucket +
             ", s3UploaderKeyPattern=" + s3UploaderKeyPattern +
             ", s3UploaderFilenameHint=" + s3UploaderFilenameHint +
+            ", directory=" + directory +
             ']';
     }
 }
diff --git a/...rc/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java b/...rc/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java
@@ -56,11 +56,13 @@ private boolean writeS3MetadataFileForRotatedFiles(boolean finished) {
     final Path serviceLogParent = serviceLogOutPath.getParent();
     final Path logrotateDirectory = serviceLogParent.resolve(configuration.getLogrotateToDirectory());
 
-    boolean result = writeS3MetadataFile("default", logrotateDirectory, String.format("%s*.gz*", taskDefinition.getServiceLogOutPath().getFileName()), Optional.<String>absent(), Optional.<String>absent(), finished);
+    boolean result = writeS3MetadataFile("default", logrotateDirectory, String.format("%ss*.gz*", taskDefinition.getServiceLogOutPath().getFileName()), Optional.<String>absent(), Optional.<String>absent(), finished);
 
     int index = 1;
     for (SingularityExecutorS3UploaderAdditionalFile additionalFile : configuration.getS3UploaderAdditionalFiles()) {
-      result = result && writeS3MetadataFile(additionalFile.getS3UploaderFilenameHint().or(String.format("extra%d", index)), logrotateDirectory, String.format("%s*.gz*", additionalFile.getFilename()), additionalFile.getS3UploaderBucket(), additionalFile.getS3UploaderKeyPattern(), finished);
+      Path directory = additionalFile.getDirectory().isPresent() ? taskDefinition.getTaskDirectoryPath().resolve(additionalFile.getDirectory().get()) : taskDefinition.getTaskDirectoryPath();
+      String fileGlob = additionalFile.getFilename() != null && additionalFile.getFilename().contains("*") ? additionalFile.getFilename() : String.format("%s*.gz*", additionalFile.getFilename());
+      result = result && writeS3MetadataFile(additionalFile.getS3UploaderFilenameHint().or(String.format("extra%d", index)), directory, fileGlob, additionalFile.getS3UploaderBucket(), additionalFile.getS3UploaderKeyPattern(), finished);
       index++;
     }