From 7ada27e40fba744a0429464d10c4fe8a5bcb84a9 Mon Sep 17 00:00:00 2001
From: Stephen Salinas <ssalinas@hubspot.com>
Date: Fri, 15 Apr 2016 15:00:09 -0400
Subject: [PATCH] make extra s3 uploader files relative to task directory and
 with optional custom directory

---
 .../SingularityExecutorS3UploaderAdditionalFile.java | 12 ++++++++++--
 .../task/SingularityExecutorTaskLogManager.java      |  6 ++++--
 2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorS3UploaderAdditionalFile.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorS3UploaderAdditionalFile.java
index 8908e604c4..c4e4ffb4cd 100644
--- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorS3UploaderAdditionalFile.java
+++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorS3UploaderAdditionalFile.java
@@ -9,21 +9,24 @@ public class SingularityExecutorS3UploaderAdditionalFile {
     private final Optional<String> s3UploaderBucket;
     private final Optional<String> s3UploaderKeyPattern;
     private final Optional<String> s3UploaderFilenameHint;
+    private final Optional<String> directory;
 
     @JsonCreator
     public static SingularityExecutorS3UploaderAdditionalFile fromString(String value) {
-        return new SingularityExecutorS3UploaderAdditionalFile(value, Optional.<String>absent(), Optional.<String>absent(), Optional.<String>absent());
+        return new SingularityExecutorS3UploaderAdditionalFile(value, Optional.<String>absent(), Optional.<String>absent(), Optional.<String>absent(), Optional.<String>absent());
     }
 
     @JsonCreator
     public SingularityExecutorS3UploaderAdditionalFile(@JsonProperty("filename") String filename,
         @JsonProperty("s3UploaderBucket") Optional<String> s3UploaderBucket,
         @JsonProperty("s3UploaderKeyPattern") Optional<String> s3UploaderKeyPattern,
-        @JsonProperty("s3UploaderFilenameHint") Optional<String> s3UploaderFilenameHint) {
+        @JsonProperty("s3UploaderFilenameHint") Optional<String> s3UploaderFilenameHint,
+        @JsonProperty("directory") Optional<String> directory) {
         this.filename = filename;
         this.s3UploaderBucket = s3UploaderBucket;
         this.s3UploaderKeyPattern = s3UploaderKeyPattern;
         this.s3UploaderFilenameHint = s3UploaderFilenameHint;
+        this.directory = directory;
     }
 
     public String getFilename() {
@@ -42,6 +45,10 @@ public Optional<String> getS3UploaderFilenameHint() {
         return s3UploaderFilenameHint;
     }
 
+    public Optional<String> getDirectory() {
+        return directory;
+    }
+
     @Override
     public String toString() {
         return "SingularityExecutorS3UploaderAdditionalFile[" +
@@ -49,6 +56,7 @@ public String toString() {
             ", s3UploaderBucket=" + s3UploaderBucket +
             ", s3UploaderKeyPattern=" + s3UploaderKeyPattern +
             ", s3UploaderFilenameHint=" + s3UploaderFilenameHint +
+            ", directory=" + directory +
             ']';
     }
 }
diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java
index c142b1e247..51f3ec8af1 100644
--- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java
+++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java
@@ -56,11 +56,13 @@ private boolean writeS3MetadataFileForRotatedFiles(boolean finished) {
     final Path serviceLogParent = serviceLogOutPath.getParent();
     final Path logrotateDirectory = serviceLogParent.resolve(configuration.getLogrotateToDirectory());
 
-    boolean result = writeS3MetadataFile("default", logrotateDirectory, String.format("%s*.gz*", taskDefinition.getServiceLogOutPath().getFileName()), Optional.<String>absent(), Optional.<String>absent(), finished);
+    boolean result = writeS3MetadataFile("default", logrotateDirectory, String.format("%ss*.gz*", taskDefinition.getServiceLogOutPath().getFileName()), Optional.<String>absent(), Optional.<String>absent(), finished);
 
     int index = 1;
     for (SingularityExecutorS3UploaderAdditionalFile additionalFile : configuration.getS3UploaderAdditionalFiles()) {
-      result = result && writeS3MetadataFile(additionalFile.getS3UploaderFilenameHint().or(String.format("extra%d", index)), logrotateDirectory, String.format("%s*.gz*", additionalFile.getFilename()), additionalFile.getS3UploaderBucket(), additionalFile.getS3UploaderKeyPattern(), finished);
+      Path directory = additionalFile.getDirectory().isPresent() ? taskDefinition.getTaskDirectoryPath().resolve(additionalFile.getDirectory().get()) : taskDefinition.getTaskDirectoryPath();
+      String fileGlob = additionalFile.getFilename() != null && additionalFile.getFilename().contains("*") ? additionalFile.getFilename() : String.format("%s*.gz*", additionalFile.getFilename());
+      result = result && writeS3MetadataFile(additionalFile.getS3UploaderFilenameHint().or(String.format("extra%d", index)), directory, fileGlob, additionalFile.getS3UploaderBucket(), additionalFile.getS3UploaderKeyPattern(), finished);
       index++;
     }