Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Tweaks to s3UploaderAdditionalFiles #1000

Merged
merged 2 commits into from Jul 20, 2016
Merged

Tweaks to s3UploaderAdditionalFiles #1000

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7ada27e
make extra s3 uploader files relative to task directory and with opti…
ssalinas Apr 15, 2016
147e502
extra char
ssalinas May 9, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
12 changes: 10 additions & 2 deletions .../com/hubspot/singularity/executor/config/SingularityExecutorS3UploaderAdditionalFile.java
View file
Open in desktop
Expand Up @@ -9,21 +9,24 @@ public class SingularityExecutorS3UploaderAdditionalFile {
private final Optional<String> s3UploaderBucket;
private final Optional<String> s3UploaderKeyPattern;
private final Optional<String> s3UploaderFilenameHint;
private final Optional<String> directory;

@JsonCreator
public static SingularityExecutorS3UploaderAdditionalFile fromString(String value) {
return new SingularityExecutorS3UploaderAdditionalFile(value, Optional.<String>absent(), Optional.<String>absent(), Optional.<String>absent());
return new SingularityExecutorS3UploaderAdditionalFile(value, Optional.<String>absent(), Optional.<String>absent(), Optional.<String>absent(), Optional.<String>absent());
}

@JsonCreator
public SingularityExecutorS3UploaderAdditionalFile(@JsonProperty("filename") String filename,
@JsonProperty("s3UploaderBucket") Optional<String> s3UploaderBucket,
@JsonProperty("s3UploaderKeyPattern") Optional<String> s3UploaderKeyPattern,
@JsonProperty("s3UploaderFilenameHint") Optional<String> s3UploaderFilenameHint) {
@JsonProperty("s3UploaderFilenameHint") Optional<String> s3UploaderFilenameHint,
@JsonProperty("directory") Optional<String> directory) {
this.filename = filename;
this.s3UploaderBucket = s3UploaderBucket;
this.s3UploaderKeyPattern = s3UploaderKeyPattern;
this.s3UploaderFilenameHint = s3UploaderFilenameHint;
this.directory = directory;
}

public String getFilename() {
Expand All @@ -42,13 +45,18 @@ public Optional<String> getS3UploaderFilenameHint() {
return s3UploaderFilenameHint;
}

public Optional<String> getDirectory() {
return directory;
}

@Override
public String toString() {
return "SingularityExecutorS3UploaderAdditionalFile[" +
"filename='" + filename + '\'' +
", s3UploaderBucket=" + s3UploaderBucket +
", s3UploaderKeyPattern=" + s3UploaderKeyPattern +
", s3UploaderFilenameHint=" + s3UploaderFilenameHint +
", directory=" + directory +
']';
}
}
4 changes: 3 additions & 1 deletion ...rc/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java
View file
Open in desktop
Expand Up @@ -60,7 +60,9 @@ private boolean writeS3MetadataFileForRotatedFiles(boolean finished) {

int index = 1;
for (SingularityExecutorS3UploaderAdditionalFile additionalFile : configuration.getS3UploaderAdditionalFiles()) {
result = result && writeS3MetadataFile(additionalFile.getS3UploaderFilenameHint().or(String.format("extra%d", index)), logrotateDirectory, String.format("%s*.gz*", additionalFile.getFilename()), additionalFile.getS3UploaderBucket(), additionalFile.getS3UploaderKeyPattern(), finished);
Path directory = additionalFile.getDirectory().isPresent() ? taskDefinition.getTaskDirectoryPath().resolve(additionalFile.getDirectory().get()) : taskDefinition.getTaskDirectoryPath();
String fileGlob = additionalFile.getFilename() != null && additionalFile.getFilename().contains("*") ? additionalFile.getFilename() : String.format("%s*.gz*", additionalFile.getFilename());
result = result && writeS3MetadataFile(additionalFile.getS3UploaderFilenameHint().or(String.format("extra%d", index)), directory, fileGlob, additionalFile.getS3UploaderBucket(), additionalFile.getS3UploaderKeyPattern(), finished);
index++;
}

Expand Down