From d049b40404d82718847fa0113dfbd4d6c8cdc9f3 Mon Sep 17 00:00:00 2001 From: tpetr Date: Tue, 13 Dec 2016 17:21:49 -0500 Subject: [PATCH 1/2] customizable --use-compress-program for tar --- .../SingularityExecutorArtifactFetcher.java | 7 +++++-- .../SingularityRunnerBaseConfiguration.java | 13 +++++++++++++ .../singularity/s3/base/ArtifactManager.java | 19 +++++++++++-------- .../config/ArtifactManagerProvider.java | 7 +++++-- 4 files changed, 34 insertions(+), 12 deletions(-) diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorArtifactFetcher.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorArtifactFetcher.java index e4bb4253a3..606a3150ce 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorArtifactFetcher.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorArtifactFetcher.java @@ -22,6 +22,7 @@ import com.hubspot.mesos.JavaUtils; import com.hubspot.singularity.executor.config.SingularityExecutorConfiguration; import com.hubspot.singularity.executor.config.SingularityExecutorModule; +import com.hubspot.singularity.runner.base.configuration.SingularityRunnerBaseConfiguration; import com.hubspot.singularity.runner.base.sentry.SingularityRunnerExceptionNotifier; import com.hubspot.singularity.s3.base.ArtifactDownloadRequest; import com.hubspot.singularity.s3.base.ArtifactManager; @@ -41,21 +42,23 @@ public class SingularityExecutorArtifactFetcher { private final SingularityS3Configuration s3Configuration; private final ObjectMapper objectMapper; private final SingularityRunnerExceptionNotifier exceptionNotifier; + private final SingularityRunnerBaseConfiguration runnerBaseConfiguration; @Inject public SingularityExecutorArtifactFetcher(@Named(SingularityExecutorModule.LOCAL_DOWNLOAD_HTTP_CLIENT) AsyncHttpClient localDownloadHttpClient, SingularityS3Configuration s3Configuration, - SingularityExecutorConfiguration executorConfiguration, ObjectMapper objectMapper, SingularityRunnerExceptionNotifier exceptionNotifier) { + SingularityExecutorConfiguration executorConfiguration, ObjectMapper objectMapper, SingularityRunnerExceptionNotifier exceptionNotifier, SingularityRunnerBaseConfiguration runnerBaseConfiguration) { this.localDownloadHttpClient = localDownloadHttpClient; this.executorConfiguration = executorConfiguration; this.s3Configuration = s3Configuration; this.objectMapper = objectMapper; this.exceptionNotifier = exceptionNotifier; + this.runnerBaseConfiguration = runnerBaseConfiguration; this.localDownloadUri = String.format(LOCAL_DOWNLOAD_STRING_FORMAT, s3Configuration.getLocalDownloadHttpPort(), s3Configuration.getLocalDownloadPath()); } public SingularityExecutorTaskArtifactFetcher buildTaskFetcher(ExecutorData executorData, SingularityExecutorTask task) { - ArtifactManager artifactManager = new ArtifactManager(s3Configuration, task.getLog(), exceptionNotifier); + ArtifactManager artifactManager = new ArtifactManager(runnerBaseConfiguration, s3Configuration, task.getLog(), exceptionNotifier); return new SingularityExecutorTaskArtifactFetcher(artifactManager, executorData, task); } diff --git a/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/configuration/SingularityRunnerBaseConfiguration.java b/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/configuration/SingularityRunnerBaseConfiguration.java index 34ba806ff4..5c9e991033 100644 --- a/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/configuration/SingularityRunnerBaseConfiguration.java +++ b/SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/configuration/SingularityRunnerBaseConfiguration.java @@ -34,6 +34,10 @@ public class SingularityRunnerBaseConfiguration extends BaseRunnerConfiguration @JsonProperty private String sentryPrefix = ""; + @NotNull + @JsonProperty + private Optional useCompressProgram = Optional.absent(); + public SingularityRunnerBaseConfiguration() { super(Optional.absent()); this.setLoggingDirectory(Optional.of("/var/log/singularity/")); @@ -87,6 +91,14 @@ public void setSentryPrefix(String sentryPrefix) { this.sentryPrefix = sentryPrefix; } + public Optional getUseCompressProgram() { + return useCompressProgram; + } + + public void setUseCompressProgram(Optional useCompressProgram) { + this.useCompressProgram = useCompressProgram; + } + @Override public String toString() { return "SingularityRunnerBaseConfiguration[" + @@ -94,6 +106,7 @@ public String toString() { ", s3UploaderMetadataSuffix='" + s3UploaderMetadataSuffix + '\'' + ", logWatcherMetadataDirectory='" + logWatcherMetadataDirectory + '\'' + ", logWatcherMetadataSuffix='" + logWatcherMetadataSuffix + '\'' + + ", useCompressProgram=" + useCompressProgram + ']'; } } diff --git a/SingularityS3Base/src/main/java/com/hubspot/singularity/s3/base/ArtifactManager.java b/SingularityS3Base/src/main/java/com/hubspot/singularity/s3/base/ArtifactManager.java index aa99d850ba..1c0bd4c7f9 100644 --- a/SingularityS3Base/src/main/java/com/hubspot/singularity/s3/base/ArtifactManager.java +++ b/SingularityS3Base/src/main/java/com/hubspot/singularity/s3/base/ArtifactManager.java @@ -13,6 +13,7 @@ import org.slf4j.Logger; +import com.google.common.base.Optional; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; import com.google.common.hash.HashCode; @@ -22,6 +23,7 @@ import com.hubspot.deploy.ExternalArtifact; import com.hubspot.deploy.RemoteArtifact; import com.hubspot.deploy.S3Artifact; +import com.hubspot.singularity.runner.base.configuration.SingularityRunnerBaseConfiguration; import com.hubspot.singularity.runner.base.sentry.SingularityRunnerExceptionNotifier; import com.hubspot.singularity.runner.base.shared.ProcessFailedException; import com.hubspot.singularity.runner.base.shared.SimpleProcessManager; @@ -32,13 +34,15 @@ public class ArtifactManager extends SimpleProcessManager { private final Path cacheDirectory; private final Logger log; private final S3ArtifactDownloader s3ArtifactDownloader; + private final Optional useCompressProgram; - public ArtifactManager(SingularityS3Configuration configuration, Logger log, SingularityRunnerExceptionNotifier exceptionNotifier) { + public ArtifactManager(SingularityRunnerBaseConfiguration runnerBaseConfiguration, SingularityS3Configuration configuration, Logger log, SingularityRunnerExceptionNotifier exceptionNotifier) { super(log); this.cacheDirectory = Paths.get(configuration.getArtifactCacheDirectory()); this.log = log; this.s3ArtifactDownloader = new S3ArtifactDownloader(configuration, log, exceptionNotifier); + this.useCompressProgram = runnerBaseConfiguration.getUseCompressProgram(); } private long getSize(Path path) { @@ -207,14 +211,13 @@ public void copy(Path source, Path destination, String destinationFilename) { public void untar(Path source, Path destination) { log.info("Untarring {} to {}", source, destination); - final List command = ImmutableList.of( - "tar", - "-oxzf", - source.toString(), - "-C", - destination.toString()); + final ImmutableList.Builder commandBuilder = ImmutableList.builder().add("tar", "-oxzf", source.toString(), "-C", destination.toString()); - runCommandAndThrowRuntimeException(command); + if (useCompressProgram.isPresent()) { + commandBuilder.add("--use-compress-program", useCompressProgram.get()); + } + + runCommandAndThrowRuntimeException(commandBuilder.build()); } private void runCommandAndThrowRuntimeException(List command) { diff --git a/SingularityS3Downloader/src/main/java/com/hubspot/singularity/s3downloader/config/ArtifactManagerProvider.java b/SingularityS3Downloader/src/main/java/com/hubspot/singularity/s3downloader/config/ArtifactManagerProvider.java index 085a40aa73..c7bc059f13 100644 --- a/SingularityS3Downloader/src/main/java/com/hubspot/singularity/s3downloader/config/ArtifactManagerProvider.java +++ b/SingularityS3Downloader/src/main/java/com/hubspot/singularity/s3downloader/config/ArtifactManagerProvider.java @@ -5,6 +5,7 @@ import com.google.inject.Inject; import com.google.inject.Provider; +import com.hubspot.singularity.runner.base.configuration.SingularityRunnerBaseConfiguration; import com.hubspot.singularity.runner.base.sentry.SingularityRunnerExceptionNotifier; import com.hubspot.singularity.s3.base.ArtifactManager; import com.hubspot.singularity.s3.base.config.SingularityS3Configuration; @@ -12,19 +13,21 @@ public class ArtifactManagerProvider implements Provider { private final Logger log; + private final SingularityRunnerBaseConfiguration runnerBaseConfiguration; private final SingularityS3Configuration s3Configuration; private final SingularityRunnerExceptionNotifier exceptionNotifier; @Inject - public ArtifactManagerProvider(SingularityS3Configuration s3Configuration, SingularityRunnerExceptionNotifier exceptionNotifier) { + public ArtifactManagerProvider(SingularityRunnerBaseConfiguration runnerBaseConfiguration, SingularityS3Configuration s3Configuration, SingularityRunnerExceptionNotifier exceptionNotifier) { this.log = LoggerFactory.getLogger(ArtifactManager.class); + this.runnerBaseConfiguration = runnerBaseConfiguration; this.s3Configuration = s3Configuration; this.exceptionNotifier = exceptionNotifier; } @Override public ArtifactManager get() { - return new ArtifactManager(s3Configuration, log, exceptionNotifier); + return new ArtifactManager(runnerBaseConfiguration, s3Configuration, log, exceptionNotifier); } } From a5998b5af20777cf9530914439e2a52e8421ebe6 Mon Sep 17 00:00:00 2001 From: tpetr Date: Tue, 13 Dec 2016 17:42:37 -0500 Subject: [PATCH 2/2] --use-compress-program and -z are mutually exclusive --- .../com/hubspot/singularity/s3/base/ArtifactManager.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/SingularityS3Base/src/main/java/com/hubspot/singularity/s3/base/ArtifactManager.java b/SingularityS3Base/src/main/java/com/hubspot/singularity/s3/base/ArtifactManager.java index 1c0bd4c7f9..3416446432 100644 --- a/SingularityS3Base/src/main/java/com/hubspot/singularity/s3/base/ArtifactManager.java +++ b/SingularityS3Base/src/main/java/com/hubspot/singularity/s3/base/ArtifactManager.java @@ -211,10 +211,12 @@ public void copy(Path source, Path destination, String destinationFilename) { public void untar(Path source, Path destination) { log.info("Untarring {} to {}", source, destination); - final ImmutableList.Builder commandBuilder = ImmutableList.builder().add("tar", "-oxzf", source.toString(), "-C", destination.toString()); + final ImmutableList.Builder commandBuilder = ImmutableList.builder().add("tar", "-oxf", source.toString(), "-C", destination.toString()); if (useCompressProgram.isPresent()) { - commandBuilder.add("--use-compress-program", useCompressProgram.get()); + commandBuilder.add("--use-compress-program=" + useCompressProgram.get()); + } else { + commandBuilder.add("-z"); } runCommandAndThrowRuntimeException(commandBuilder.build());