Skip to content

Commit

Permalink
Let credentials be a Map<String, Object>
Browse files Browse the repository at this point in the history
  • Loading branch information
@ssalinas
ssalinas committed Jan 24, 2018
1 parent d9621ec commit 4e368d1
Show file tree
Hide file tree
Showing 5 changed files with 24 additions and 27 deletions.
3 changes: 2 additions & 1 deletion ...rc/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java
View file
Expand Up @@ -6,6 +6,7 @@
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;

import org.slf4j.Logger;
Expand Down Expand Up @@ -254,7 +255,7 @@ private boolean writeS3MetadataFile(String filenameHint, Path pathToS3Directory,
}

S3UploadMetadata s3UploadMetadata = new S3UploadMetadata(pathToS3Directory.toString(), globForS3Files, s3UploaderBucket, getS3KeyPattern(s3KeyPattern.or(taskDefinition.getExecutorData().getS3UploaderKeyPattern())), finished, Optional.<String> absent(),
Optional. absent(), Optional. absent(), Optional. absent(), Optional. absent(), s3StorageClass, applyS3StorageClassAfterBytes, Optional.of(finished), Optional.of(checkSubdirectories), Optional.absent(), Optional.absent());
Optional. absent(), Optional. absent(), Optional. absent(), Optional. absent(), s3StorageClass, applyS3StorageClassAfterBytes, Optional.of(finished), Optional.of(checkSubdirectories), Optional.absent(), Collections.emptyMap());

String s3UploadMetadataFileName = String.format("%s-%s%s", taskDefinition.getTaskId(), filenameHint, baseConfiguration.getS3UploaderMetadataSuffix());

Expand Down
7 changes: 7 additions & 0 deletions ...erBase/src/main/java/com/hubspot/singularity/runner/base/shared/JsonObjectFileHelper.java
View file
Expand Up @@ -2,11 +2,14 @@

import static java.nio.charset.StandardCharsets.UTF_8;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.Map;

import org.slf4j.Logger;

Expand Down Expand Up @@ -51,6 +54,10 @@ public <T> Optional<T> read(Path file, Logger log, Class<T> clazz) throws IOExce
return Optional.absent();
}

public InputStream toInputStream(Map<String, Object> input) throws IOException {
return new ByteArrayInputStream(objectMapper.writeValueAsBytes(input));
}

public boolean writeObject(Object object, Path path, Logger log) {
final long start = System.currentTimeMillis();

Expand Down
11 changes: 7 additions & 4 deletions ...RunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/S3UploadMetadata.java
View file
@@ -1,5 +1,8 @@
package com.hubspot.singularity.runner.base.shared;

import java.util.Collections;
import java.util.Map;

import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Optional;
Expand Down Expand Up @@ -53,7 +56,7 @@ public class S3UploadMetadata {
private final Optional<Boolean> uploadImmediately;
private final boolean checkSubdirectories;
private final SingularityUploaderType uploaderType;
private final Optional<GCSCredentials> gcsCredentials;
private final Map<String, Object> gcsCredentials;

@JsonCreator
public S3UploadMetadata(@JsonProperty("directory") String directory,
Expand All @@ -71,7 +74,7 @@ public S3UploadMetadata(@JsonProperty("directory") String directory,
@JsonProperty("uploadImmediately") Optional<Boolean> uploadImmediately,
@JsonProperty("checkSubdirectories") Optional<Boolean> checkSubdirectories,
@JsonProperty("uploaderType") Optional<SingularityUploaderType> uploaderType,
@JsonProperty("gcsCredentials") Optional<GCSCredentials> gcsCredentials) {
@JsonProperty("gcsCredentials") Map<String, Object> gcsCredentials) {
Preconditions.checkNotNull(directory);
Preconditions.checkNotNull(fileGlob);
Preconditions.checkNotNull(s3Bucket);
Expand All @@ -92,7 +95,7 @@ public S3UploadMetadata(@JsonProperty("directory") String directory,
this.uploadImmediately = uploadImmediately;
this.checkSubdirectories = checkSubdirectories.or(false);
this.uploaderType = uploaderType.or(SingularityUploaderType.S3);
this.gcsCredentials = gcsCredentials;
this.gcsCredentials = gcsCredentials != null ? gcsCredentials : Collections.emptyMap();
}


Expand Down Expand Up @@ -195,7 +198,7 @@ public SingularityUploaderType getUploaderType() {
return uploaderType;
}

public Optional<GCSCredentials> getGcsCredentials() {
public Map<String, Object> getGcsCredentials() {
return gcsCredentials;
}

Expand Down
28 changes: 7 additions & 21 deletions ...tyS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityGCSUploader.java
View file
Expand Up @@ -15,8 +15,6 @@
import com.github.rholder.retry.StopStrategies;
import com.github.rholder.retry.WaitStrategies;
import com.google.auth.oauth2.GoogleCredentials;
import com.google.auth.oauth2.ServiceAccountCredentials;
import com.google.auth.oauth2.UserCredentials;
import com.google.cloud.storage.BlobInfo;
import com.google.cloud.storage.Storage;
import com.google.cloud.storage.StorageClass;
Expand All @@ -26,7 +24,7 @@
import com.hubspot.singularity.SingularityS3FormatHelper;
import com.hubspot.singularity.SingularityS3Log;
import com.hubspot.singularity.runner.base.sentry.SingularityRunnerExceptionNotifier;
import com.hubspot.singularity.runner.base.shared.GCSCredentials;
import com.hubspot.singularity.runner.base.shared.JsonObjectFileHelper;
import com.hubspot.singularity.runner.base.shared.S3UploadMetadata;
import com.hubspot.singularity.s3uploader.config.SingularityS3UploaderConfiguration;
import com.hubspot.singularity.s3uploader.config.SingularityS3UploaderContentHeaders;
Expand All @@ -35,31 +33,19 @@ public class SingularityGCSUploader extends SingularityUploader {
private final Storage storage;

public SingularityGCSUploader(S3UploadMetadata uploadMetadata, FileSystem fileSystem, SingularityS3UploaderMetrics metrics, Path metadataPath,
SingularityS3UploaderConfiguration configuration, String hostname, SingularityRunnerExceptionNotifier exceptionNotifier) {
SingularityS3UploaderConfiguration configuration, String hostname, SingularityRunnerExceptionNotifier exceptionNotifier,
JsonObjectFileHelper jsonHelper) {
super(uploadMetadata, fileSystem, metrics, metadataPath, configuration, hostname, exceptionNotifier);
this.storage = StorageOptions.newBuilder()
.setCredentials(loadCredentials(uploadMetadata))
.setCredentials(loadCredentials(uploadMetadata, jsonHelper))
.build()
.getService();
}

public static GoogleCredentials loadCredentials(S3UploadMetadata uploadMetadata) {
public static GoogleCredentials loadCredentials(S3UploadMetadata uploadMetadata, JsonObjectFileHelper jsonHelper) {
try {
if (uploadMetadata.getGcsCredentials().isPresent()) {
GCSCredentials gcsCredentials = uploadMetadata.getGcsCredentials().get();
switch (gcsCredentials.getType()) {
case USER:
return UserCredentials.newBuilder()
.setClientId(gcsCredentials.getClientId())
.setClientSecret(gcsCredentials.getClientSecret())
.setRefreshToken(gcsCredentials.getRefreshToken())
.build();
case SERVICE_ACCOUNT:
return ServiceAccountCredentials.fromPkcs8(
gcsCredentials.getClientId(), gcsCredentials.getClientEmail(), gcsCredentials.getPrivateKey(), gcsCredentials.getPrivateKeyId(), gcsCredentials.getScopes());
default:
throw new RuntimeException(String.format("Cannot handle gcs credential type of %s (must be one of: USER, SERVICE_ACCOUNT)", gcsCredentials.getType()));
}
if (!uploadMetadata.getGcsCredentials().isEmpty()) {
return GoogleCredentials.fromStream(jsonHelper.toInputStream(uploadMetadata.getGcsCredentials()));
}

// Load from default credentials as determined by GOOGLE_APPLICATION_CREDENTIALS var if none provided in metadata
Expand Down
2 changes: 1 addition & 1 deletion ...ploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3UploaderDriver.java
View file
Expand Up @@ -448,7 +448,7 @@ private boolean handleNewOrModifiedS3Metadata(Path filename) throws IOException
if (metadata.getUploaderType() == SingularityUploaderType.S3) {
uploader = new SingularityS3Uploader(bucketCreds.or(defaultCredentials), metadata, fileSystem, metrics, filename, configuration, hostname, exceptionNotifier);
} else {
uploader = new SingularityGCSUploader(metadata, fileSystem, metrics, filename, configuration, hostname, exceptionNotifier);
uploader = new SingularityGCSUploader(metadata, fileSystem, metrics, filename, configuration, hostname, exceptionNotifier, jsonObjectFileHelper);
}

if (metadata.isFinished()) {
Expand Down

0 comments on commit 4e368d1

Please sign in to comment.