Skip to content

Commit

Permalink
Support KMS-managed keys S3 server-side encryption
Browse files Browse the repository at this point in the history
  • Loading branch information
@chappidim @electrum
chappidim authored and electrum committed Feb 10, 2017
1 parent e5529c4 commit 0f2909d
Show file tree
Hide file tree
Showing 6 changed files with 99 additions and 5 deletions.
7 changes: 7 additions & 0 deletions presto-docs/src/main/sphinx/connector/hive.rst
View file
Expand Up @@ -225,6 +225,13 @@ Property Name Description

``hive.s3.sse.enabled`` Use S3 server-side encryption (defaults to ``false``).

``hive.s3.sse.type`` The type of key management for S3 server-side encryption.
Use ``S3`` for S3 managed or ``KMS`` for KMS-managed keys
(defaults to ``S3``).

``hive.s3.ss3.kms-key-id`` The KMS Key ID to use for S3 server-side encryption with
KMS-managed keys. If not set, the default key is used.

``hive.s3.kms-key-id`` If set, use S3 client-side encryption and use the AWS
KMS to store encryption keys and use the value of
this property as the KMS Key ID for newly created
Expand Down
8 changes: 8 additions & 0 deletions presto-hive/src/main/java/com/facebook/presto/hive/HdfsConfigurationUpdater.java
View file
Expand Up @@ -59,8 +59,10 @@ public class HdfsConfigurationUpdater
private final boolean s3UseInstanceCredentials;
private final boolean s3SslEnabled;
private final boolean s3SseEnabled;
private final PrestoS3SseType s3SseType;
private final String s3EncryptionMaterialsProvider;
private final String s3KmsKeyId;
private final String s3SseKmsKeyId;
private final int s3MaxClientRetries;
private final int s3MaxErrorRetries;
private final Duration s3MaxBackoffTime;
Expand Down Expand Up @@ -97,8 +99,10 @@ public HdfsConfigurationUpdater(HiveClientConfig hiveClientConfig, HiveS3Config
this.s3UseInstanceCredentials = s3Config.isS3UseInstanceCredentials();
this.s3SslEnabled = s3Config.isS3SslEnabled();
this.s3SseEnabled = s3Config.isS3SseEnabled();
this.s3SseType = s3Config.getS3SseType();
this.s3EncryptionMaterialsProvider = s3Config.getS3EncryptionMaterialsProvider();
this.s3KmsKeyId = s3Config.getS3KmsKeyId();
this.s3SseKmsKeyId = s3Config.getS3SseKmsKeyId();
this.s3MaxClientRetries = s3Config.getS3MaxClientRetries();
this.s3MaxErrorRetries = s3Config.getS3MaxErrorRetries();
this.s3MaxBackoffTime = s3Config.getS3MaxBackoffTime();
Expand Down Expand Up @@ -186,12 +190,16 @@ public void updateConfiguration(Configuration config)
config.setBoolean(PrestoS3FileSystem.S3_USE_INSTANCE_CREDENTIALS, s3UseInstanceCredentials);
config.setBoolean(PrestoS3FileSystem.S3_SSL_ENABLED, s3SslEnabled);
config.setBoolean(PrestoS3FileSystem.S3_SSE_ENABLED, s3SseEnabled);
config.set(PrestoS3FileSystem.S3_SSE_TYPE, s3SseType.name());
if (s3EncryptionMaterialsProvider != null) {
config.set(PrestoS3FileSystem.S3_ENCRYPTION_MATERIALS_PROVIDER, s3EncryptionMaterialsProvider);
}
if (s3KmsKeyId != null) {
config.set(PrestoS3FileSystem.S3_KMS_KEY_ID, s3KmsKeyId);
}
if (s3SseKmsKeyId != null) {
config.set(PrestoS3FileSystem.S3_SSE_KMS_KEY_ID, s3SseKmsKeyId);
}
config.setInt(PrestoS3FileSystem.S3_MAX_CLIENT_RETRIES, s3MaxClientRetries);
config.setInt(PrestoS3FileSystem.S3_MAX_ERROR_RETRIES, s3MaxErrorRetries);
config.set(PrestoS3FileSystem.S3_MAX_BACKOFF_TIME, s3MaxBackoffTime.toString());
Expand Down
29 changes: 29 additions & 0 deletions presto-hive/src/main/java/com/facebook/presto/hive/HiveS3Config.java
View file
Expand Up @@ -38,8 +38,10 @@ public class HiveS3Config
private boolean s3UseInstanceCredentials = true;
private boolean s3SslEnabled = true;
private boolean s3SseEnabled;
private PrestoS3SseType s3SseType = PrestoS3SseType.S3;
private String s3EncryptionMaterialsProvider;
private String s3KmsKeyId;
private String s3SseKmsKeyId;
private int s3MaxClientRetries = 3;
private int s3MaxErrorRetries = 10;
private Duration s3MaxBackoffTime = new Duration(10, TimeUnit.MINUTES);
Expand Down Expand Up @@ -151,6 +153,19 @@ public HiveS3Config setS3KmsKeyId(String s3KmsKeyId)
return this;
}

public String getS3SseKmsKeyId()
{
return s3SseKmsKeyId;
}

@Config("hive.s3.sse.kms-key-id")
@ConfigDescription("KMS Key ID to use for S3 server-side encryption with KMS-managed key")
public HiveS3Config setS3SseKmsKeyId(String s3SseKmsKeyId)
{
this.s3SseKmsKeyId = s3SseKmsKeyId;
return this;
}

public boolean isS3SseEnabled()
{
return s3SseEnabled;
Expand All @@ -164,6 +179,20 @@ public HiveS3Config setS3SseEnabled(boolean s3SseEnabled)
return this;
}

@NotNull
public PrestoS3SseType getS3SseType()
{
return s3SseType;
}

@Config("hive.s3.sse.type")
@ConfigDescription("Key management type for S3 server-side encryption (S3 or KMS)")
public HiveS3Config setS3SseType(PrestoS3SseType s3SseType)
{
this.s3SseType = s3SseType;
return this;
}

@Min(0)
public int getS3MaxClientRetries()
{
Expand Down
34 changes: 29 additions & 5 deletions presto-hive/src/main/java/com/facebook/presto/hive/PrestoS3FileSystem.java
View file
Expand Up @@ -41,6 +41,7 @@
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.S3ObjectInputStream;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.amazonaws.services.s3.model.SSEAwsKeyManagementParams;
import com.amazonaws.services.s3.transfer.Transfer;
import com.amazonaws.services.s3.transfer.TransferManager;
import com.amazonaws.services.s3.transfer.TransferManagerConfiguration;
Expand Down Expand Up @@ -137,7 +138,9 @@ public static PrestoS3FileSystemStats getFileSystemStats()
public static final String S3_PIN_CLIENT_TO_CURRENT_REGION = "presto.s3.pin-client-to-current-region";
public static final String S3_ENCRYPTION_MATERIALS_PROVIDER = "presto.s3.encryption-materials-provider";
public static final String S3_KMS_KEY_ID = "presto.s3.kms-key-id";
public static final String S3_SSE_KMS_KEY_ID = "presto.s3.sse.kms-key-id";
public static final String S3_SSE_ENABLED = "presto.s3.sse.enabled";
public static final String S3_SSE_TYPE = "presto.s3.sse.type";
public static final String S3_CREDENTIALS_PROVIDER = "presto.s3.credentials-provider";
public static final String S3_USER_AGENT_PREFIX = "presto.s3.user-agent-prefix";
public static final String S3_USER_AGENT_SUFFIX = "presto";
Expand All @@ -159,6 +162,8 @@ public static PrestoS3FileSystemStats getFileSystemStats()
private boolean useInstanceCredentials;
private boolean pinS3ClientToCurrentRegion;
private boolean sseEnabled;
private PrestoS3SseType sseType;
private String sseKmsKeyId;

@Override
public void initialize(URI uri, Configuration conf)
Expand Down Expand Up @@ -187,6 +192,8 @@ public void initialize(URI uri, Configuration conf)
this.useInstanceCredentials = conf.getBoolean(S3_USE_INSTANCE_CREDENTIALS, defaults.isS3UseInstanceCredentials());
this.pinS3ClientToCurrentRegion = conf.getBoolean(S3_PIN_CLIENT_TO_CURRENT_REGION, defaults.isPinS3ClientToCurrentRegion());
this.sseEnabled = conf.getBoolean(S3_SSE_ENABLED, defaults.isS3SseEnabled());
this.sseType = PrestoS3SseType.valueOf(conf.get(S3_SSE_TYPE, defaults.getS3SseType().name()));
this.sseKmsKeyId = conf.get(S3_SSE_KMS_KEY_ID, defaults.getS3SseKmsKeyId());
String userAgentPrefix = conf.get(S3_USER_AGENT_PREFIX, defaults.getS3UserAgentPrefix());

ClientConfiguration configuration = new ClientConfiguration()
Expand Down Expand Up @@ -349,7 +356,7 @@ public FSDataOutputStream create(Path path, FsPermission permission, boolean ove

String key = keyFromPath(qualifiedPath(path));
return new FSDataOutputStream(
new PrestoS3OutputStream(s3, transferConfig, uri.getHost(), key, tempFile, sseEnabled),
new PrestoS3OutputStream(s3, transferConfig, uri.getHost(), key, tempFile, sseEnabled, sseType, sseKmsKeyId),
statistics);
}

Expand Down Expand Up @@ -934,10 +941,12 @@ private static class PrestoS3OutputStream
private final String key;
private final File tempFile;
private final boolean sseEnabled;
private final PrestoS3SseType sseType;
private final String sseKmsKeyId;

private boolean closed;

public PrestoS3OutputStream(AmazonS3 s3, TransferManagerConfiguration config, String host, String key, File tempFile, boolean sseEnabled)
public PrestoS3OutputStream(AmazonS3 s3, TransferManagerConfiguration config, String host, String key, File tempFile, boolean sseEnabled, PrestoS3SseType sseType, String sseKmsKeyId)
throws IOException
{
super(new BufferedOutputStream(new FileOutputStream(requireNonNull(tempFile, "tempFile is null"))));
Expand All @@ -949,6 +958,8 @@ public PrestoS3OutputStream(AmazonS3 s3, TransferManagerConfiguration config, St
this.key = requireNonNull(key, "key is null");
this.tempFile = tempFile;
this.sseEnabled = sseEnabled;
this.sseType = requireNonNull(sseType, "sseType is null");
this.sseKmsKeyId = sseKmsKeyId;

log.debug("OutputStream for key '%s' using file: %s", key, tempFile);
}
Expand Down Expand Up @@ -984,10 +995,23 @@ private void uploadObject()

PutObjectRequest request = new PutObjectRequest(host, key, tempFile);
if (sseEnabled) {
ObjectMetadata metadata = new ObjectMetadata();
metadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
request.setMetadata(metadata);
switch (sseType) {
case KMS:
if (sseKmsKeyId != null) {
request.withSSEAwsKeyManagementParams(new SSEAwsKeyManagementParams(sseKmsKeyId));
}
else {
request.withSSEAwsKeyManagementParams(new SSEAwsKeyManagementParams());
}
break;
case S3:
ObjectMetadata metadata = new ObjectMetadata();
metadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
request.setMetadata(metadata);
break;
}
}

Upload upload = transferManager.upload(request);

if (log.isDebugEnabled()) {
Expand Down
20 changes: 20 additions & 0 deletions presto-hive/src/main/java/com/facebook/presto/hive/PrestoS3SseType.java
View file
@@ -0,0 +1,20 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.facebook.presto.hive;

public enum PrestoS3SseType
{
KMS,
S3;
}
6 changes: 6 additions & 0 deletions presto-hive/src/test/java/com/facebook/presto/hive/TestHiveS3Config.java
View file
Expand Up @@ -41,6 +41,8 @@ public void testDefaults()
.setS3UseInstanceCredentials(true)
.setS3SslEnabled(true)
.setS3SseEnabled(false)
.setS3SseType(PrestoS3SseType.S3)
.setS3SseKmsKeyId(null)
.setS3KmsKeyId(null)
.setS3EncryptionMaterialsProvider(null)
.setS3MaxClientRetries(3)
Expand Down Expand Up @@ -68,6 +70,8 @@ public void testExplicitPropertyMappings()
.put("hive.s3.use-instance-credentials", "false")
.put("hive.s3.ssl.enabled", "false")
.put("hive.s3.sse.enabled", "true")
.put("hive.s3.sse.type", "KMS")
.put("hive.s3.sse.kms-key-id", "KMS_KEY_ID")
.put("hive.s3.encryption-materials-provider", "EMP_CLASS")
.put("hive.s3.kms-key-id", "KEY_ID")
.put("hive.s3.max-client-retries", "9")
Expand All @@ -92,6 +96,8 @@ public void testExplicitPropertyMappings()
.setS3UseInstanceCredentials(false)
.setS3SslEnabled(false)
.setS3SseEnabled(true)
.setS3SseType(PrestoS3SseType.KMS)
.setS3SseKmsKeyId("KMS_KEY_ID")
.setS3EncryptionMaterialsProvider("EMP_CLASS")
.setS3KmsKeyId("KEY_ID")
.setS3MaxClientRetries(9)
Expand Down

0 comments on commit 0f2909d

Please sign in to comment.