Skip to content

Commit

Permalink
HADOOP-14705. Add batched interface reencryptEncryptedKeys to KMS.
Browse files Browse the repository at this point in the history
  • Loading branch information
@xiao-chen
xiao-chen committed Aug 22, 2017
1 parent 27ab5f7 commit 4ec5acc
Show file tree
Hide file tree
Showing 15 changed files with 673 additions and 198 deletions.
147 changes: 126 additions & 21 deletions .../hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
View file
Expand Up @@ -22,6 +22,8 @@
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.security.GeneralSecurityException; import java.security.GeneralSecurityException;
import java.security.SecureRandom; import java.security.SecureRandom;
import java.util.List;
import java.util.ListIterator;


import javax.crypto.Cipher; import javax.crypto.Cipher;
import javax.crypto.spec.IvParameterSpec; import javax.crypto.spec.IvParameterSpec;
Expand Down Expand Up @@ -247,6 +249,25 @@ public KeyVersion decryptEncryptedKey(
*/ */
EncryptedKeyVersion reencryptEncryptedKey(EncryptedKeyVersion ekv) EncryptedKeyVersion reencryptEncryptedKey(EncryptedKeyVersion ekv)
throws IOException, GeneralSecurityException; throws IOException, GeneralSecurityException;

/**
* Batched version of {@link #reencryptEncryptedKey(EncryptedKeyVersion)}.
* <p>
* For each encrypted key version, re-encrypts an encrypted key version,
* using its initialization vector and key material, but with the latest
* key version name of its key name. If the latest key version name in the
* provider is the same as the one encrypted the passed-in encrypted key
* version, the same encrypted key version is returned.
* <p>
* NOTE: The generated key is not stored by the <code>KeyProvider</code>
*
* @param ekvs List containing the EncryptedKeyVersion's
* @throws IOException If any EncryptedKeyVersion could not be re-encrypted
* @throws GeneralSecurityException If any EncryptedKeyVersion could not be
* re-encrypted because of a cryptographic issue.
*/
void reencryptEncryptedKeys(List<EncryptedKeyVersion> ekvs)
throws IOException, GeneralSecurityException;
} }


private static class DefaultCryptoExtension implements CryptoExtension { private static class DefaultCryptoExtension implements CryptoExtension {
Expand Down Expand Up @@ -315,7 +336,7 @@ public EncryptedKeyVersion reencryptEncryptedKey(EncryptedKeyVersion ekv)
.checkNotNull(ekNow, "KeyVersion name '%s' does not exist", ekName); .checkNotNull(ekNow, "KeyVersion name '%s' does not exist", ekName);
Preconditions.checkArgument(ekv.getEncryptedKeyVersion().getVersionName() Preconditions.checkArgument(ekv.getEncryptedKeyVersion().getVersionName()
.equals(KeyProviderCryptoExtension.EEK), .equals(KeyProviderCryptoExtension.EEK),
"encryptedKey version name must be '%s', is '%s'", "encryptedKey version name must be '%s', but found '%s'",
KeyProviderCryptoExtension.EEK, KeyProviderCryptoExtension.EEK,
ekv.getEncryptedKeyVersion().getVersionName()); ekv.getEncryptedKeyVersion().getVersionName());


Expand All @@ -336,30 +357,67 @@ public EncryptedKeyVersion reencryptEncryptedKey(EncryptedKeyVersion ekv)
} }


@Override @Override
public KeyVersion decryptEncryptedKey( public void reencryptEncryptedKeys(List<EncryptedKeyVersion> ekvs)
EncryptedKeyVersion encryptedKeyVersion) throws IOException, throws IOException, GeneralSecurityException {
GeneralSecurityException { Preconditions.checkNotNull(ekvs, "Input list is null");
// Fetch the encryption key material KeyVersion ekNow = null;
final String encryptionKeyVersionName = Decryptor decryptor = null;
encryptedKeyVersion.getEncryptionKeyVersionName(); Encryptor encryptor = null;
final KeyVersion encryptionKey = try (CryptoCodec cc = CryptoCodec.getInstance(keyProvider.getConf())) {
keyProvider.getKeyVersion(encryptionKeyVersionName); decryptor = cc.createDecryptor();
Preconditions.checkNotNull(encryptionKey, encryptor = cc.createEncryptor();
"KeyVersion name '%s' does not exist", encryptionKeyVersionName); ListIterator<EncryptedKeyVersion> iter = ekvs.listIterator();
Preconditions.checkArgument( while (iter.hasNext()) {
encryptedKeyVersion.getEncryptedKeyVersion().getVersionName() final EncryptedKeyVersion ekv = iter.next();
.equals(KeyProviderCryptoExtension.EEK), Preconditions.checkNotNull(ekv, "EncryptedKeyVersion is null");
"encryptedKey version name must be '%s', is '%s'", final String ekName = ekv.getEncryptionKeyName();
KeyProviderCryptoExtension.EEK, Preconditions.checkNotNull(ekName, "Key name is null");
encryptedKeyVersion.getEncryptedKeyVersion().getVersionName() Preconditions.checkNotNull(ekv.getEncryptedKeyVersion(),
); "EncryptedKeyVersion is null");
Preconditions.checkArgument(
ekv.getEncryptedKeyVersion().getVersionName()
.equals(KeyProviderCryptoExtension.EEK),
"encryptedKey version name must be '%s', but found '%s'",
KeyProviderCryptoExtension.EEK,
ekv.getEncryptedKeyVersion().getVersionName());

if (ekNow == null) {
ekNow = keyProvider.getCurrentKey(ekName);
Preconditions
.checkNotNull(ekNow, "Key name '%s' does not exist", ekName);
} else {
Preconditions.checkArgument(ekNow.getName().equals(ekName),
"All keys must have the same key name. Expected '%s' "
+ "but found '%s'", ekNow.getName(), ekName);
}

final String encryptionKeyVersionName =
ekv.getEncryptionKeyVersionName();
final KeyVersion encryptionKey =
keyProvider.getKeyVersion(encryptionKeyVersionName);
Preconditions.checkNotNull(encryptionKey,
"KeyVersion name '%s' does not exist", encryptionKeyVersionName);
if (encryptionKey.equals(ekNow)) {
// no-op if same key version
continue;
}

final KeyVersion ek =
decryptEncryptedKey(decryptor, encryptionKey, ekv);
iter.set(generateEncryptedKey(encryptor, ekNow, ek.getMaterial(),
ekv.getEncryptedKeyIv()));
}
}
}


private KeyVersion decryptEncryptedKey(final Decryptor decryptor,
final KeyVersion encryptionKey,
final EncryptedKeyVersion encryptedKeyVersion)
throws IOException, GeneralSecurityException {
// Encryption key IV is determined from encrypted key's IV // Encryption key IV is determined from encrypted key's IV
final byte[] encryptionIV = final byte[] encryptionIV =
EncryptedKeyVersion.deriveIV(encryptedKeyVersion.getEncryptedKeyIv()); EncryptedKeyVersion.deriveIV(encryptedKeyVersion.getEncryptedKeyIv());


CryptoCodec cc = CryptoCodec.getInstance(keyProvider.getConf());
Decryptor decryptor = cc.createDecryptor();
decryptor.init(encryptionKey.getMaterial(), encryptionIV); decryptor.init(encryptionKey.getMaterial(), encryptionIV);
final KeyVersion encryptedKV = final KeyVersion encryptedKV =
encryptedKeyVersion.getEncryptedKeyVersion(); encryptedKeyVersion.getEncryptedKeyVersion();
Expand All @@ -372,10 +430,35 @@ public KeyVersion decryptEncryptedKey(
bbOut.flip(); bbOut.flip();
byte[] decryptedKey = new byte[keyLen]; byte[] decryptedKey = new byte[keyLen];
bbOut.get(decryptedKey); bbOut.get(decryptedKey);
cc.close();
return new KeyVersion(encryptionKey.getName(), EK, decryptedKey); return new KeyVersion(encryptionKey.getName(), EK, decryptedKey);
} }


@Override
public KeyVersion decryptEncryptedKey(
EncryptedKeyVersion encryptedKeyVersion)
throws IOException, GeneralSecurityException {
// Fetch the encryption key material
final String encryptionKeyVersionName =
encryptedKeyVersion.getEncryptionKeyVersionName();
final KeyVersion encryptionKey =
keyProvider.getKeyVersion(encryptionKeyVersionName);
Preconditions
.checkNotNull(encryptionKey, "KeyVersion name '%s' does not exist",
encryptionKeyVersionName);
Preconditions.checkArgument(
encryptedKeyVersion.getEncryptedKeyVersion().getVersionName()
.equals(KeyProviderCryptoExtension.EEK),
"encryptedKey version name must be '%s', but found '%s'",
KeyProviderCryptoExtension.EEK,
encryptedKeyVersion.getEncryptedKeyVersion().getVersionName());

try (CryptoCodec cc = CryptoCodec.getInstance(keyProvider.getConf())) {
final Decryptor decryptor = cc.createDecryptor();
return decryptEncryptedKey(decryptor, encryptionKey,
encryptedKeyVersion);
}
}

@Override @Override
public void warmUpEncryptedKeys(String... keyNames) public void warmUpEncryptedKeys(String... keyNames)
throws IOException { throws IOException {
Expand Down Expand Up @@ -470,6 +553,28 @@ public EncryptedKeyVersion reencryptEncryptedKey(EncryptedKeyVersion ekv)
return getExtension().reencryptEncryptedKey(ekv); return getExtension().reencryptEncryptedKey(ekv);
} }


/**
* Batched version of {@link #reencryptEncryptedKey(EncryptedKeyVersion)}.
* <p>
* For each encrypted key version, re-encrypts an encrypted key version,
* using its initialization vector and key material, but with the latest
* key version name of its key name. If the latest key version name in the
* provider is the same as the one encrypted the passed-in encrypted key
* version, the same encrypted key version is returned.
* <p>
* NOTE: The generated key is not stored by the <code>KeyProvider</code>
*
* @param ekvs List containing the EncryptedKeyVersion's
* @return The re-encrypted EncryptedKeyVersion's, in the same order.
* @throws IOException If any EncryptedKeyVersion could not be re-encrypted
* @throws GeneralSecurityException If any EncryptedKeyVersion could not be
* re-encrypted because of a cryptographic issue.
*/
public void reencryptEncryptedKeys(List<EncryptedKeyVersion> ekvs)
throws IOException, GeneralSecurityException {
getExtension().reencryptEncryptedKeys(ekvs);
}

/** /**
* Creates a <code>KeyProviderCryptoExtension</code> using a given * Creates a <code>KeyProviderCryptoExtension</code> using a given
* {@link KeyProvider}. * {@link KeyProvider}.
Expand Down
145 changes: 54 additions & 91 deletions ...oject/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
View file
Expand Up @@ -70,7 +70,6 @@
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Date; import java.util.Date;
import java.util.HashMap; import java.util.HashMap;
import java.util.LinkedList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Queue; import java.util.Queue;
Expand All @@ -84,6 +83,13 @@
import com.google.common.base.Preconditions; import com.google.common.base.Preconditions;
import com.google.common.base.Strings; import com.google.common.base.Strings;


import static org.apache.hadoop.util.KMSUtil.checkNotEmpty;
import static org.apache.hadoop.util.KMSUtil.checkNotNull;
import static org.apache.hadoop.util.KMSUtil.parseJSONEncKeyVersion;
import static org.apache.hadoop.util.KMSUtil.parseJSONEncKeyVersions;
import static org.apache.hadoop.util.KMSUtil.parseJSONKeyVersion;
import static org.apache.hadoop.util.KMSUtil.parseJSONMetadata;

/** /**
* KMS client <code>KeyProvider</code> implementation. * KMS client <code>KeyProvider</code> implementation.
*/ */
Expand Down Expand Up @@ -219,77 +225,11 @@ public KMSEncryptedKeyVersion(String keyName, String keyVersionName,
} }
} }


@SuppressWarnings("rawtypes") private static void writeJson(Object obj, OutputStream os)
private static List<EncryptedKeyVersion> throws IOException {
parseJSONEncKeyVersions(String keyName, List valueList) {
List<EncryptedKeyVersion> ekvs = new LinkedList<EncryptedKeyVersion>();
if (!valueList.isEmpty()) {
for (Object values : valueList) {
Map valueMap = (Map) values;
ekvs.add(parseJSONEncKeyVersion(keyName, valueMap));
}
}
return ekvs;
}

private static EncryptedKeyVersion parseJSONEncKeyVersion(String keyName,
Map valueMap) {
String versionName = checkNotNull(
(String) valueMap.get(KMSRESTConstants.VERSION_NAME_FIELD),
KMSRESTConstants.VERSION_NAME_FIELD);

byte[] iv = Base64.decodeBase64(checkNotNull(
(String) valueMap.get(KMSRESTConstants.IV_FIELD),
KMSRESTConstants.IV_FIELD));

Map encValueMap = checkNotNull((Map)
valueMap.get(KMSRESTConstants.ENCRYPTED_KEY_VERSION_FIELD),
KMSRESTConstants.ENCRYPTED_KEY_VERSION_FIELD);

String encVersionName = checkNotNull((String)
encValueMap.get(KMSRESTConstants.VERSION_NAME_FIELD),
KMSRESTConstants.VERSION_NAME_FIELD);

byte[] encKeyMaterial = Base64.decodeBase64(checkNotNull((String)
encValueMap.get(KMSRESTConstants.MATERIAL_FIELD),
KMSRESTConstants.MATERIAL_FIELD));

return new KMSEncryptedKeyVersion(keyName, versionName, iv,
encVersionName, encKeyMaterial);
}

private static KeyVersion parseJSONKeyVersion(Map valueMap) {
KeyVersion keyVersion = null;
if (!valueMap.isEmpty()) {
byte[] material = (valueMap.containsKey(KMSRESTConstants.MATERIAL_FIELD))
? Base64.decodeBase64((String) valueMap.get(KMSRESTConstants.MATERIAL_FIELD))
: null;
String versionName = (String)valueMap.get(KMSRESTConstants.VERSION_NAME_FIELD);
String keyName = (String)valueMap.get(KMSRESTConstants.NAME_FIELD);
keyVersion = new KMSKeyVersion(keyName, versionName, material);
}
return keyVersion;
}

@SuppressWarnings("unchecked")
private static Metadata parseJSONMetadata(Map valueMap) {
Metadata metadata = null;
if (!valueMap.isEmpty()) {
metadata = new KMSMetadata(
(String) valueMap.get(KMSRESTConstants.CIPHER_FIELD),
(Integer) valueMap.get(KMSRESTConstants.LENGTH_FIELD),
(String) valueMap.get(KMSRESTConstants.DESCRIPTION_FIELD),
(Map<String, String>) valueMap.get(KMSRESTConstants.ATTRIBUTES_FIELD),
new Date((Long) valueMap.get(KMSRESTConstants.CREATED_FIELD)),
(Integer) valueMap.get(KMSRESTConstants.VERSIONS_FIELD));
}
return metadata;
}

private static void writeJson(Map map, OutputStream os) throws IOException {
Writer writer = new OutputStreamWriter(os, StandardCharsets.UTF_8); Writer writer = new OutputStreamWriter(os, StandardCharsets.UTF_8);
ObjectMapper jsonMapper = new ObjectMapper(); ObjectMapper jsonMapper = new ObjectMapper();
jsonMapper.writerWithDefaultPrettyPrinter().writeValue(writer, map); jsonMapper.writerWithDefaultPrettyPrinter().writeValue(writer, obj);
} }


/** /**
Expand Down Expand Up @@ -360,25 +300,6 @@ private KeyProvider createProvider(Configuration conf,
} }
} }


public static <T> T checkNotNull(T o, String name)
throws IllegalArgumentException {
if (o == null) {
throw new IllegalArgumentException("Parameter '" + name +
"' cannot be null");
}
return o;
}

public static String checkNotEmpty(String s, String name)
throws IllegalArgumentException {
checkNotNull(s, name);
if (s.isEmpty()) {
throw new IllegalArgumentException("Parameter '" + name +
"' cannot be empty");
}
return s;
}

private String kmsUrl; private String kmsUrl;
private SSLFactory sslFactory; private SSLFactory sslFactory;
private ConnectionConfigurator configurator; private ConnectionConfigurator configurator;
Expand Down Expand Up @@ -560,12 +481,12 @@ public HttpURLConnection run() throws Exception {
return conn; return conn;
} }


private <T> T call(HttpURLConnection conn, Map jsonOutput, private <T> T call(HttpURLConnection conn, Object jsonOutput,
int expectedResponse, Class<T> klass) throws IOException { int expectedResponse, Class<T> klass) throws IOException {
return call(conn, jsonOutput, expectedResponse, klass, authRetry); return call(conn, jsonOutput, expectedResponse, klass, authRetry);
} }


private <T> T call(HttpURLConnection conn, Map jsonOutput, private <T> T call(HttpURLConnection conn, Object jsonOutput,
int expectedResponse, Class<T> klass, int authRetryCount) int expectedResponse, Class<T> klass, int authRetryCount)
throws IOException { throws IOException {
T ret = null; T ret = null;
Expand Down Expand Up @@ -884,6 +805,48 @@ public EncryptedKeyVersion reencryptEncryptedKey(EncryptedKeyVersion ekv)
return parseJSONEncKeyVersion(ekv.getEncryptionKeyName(), response); return parseJSONEncKeyVersion(ekv.getEncryptionKeyName(), response);
} }


@Override
public void reencryptEncryptedKeys(List<EncryptedKeyVersion> ekvs)
throws IOException, GeneralSecurityException {
checkNotNull(ekvs, "ekvs");
if (ekvs.isEmpty()) {
return;
}
final List<Map> jsonPayload = new ArrayList<>();
String keyName = null;
for (EncryptedKeyVersion ekv : ekvs) {
checkNotNull(ekv.getEncryptionKeyName(), "keyName");
checkNotNull(ekv.getEncryptionKeyVersionName(), "versionName");
checkNotNull(ekv.getEncryptedKeyIv(), "iv");
checkNotNull(ekv.getEncryptedKeyVersion(), "encryptedKey");
Preconditions.checkArgument(ekv.getEncryptedKeyVersion().getVersionName()
.equals(KeyProviderCryptoExtension.EEK),
"encryptedKey version name must be '%s', is '%s'",
KeyProviderCryptoExtension.EEK,
ekv.getEncryptedKeyVersion().getVersionName());
if (keyName == null) {
keyName = ekv.getEncryptionKeyName();
} else {
Preconditions.checkArgument(keyName.equals(ekv.getEncryptionKeyName()),
"All EncryptedKey must have the same key name.");
}
jsonPayload.add(KMSUtil.toJSON(ekv));
}
final URL url = createURL(KMSRESTConstants.KEY_RESOURCE, keyName,
KMSRESTConstants.REENCRYPT_BATCH_SUB_RESOURCE, null);
final HttpURLConnection conn = createConnection(url, HTTP_POST);
conn.setRequestProperty(CONTENT_TYPE, APPLICATION_JSON_MIME);
final List<Map> response =
call(conn, jsonPayload, HttpURLConnection.HTTP_OK, List.class);
Preconditions.checkArgument(response.size() == ekvs.size(),
"Response size is different than input size.");
for (int i = 0; i < response.size(); ++i) {
final Map item = response.get(i);
final EncryptedKeyVersion ekv = parseJSONEncKeyVersion(keyName, item);
ekvs.set(i, ekv);
}
}

@Override @Override
public List<KeyVersion> getKeyVersions(String name) throws IOException { public List<KeyVersion> getKeyVersions(String name) throws IOException {
checkNotEmpty(name, "name"); checkNotEmpty(name, "name");
Expand Down
1 change: 1 addition & 0 deletions ...roject/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSRESTConstants.java
View file
Expand Up @@ -37,6 +37,7 @@ public class KMSRESTConstants {
public static final String EEK_SUB_RESOURCE = "_eek"; public static final String EEK_SUB_RESOURCE = "_eek";
public static final String CURRENT_VERSION_SUB_RESOURCE = "_currentversion"; public static final String CURRENT_VERSION_SUB_RESOURCE = "_currentversion";
public static final String INVALIDATECACHE_RESOURCE = "_invalidatecache"; public static final String INVALIDATECACHE_RESOURCE = "_invalidatecache";
public static final String REENCRYPT_BATCH_SUB_RESOURCE = "_reencryptbatch";


public static final String KEY = "key"; public static final String KEY = "key";
public static final String EEK_OP = "eek_op"; public static final String EEK_OP = "eek_op";
Expand Down

0 comments on commit 4ec5acc

Please sign in to comment.