From 25b7108d7befc744a27caf1abeb35c3c36d64288 Mon Sep 17 00:00:00 2001 From: Andy LoPresto Date: Wed, 2 Dec 2015 15:20:46 -0800 Subject: [PATCH 1/6] NIFI-1240: Added explicit reference to Sun Java Cryptographic Service Provider in PasswordBasedEncryptor. Removed manual seeding of SecureRandom in PasswordBasedEncryptor. --- .../standard/util/PasswordBasedEncryptor.java | 24 +++++++------------ 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/PasswordBasedEncryptor.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/PasswordBasedEncryptor.java index 419e66d5a803..405b4e71ce62 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/PasswordBasedEncryptor.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/PasswordBasedEncryptor.java @@ -16,25 +16,20 @@ */ package org.apache.nifi.processors.standard.util; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.io.StreamCallback; +import org.apache.nifi.processors.standard.EncryptContent.Encryptor; +import org.apache.nifi.stream.io.StreamUtils; + +import javax.crypto.*; +import javax.crypto.spec.PBEKeySpec; +import javax.crypto.spec.PBEParameterSpec; import java.io.EOFException; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.security.SecureRandom; -import javax.crypto.BadPaddingException; -import javax.crypto.Cipher; -import javax.crypto.IllegalBlockSizeException; -import javax.crypto.SecretKey; -import javax.crypto.SecretKeyFactory; -import javax.crypto.spec.PBEKeySpec; -import javax.crypto.spec.PBEParameterSpec; - -import org.apache.nifi.processor.exception.ProcessException; -import org.apache.nifi.processor.io.StreamCallback; -import org.apache.nifi.processors.standard.EncryptContent.Encryptor; -import org.apache.nifi.stream.io.StreamUtils; - public class PasswordBasedEncryptor implements Encryptor { private Cipher cipher; @@ -65,8 +60,7 @@ public PasswordBasedEncryptor(final String algorithm, final String providerName, public StreamCallback getEncryptionCallback() throws ProcessException { try { byte[] salt = new byte[saltSize]; - SecureRandom secureRandom = SecureRandom.getInstance(SECURE_RANDOM_ALGORITHM); - secureRandom.setSeed(System.currentTimeMillis()); + SecureRandom secureRandom = SecureRandom.getInstance(SECURE_RANDOM_ALGORITHM, "SUN"); secureRandom.nextBytes(salt); return new EncryptCallback(salt); } catch (Exception e) { From c04844c0480d208074be31ccd9dc72ec92b642df Mon Sep 17 00:00:00 2001 From: Andy LoPresto Date: Wed, 2 Dec 2015 19:05:53 -0800 Subject: [PATCH 2/6] NIFI-1242: Added enum for KeyDerivationFunction. --- .../security/util/KeyDerivationFunction.java | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/KeyDerivationFunction.java diff --git a/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/KeyDerivationFunction.java b/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/KeyDerivationFunction.java new file mode 100644 index 000000000000..844853dec0bb --- /dev/null +++ b/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/KeyDerivationFunction.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.security.util; + +import org.apache.commons.lang3.builder.ToStringBuilder; +import org.apache.commons.lang3.builder.ToStringStyle; + +/** + * Enumeration capturing essential information about the various key derivation functions that might be supported. + */ +public enum KeyDerivationFunction { + + NIFI_LEGACY("NiFi legacy KDF", "MD5 @ 1000 iterations"), + OPENSSL_EVP_BYTES_TO_KEY("OpenSSL EVP_BytesToKey", "Single iteration MD5 compatible with PKCS#5 v1.5"); + // TODO: Implement bcrypt, scrypt, and PBKDF2 + + private final String name; + private final String description; + + KeyDerivationFunction(String name, String description) { + this.name = name; + this.description = description; + } + + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + + @Override + public String toString() { + final ToStringBuilder builder = new ToStringBuilder(this); + ToStringBuilder.setDefaultStyle(ToStringStyle.SHORT_PREFIX_STYLE); + builder.append("KDF Name", name); + builder.append("Description", description); + return builder.toString(); + } +} From af35c370d3c27d5bd47c63d4b889d6cc21284193 Mon Sep 17 00:00:00 2001 From: Andy LoPresto Date: Wed, 2 Dec 2015 19:09:27 -0800 Subject: [PATCH 3/6] NIFI-1242: Added cipher init for legacy and OpenSSL KDFs. --- .../standard/util/PasswordBasedEncryptor.java | 76 +++++++++++++++++-- 1 file changed, 69 insertions(+), 7 deletions(-) diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/PasswordBasedEncryptor.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/PasswordBasedEncryptor.java index 405b4e71ce62..dc6c12b4c51f 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/PasswordBasedEncryptor.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/PasswordBasedEncryptor.java @@ -19,33 +19,51 @@ import org.apache.nifi.processor.exception.ProcessException; import org.apache.nifi.processor.io.StreamCallback; import org.apache.nifi.processors.standard.EncryptContent.Encryptor; +import org.apache.nifi.security.util.KeyDerivationFunction; import org.apache.nifi.stream.io.StreamUtils; -import javax.crypto.*; +import javax.crypto.BadPaddingException; +import javax.crypto.Cipher; +import javax.crypto.IllegalBlockSizeException; +import javax.crypto.SecretKey; +import javax.crypto.SecretKeyFactory; import javax.crypto.spec.PBEKeySpec; import javax.crypto.spec.PBEParameterSpec; import java.io.EOFException; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.charset.StandardCharsets; import java.security.SecureRandom; +import java.util.Arrays; public class PasswordBasedEncryptor implements Encryptor { private Cipher cipher; private int saltSize; private SecretKey secretKey; + private KeyDerivationFunction kdf; public static final String SECURE_RANDOM_ALGORITHM = "SHA1PRNG"; public static final int DEFAULT_SALT_SIZE = 8; + // TODO: Eventually KDF-specific values should be refactored into injectable interface impls + public static final int OPENSSL_EVP_HEADER_SIZE = 8; + public static final int OPENSSL_EVP_SALT_SIZE = 8; + public static final String OPENSSL_EVP_HEADER_MARKER = "Salted__"; - public PasswordBasedEncryptor(final String algorithm, final String providerName, final char[] password) { + public PasswordBasedEncryptor(final String algorithm, final String providerName, final char[] password, KeyDerivationFunction kdf) { super(); try { // initialize cipher this.cipher = Cipher.getInstance(algorithm, providerName); - int algorithmBlockSize = cipher.getBlockSize(); - this.saltSize = (algorithmBlockSize > 0) ? algorithmBlockSize : DEFAULT_SALT_SIZE; + this.kdf = kdf; + + if (KeyDerivationFunction.OPENSSL_EVP_BYTES_TO_KEY.equals(kdf)) { + this.saltSize = OPENSSL_EVP_SALT_SIZE; + } else { + int algorithmBlockSize = cipher.getBlockSize(); + this.saltSize = (algorithmBlockSize > 0) ? algorithmBlockSize : DEFAULT_SALT_SIZE; + } // initialize SecretKey from password PBEKeySpec pbeKeySpec = new PBEKeySpec(password); @@ -60,7 +78,7 @@ public PasswordBasedEncryptor(final String algorithm, final String providerName, public StreamCallback getEncryptionCallback() throws ProcessException { try { byte[] salt = new byte[saltSize]; - SecureRandom secureRandom = SecureRandom.getInstance(SECURE_RANDOM_ALGORITHM, "SUN"); + SecureRandom secureRandom = SecureRandom.getInstance(SECURE_RANDOM_ALGORITHM); secureRandom.nextBytes(salt); return new EncryptCallback(salt); } catch (Exception e) { @@ -78,16 +96,60 @@ private class DecryptCallback implements StreamCallback { public DecryptCallback() { } + private void initCipher(final byte[] salt) { + PBEParameterSpec saltParameterSpec; + + // TODO: Handle other KDFs + // If the KDF is OpenSSL, derive the OpenSSL PBE Parameters + if (KeyDerivationFunction.OPENSSL_EVP_BYTES_TO_KEY.equals(kdf)) { + saltParameterSpec = new PBEParameterSpec(salt, 0); + } else { + // Else use the legacy KDF + saltParameterSpec = new PBEParameterSpec(salt, 1000); + } + + try { + cipher.init(Cipher.DECRYPT_MODE, secretKey, saltParameterSpec); + } catch (final Exception e) { + throw new ProcessException(e); + } + } + @Override public void process(final InputStream in, final OutputStream out) throws IOException { - final byte[] salt = new byte[saltSize]; + byte[] salt = new byte[saltSize]; + + // The legacy default value + int kdfIterations = 1000; try { + // If the KDF is OpenSSL, try to read the salt from the input stream + if (KeyDerivationFunction.OPENSSL_EVP_BYTES_TO_KEY.equals(kdf)) { + // Set the iteration count to 0 + kdfIterations = 0; + + // Try to read the header and salt from the input + byte[] header = new byte[PasswordBasedEncryptor.OPENSSL_EVP_HEADER_SIZE]; + + // Mark the stream in case there is no salt + in.mark(OPENSSL_EVP_HEADER_SIZE + 1); + StreamUtils.fillBuffer(in, header); + + final byte[] headerMarkerBytes = OPENSSL_EVP_HEADER_MARKER.getBytes(StandardCharsets.US_ASCII); + + if (!Arrays.equals(headerMarkerBytes, header)) { + // No salt present + salt = new byte[0]; + // Reset the stream because we skipped 8 bytes of cipher text + in.reset(); + } + } + StreamUtils.fillBuffer(in, salt); } catch (final EOFException e) { throw new ProcessException("Cannot decrypt because file size is smaller than salt size", e); } - final PBEParameterSpec parameterSpec = new PBEParameterSpec(salt, 1000); + final PBEParameterSpec parameterSpec = new PBEParameterSpec(salt, kdfIterations); try { cipher.init(Cipher.DECRYPT_MODE, secretKey, parameterSpec); } catch (final Exception e) { From 940b6e370ab9cd23a4dbe7dba4bb0d01c02e8cdf Mon Sep 17 00:00:00 2001 From: Andy LoPresto Date: Wed, 2 Dec 2015 20:21:03 -0800 Subject: [PATCH 4/6] NIFI-1242: Added test resources. plain.txt: This is a plaintext message. 0s @ 19:48:36 $ openssl enc -aes-256-cbc -e -in plain.txt -out salted_raw.enc -k thisIsABadPassword -p salt=43CD63B29E1CE1F2 key=8A78D6762B279416B62E1AF50A73339C5D753FBF956092B4235323AD4147D8BE iv =ED4ED4827BD23DE74D21C8E63DA0B8AE 0s @ 19:49:24 $ xxd salted_raw.enc 0000000: 5361 6c74 6564 5f5f 43cd 63b2 9e1c e1f2 Salted__C.c..... 0000010: c972 11ac 1baa b424 7fb3 156e 4cc6 1f47 .r.....$...nL..G 0000020: 11e5 cedc e343 5203 952d 4500 f9ab 3cbe .....CR..-E...<. 0s @ 20:14:00 $ openssl enc -aes-256-cbc -e -in plain.txt -out unsalted_raw.enc -k thisIsABadPassword -p -nosalt key=711E85689CE7AFF6F410AEA43ABC5446842F685B84879B2E00F977C22B9E9A7D iv =0C90ABF8ECE84B92BAA2CD448EC760F0 0s @ 20:14:17 $ xxd unsalted_raw.enc 0000000: 70cd 2984 fdbb 0e7c c01b 7206 88b1 6b50 p.)....|..r...kP 0000010: 5eeb e4f3 4036 773b 00ce dd8e 85d8 f90a ^...@6w;........ --- .../test/resources/TestEncryptContent/plain.txt | 1 + .../resources/TestEncryptContent/salted_raw.enc | Bin 0 -> 48 bytes .../resources/TestEncryptContent/unsalted_raw.enc | Bin 0 -> 32 bytes 3 files changed, 1 insertion(+) create mode 100644 nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestEncryptContent/plain.txt create mode 100644 nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestEncryptContent/salted_raw.enc create mode 100644 nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestEncryptContent/unsalted_raw.enc diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestEncryptContent/plain.txt b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestEncryptContent/plain.txt new file mode 100644 index 000000000000..aae2266a1bb7 --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestEncryptContent/plain.txt @@ -0,0 +1 @@ +This is a plaintext message. \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestEncryptContent/salted_raw.enc b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/TestEncryptContent/salted_raw.enc new file mode 100644 index 0000000000000000000000000000000000000000..2612c702b063e66de34d88a8b3befe18ed76825b GIT binary patch literal 48 zcmV-00MGwZVQh3|WM5xH&115j9O3fGauKW>s Date: Wed, 2 Dec 2015 20:24:27 -0800 Subject: [PATCH 5/6] NIFI-1242: Implemented KDF-dependent PBE in PasswordBasedEncryptor. Added KDF property to EncryptContent processor and provided to PasswordBasedEncryptor. Added unit tests for salted and unsalted raw OpenSSL encrypted file decryption. --- .../processors/standard/EncryptContent.java | 61 ++++++---- .../standard/util/PasswordBasedEncryptor.java | 54 ++++---- .../standard/TestEncryptContent.java | 115 ++++++++++++++++-- 3 files changed, 172 insertions(+), 58 deletions(-) diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EncryptContent.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EncryptContent.java index c63edc91b8a8..2d23aeefbf50 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EncryptContent.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/EncryptContent.java @@ -16,16 +16,7 @@ */ package org.apache.nifi.processors.standard; -import java.security.Security; -import java.text.Normalizer; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.concurrent.TimeUnit; - +import org.apache.commons.lang3.StringUtils; import org.apache.nifi.annotation.behavior.EventDriven; import org.apache.nifi.annotation.behavior.InputRequirement; import org.apache.nifi.annotation.behavior.InputRequirement.Requirement; @@ -51,9 +42,20 @@ import org.apache.nifi.processors.standard.util.OpenPGPPasswordBasedEncryptor; import org.apache.nifi.processors.standard.util.PasswordBasedEncryptor; import org.apache.nifi.security.util.EncryptionMethod; +import org.apache.nifi.security.util.KeyDerivationFunction; import org.apache.nifi.util.StopWatch; import org.bouncycastle.jce.provider.BouncyCastleProvider; +import java.security.Security; +import java.text.Normalizer; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; + @EventDriven @SideEffectFree @SupportsBatching @@ -72,6 +74,14 @@ public class EncryptContent extends AbstractProcessor { .allowableValues(ENCRYPT_MODE, DECRYPT_MODE) .defaultValue(ENCRYPT_MODE) .build(); + public static final PropertyDescriptor KEY_DERIVATION_FUNCTION = new PropertyDescriptor.Builder() + .name("key-derivation-function") + .displayName("Key Derivation Function") + .description("Specifies the key derivation function to generate the key from the password (and salt)") + .required(true) + .allowableValues(KeyDerivationFunction.values()) + .defaultValue(KeyDerivationFunction.NIFI_LEGACY.name()) + .build(); public static final PropertyDescriptor ENCRYPTION_ALGORITHM = new PropertyDescriptor.Builder() .name("Encryption Algorithm") .description("The Encryption Algorithm to use") @@ -133,6 +143,7 @@ public class EncryptContent extends AbstractProcessor { protected void init(final ProcessorInitializationContext context) { final List properties = new ArrayList<>(); properties.add(MODE); + properties.add(KEY_DERIVATION_FUNCTION); properties.add(ENCRYPTION_ALGORITHM); properties.add(PASSWORD); properties.add(PUBLIC_KEYRING); @@ -171,6 +182,7 @@ protected Collection customValidate(final ValidationContext co final String method = context.getProperty(ENCRYPTION_ALGORITHM).getValue(); final String algorithm = EncryptionMethod.valueOf(method).getAlgorithm(); final String password = context.getProperty(PASSWORD).getValue(); + final String kdf = context.getProperty(KEY_DERIVATION_FUNCTION).getValue(); if (isPGPAlgorithm(algorithm)) { if (password == null) { final boolean encrypt = context.getProperty(MODE).getValue().equalsIgnoreCase(ENCRYPT_MODE); @@ -227,9 +239,15 @@ protected Collection customValidate(final ValidationContext co } } } - } else if (password == null) { - validationResults.add(new ValidationResult.Builder().subject(PASSWORD.getName()) - .explanation(PASSWORD.getDisplayName() + " is required when using algorithm " + algorithm).build()); + } else { // PBE + if (StringUtils.isEmpty(password)) { + validationResults.add(new ValidationResult.Builder().subject(PASSWORD.getName()) + .explanation(PASSWORD.getDisplayName() + " is required when using algorithm " + algorithm).build()); + } + if (StringUtils.isEmpty(kdf)) { + validationResults.add(new ValidationResult.Builder().subject(KEY_DERIVATION_FUNCTION.getName()) + .explanation(KEY_DERIVATION_FUNCTION.getDisplayName() + " is required when using algorithm " + algorithm).build()); + } } return validationResults; } @@ -247,6 +265,7 @@ public void onTrigger(final ProcessContext context, final ProcessSession session final String providerName = encryptionMethod.getProvider(); final String algorithm = encryptionMethod.getAlgorithm(); final String password = context.getProperty(PASSWORD).getValue(); + final KeyDerivationFunction kdf = KeyDerivationFunction.valueOf(context.getProperty(KEY_DERIVATION_FUNCTION).getValue()); final boolean encrypt = context.getProperty(MODE).getValue().equalsIgnoreCase(ENCRYPT_MODE); Encryptor encryptor; @@ -267,9 +286,9 @@ public void onTrigger(final ProcessContext context, final ProcessSession session final char[] passphrase = Normalizer.normalize(password, Normalizer.Form.NFC).toCharArray(); encryptor = new OpenPGPPasswordBasedEncryptor(algorithm, providerName, passphrase, filename); } - } else { + } else { // PBE final char[] passphrase = Normalizer.normalize(password, Normalizer.Form.NFC).toCharArray(); - encryptor = new PasswordBasedEncryptor(algorithm, providerName, passphrase); + encryptor = new PasswordBasedEncryptor(algorithm, providerName, passphrase, kdf); } if (encrypt) { @@ -279,7 +298,7 @@ public void onTrigger(final ProcessContext context, final ProcessSession session } } catch (final Exception e) { - logger.error("Failed to initialize {}cryption algorithm because - ", new Object[] { encrypt ? "en" : "de", e }); + logger.error("Failed to initialize {}cryption algorithm because - ", new Object[]{encrypt ? "en" : "de", e}); session.rollback(); context.yield(); return; @@ -288,20 +307,20 @@ public void onTrigger(final ProcessContext context, final ProcessSession session try { final StopWatch stopWatch = new StopWatch(true); flowFile = session.write(flowFile, callback); - logger.info("successfully {}crypted {}", new Object[] { encrypt ? "en" : "de", flowFile }); + logger.info("successfully {}crypted {}", new Object[]{encrypt ? "en" : "de", flowFile}); session.getProvenanceReporter().modifyContent(flowFile, stopWatch.getElapsed(TimeUnit.MILLISECONDS)); session.transfer(flowFile, REL_SUCCESS); } catch (final ProcessException e) { - logger.error("Cannot {}crypt {} - ", new Object[] { encrypt ? "en" : "de", flowFile, e }); + logger.error("Cannot {}crypt {} - ", new Object[]{encrypt ? "en" : "de", flowFile, e}); session.transfer(flowFile, REL_FAILURE); return; } } - public static interface Encryptor { - public StreamCallback getEncryptionCallback() throws Exception; + public interface Encryptor { + StreamCallback getEncryptionCallback() throws Exception; - public StreamCallback getDecryptionCallback() throws Exception; + StreamCallback getDecryptionCallback() throws Exception; } } \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/PasswordBasedEncryptor.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/PasswordBasedEncryptor.java index dc6c12b4c51f..e07cd34c8b09 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/PasswordBasedEncryptor.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/PasswordBasedEncryptor.java @@ -39,10 +39,13 @@ public class PasswordBasedEncryptor implements Encryptor { + private static final int LEGACY_KDF_ITERATIONS = 1000; + private Cipher cipher; private int saltSize; private SecretKey secretKey; private KeyDerivationFunction kdf; + private int iterationsCount = LEGACY_KDF_ITERATIONS; public static final String SECURE_RANDOM_ALGORITHM = "SHA1PRNG"; public static final int DEFAULT_SALT_SIZE = 8; @@ -50,6 +53,7 @@ public class PasswordBasedEncryptor implements Encryptor { public static final int OPENSSL_EVP_HEADER_SIZE = 8; public static final int OPENSSL_EVP_SALT_SIZE = 8; public static final String OPENSSL_EVP_HEADER_MARKER = "Salted__"; + public static final int OPENSSL_EVP_KDF_ITERATIONS = 0; public PasswordBasedEncryptor(final String algorithm, final String providerName, final char[] password, KeyDerivationFunction kdf) { super(); @@ -58,8 +62,9 @@ public PasswordBasedEncryptor(final String algorithm, final String providerName, this.cipher = Cipher.getInstance(algorithm, providerName); this.kdf = kdf; - if (KeyDerivationFunction.OPENSSL_EVP_BYTES_TO_KEY.equals(kdf)) { + if (isOpenSSLKDF()) { this.saltSize = OPENSSL_EVP_SALT_SIZE; + this.iterationsCount = OPENSSL_EVP_KDF_ITERATIONS; } else { int algorithmBlockSize = cipher.getBlockSize(); this.saltSize = (algorithmBlockSize > 0) ? algorithmBlockSize : DEFAULT_SALT_SIZE; @@ -91,41 +96,26 @@ public StreamCallback getDecryptionCallback() throws ProcessException { return new DecryptCallback(); } - private class DecryptCallback implements StreamCallback { - - public DecryptCallback() { - } + public int getIterationsCount() { + return iterationsCount; + } - private void initCipher(final byte[] salt) { - PBEParameterSpec saltParameterSpec; + private boolean isOpenSSLKDF() { + return KeyDerivationFunction.OPENSSL_EVP_BYTES_TO_KEY.equals(kdf); + } - // TODO: Handle other KDFs - // If the KDF is OpenSSL, derive the OpenSSL PBE Parameters - if (KeyDerivationFunction.OPENSSL_EVP_BYTES_TO_KEY.equals(kdf)) { - saltParameterSpec = new PBEParameterSpec(salt, 0); - } else { - // Else use the legacy KDF - saltParameterSpec = new PBEParameterSpec(salt, 1000); - } + private class DecryptCallback implements StreamCallback { - try { - cipher.init(Cipher.DECRYPT_MODE, secretKey, saltParameterSpec); - } catch (final Exception e) { - throw new ProcessException(e); - } + public DecryptCallback() { } - @Override public void process(final InputStream in, final OutputStream out) throws IOException { byte[] salt = new byte[saltSize]; - // The legacy default value - int kdfIterations = 1000; try { // If the KDF is OpenSSL, try to read the salt from the input stream - if (KeyDerivationFunction.OPENSSL_EVP_BYTES_TO_KEY.equals(kdf)) { - // Set the iteration count to 0 - kdfIterations = 0; + if (isOpenSSLKDF()) { + // The header and salt format is "Salted__salt x8b" in ASCII // Try to read the header and salt from the input byte[] header = new byte[PasswordBasedEncryptor.OPENSSL_EVP_HEADER_SIZE]; @@ -149,7 +139,7 @@ public void process(final InputStream in, final OutputStream out) throws IOExcep throw new ProcessException("Cannot decrypt because file size is smaller than salt size", e); } - final PBEParameterSpec parameterSpec = new PBEParameterSpec(salt, kdfIterations); + final PBEParameterSpec parameterSpec = new PBEParameterSpec(salt, getIterationsCount()); try { cipher.init(Cipher.DECRYPT_MODE, secretKey, parameterSpec); } catch (final Exception e) { @@ -171,6 +161,7 @@ public void process(final InputStream in, final OutputStream out) throws IOExcep throw new ProcessException(e); } } + } private class EncryptCallback implements StreamCallback { @@ -180,16 +171,20 @@ private class EncryptCallback implements StreamCallback { public EncryptCallback(final byte[] salt) { this.salt = salt; } - @Override public void process(final InputStream in, final OutputStream out) throws IOException { - final PBEParameterSpec parameterSpec = new PBEParameterSpec(salt, 1000); + final PBEParameterSpec parameterSpec = new PBEParameterSpec(salt, getIterationsCount()); try { cipher.init(Cipher.ENCRYPT_MODE, secretKey, parameterSpec); } catch (final Exception e) { throw new ProcessException(e); } + // If this is OpenSSL EVP, the salt must be preceded by the header + if (isOpenSSLKDF()) { + out.write(OPENSSL_EVP_HEADER_MARKER.getBytes(StandardCharsets.US_ASCII)); + } + out.write(salt); final byte[] buffer = new byte[65536]; @@ -207,5 +202,6 @@ public void process(final InputStream in, final OutputStream out) throws IOExcep throw new ProcessException(e); } } + } } diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEncryptContent.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEncryptContent.java index 8e7bc05d410a..ba108348dc70 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEncryptContent.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestEncryptContent.java @@ -16,23 +16,37 @@ */ package org.apache.nifi.processors.standard; -import java.io.File; -import java.io.IOException; -import java.nio.file.Paths; -import java.util.Collection; -import java.util.HashSet; - +import org.apache.commons.codec.binary.Hex; import org.apache.nifi.components.ValidationResult; import org.apache.nifi.security.util.EncryptionMethod; +import org.apache.nifi.security.util.KeyDerivationFunction; import org.apache.nifi.util.MockFlowFile; import org.apache.nifi.util.MockProcessContext; import org.apache.nifi.util.TestRunner; import org.apache.nifi.util.TestRunners; +import org.bouncycastle.jce.provider.BouncyCastleProvider; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Paths; +import java.security.Security; +import java.util.Collection; +import java.util.HashSet; public class TestEncryptContent { + private static final Logger logger = LoggerFactory.getLogger(TestEncryptContent.class); + + @Before + public void setUp() { + Security.addProvider(new BouncyCastleProvider()); + } + @Test public void testRoundTrip() throws IOException { final TestRunner testRunner = TestRunners.newTestRunner(new EncryptContent()); @@ -60,11 +74,96 @@ public void testRoundTrip() throws IOException { testRunner.run(); testRunner.assertAllFlowFilesTransferred(EncryptContent.REL_SUCCESS, 1); + logger.info("Successfully decrypted {}", method.name()); + flowFile = testRunner.getFlowFilesForRelationship(EncryptContent.REL_SUCCESS).get(0); flowFile.assertContentEquals(new File("src/test/resources/hello.txt")); } } + @Test + public void testShouldDecryptOpenSSLRawSalted() throws IOException { + // Arrange + final TestRunner testRunner = TestRunners.newTestRunner(new EncryptContent()); + + final String password = "thisIsABadPassword"; + final EncryptionMethod method = EncryptionMethod.MD5_256AES; + final KeyDerivationFunction kdf = KeyDerivationFunction.OPENSSL_EVP_BYTES_TO_KEY; + + testRunner.setProperty(EncryptContent.PASSWORD, password); + testRunner.setProperty(EncryptContent.KEY_DERIVATION_FUNCTION, kdf.name()); + testRunner.setProperty(EncryptContent.ENCRYPTION_ALGORITHM, method.name()); + testRunner.setProperty(EncryptContent.MODE, EncryptContent.DECRYPT_MODE); + + // Act + testRunner.enqueue(Paths.get("src/test/resources/TestEncryptContent/salted_raw.enc")); + testRunner.clearTransferState(); + testRunner.run(); + + // Assert + testRunner.assertAllFlowFilesTransferred(EncryptContent.REL_SUCCESS, 1); + testRunner.assertQueueEmpty(); + + MockFlowFile flowFile = testRunner.getFlowFilesForRelationship(EncryptContent.REL_SUCCESS).get(0); + logger.info("Decrypted contents (hex): {}", Hex.encodeHexString(flowFile.toByteArray())); + logger.info("Decrypted contents: {}", new String(flowFile.toByteArray(), "UTF-8")); + + // Assert + flowFile.assertContentEquals(new File("src/test/resources/TestEncryptContent/plain.txt")); + } + + @Test + public void testShouldDecryptOpenSSLRawUnsalted() throws IOException { + // Arrange + final TestRunner testRunner = TestRunners.newTestRunner(new EncryptContent()); + + final String password = "thisIsABadPassword"; + final EncryptionMethod method = EncryptionMethod.MD5_256AES; + final KeyDerivationFunction kdf = KeyDerivationFunction.OPENSSL_EVP_BYTES_TO_KEY; + + testRunner.setProperty(EncryptContent.PASSWORD, password); + testRunner.setProperty(EncryptContent.KEY_DERIVATION_FUNCTION, kdf.name()); + testRunner.setProperty(EncryptContent.ENCRYPTION_ALGORITHM, method.name()); + testRunner.setProperty(EncryptContent.MODE, EncryptContent.DECRYPT_MODE); + + // Act + testRunner.enqueue(Paths.get("src/test/resources/TestEncryptContent/unsalted_raw.enc")); + testRunner.clearTransferState(); + testRunner.run(); + + // Assert + testRunner.assertAllFlowFilesTransferred(EncryptContent.REL_SUCCESS, 1); + testRunner.assertQueueEmpty(); + + MockFlowFile flowFile = testRunner.getFlowFilesForRelationship(EncryptContent.REL_SUCCESS).get(0); + logger.info("Decrypted contents (hex): {}", Hex.encodeHexString(flowFile.toByteArray())); + logger.info("Decrypted contents: {}", new String(flowFile.toByteArray(), "UTF-8")); + + // Assert + flowFile.assertContentEquals(new File("src/test/resources/TestEncryptContent/plain.txt")); + } + + @Test + public void testDecryptShouldDefaultToLegacyKDF() throws IOException { + // Arrange + final TestRunner testRunner = TestRunners.newTestRunner(new EncryptContent()); + + final String password = "thisIsABadPassword"; + final EncryptionMethod method = EncryptionMethod.MD5_256AES; + + testRunner.setProperty(EncryptContent.PASSWORD, password); + testRunner.setProperty(EncryptContent.ENCRYPTION_ALGORITHM, method.name()); + testRunner.setProperty(EncryptContent.MODE, EncryptContent.DECRYPT_MODE); + + // Don't set the KDF property + + // Act + testRunner.run(); + + // Assert + assert testRunner.getProcessor().getPropertyDescriptor(EncryptContent.KEY_DERIVATION_FUNCTION.getName()).getDefaultValue().equals(KeyDerivationFunction.NIFI_LEGACY.name()); + } + @Test public void testDecryptSmallerThanSaltSize() { final TestRunner runner = TestRunners.newTestRunner(EncryptContent.class); @@ -143,8 +242,8 @@ public void testValidation() { for (final ValidationResult vr : results) { Assert.assertTrue(vr.toString().contains( " decryption without a " + EncryptContent.PASSWORD.getDisplayName() + " requires both " - + EncryptContent.PRIVATE_KEYRING.getDisplayName() + " and " - + EncryptContent.PRIVATE_KEYRING_PASSPHRASE.getDisplayName())); + + EncryptContent.PRIVATE_KEYRING.getDisplayName() + " and " + + EncryptContent.PRIVATE_KEYRING_PASSPHRASE.getDisplayName())); } From ea0cb2f778651765128e1c8f2dff2d6c95af380f Mon Sep 17 00:00:00 2001 From: Andy LoPresto Date: Thu, 3 Dec 2015 11:48:46 -0800 Subject: [PATCH 6/6] Merged from upstream/master. --- .../processors/avro/ConvertAvroToJSON.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/nifi-nar-bundles/nifi-avro-bundle/nifi-avro-processors/src/main/java/org/apache/nifi/processors/avro/ConvertAvroToJSON.java b/nifi-nar-bundles/nifi-avro-bundle/nifi-avro-processors/src/main/java/org/apache/nifi/processors/avro/ConvertAvroToJSON.java index d9fa4ffca293..521a00b25f49 100644 --- a/nifi-nar-bundles/nifi-avro-bundle/nifi-avro-processors/src/main/java/org/apache/nifi/processors/avro/ConvertAvroToJSON.java +++ b/nifi-nar-bundles/nifi-avro-bundle/nifi-avro-processors/src/main/java/org/apache/nifi/processors/avro/ConvertAvroToJSON.java @@ -16,18 +16,6 @@ */ package org.apache.nifi.processors.avro; -import java.io.BufferedInputStream; -import java.io.BufferedOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - import org.apache.avro.file.DataFileStream; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericDatumReader; @@ -50,6 +38,18 @@ import org.apache.nifi.processor.exception.ProcessException; import org.apache.nifi.processor.io.StreamCallback; +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + @SideEffectFree @SupportsBatching @Tags({"avro", "convert", "json"})