From 2404567af3e3f98ef1f3f4eda59453fe3a3e3c22 Mon Sep 17 00:00:00 2001 From: Kevin Doran Date: Tue, 5 Dec 2017 14:44:24 -0500 Subject: [PATCH 1/2] NIFIREG-61 Add support for encrypted config files Allows sensitive property values to be encrypted in the following configuration files: - nifi-registry.properties - identity-providers.xml - authorizers.xml A master decryption key can be configured that allows decrypting protected properties at runtime, specifically: - Adds CryptoKeyProvider interface for injecting key into application - Provides implementation that is backed by bootstrap.conf - Provides implementation that keeps key in memory - Provides mechanism for removing CryptoKeyProvider from scope after Application Context is done loading --- .../IdentityProviderFactory.java | 38 +- .../authorization/AuthorizerFactory.java | 36 +- ...ensitivePropertyProviderConfiguration.java | 67 ++ .../src/main/xsd/authorizers.xsd | 1 + .../nifi/registry/jetty/JettyServer.java | 6 +- nifi-registry-properties/pom.xml | 42 + .../AESSensitivePropertyProvider.java | 265 +++++++ .../AESSensitivePropertyProviderFactory.java | 54 ++ ...eSensitivePropertyProtectionException.java | 129 +++ .../NiFiRegistryPropertiesLoader.java | 148 ++++ .../ProtectedNiFiRegistryProperties.java | 528 +++++++++++++ .../SensitivePropertyProtectionException.java | 89 +++ .../properties/SensitivePropertyProvider.java | 52 ++ .../SensitivePropertyProviderFactory.java | 23 + .../BootstrapFileCryptoKeyProvider.java | 81 ++ .../security/crypto/CryptoKeyLoader.java | 139 ++++ .../security/crypto/CryptoKeyProvider.java | 68 ++ .../crypto/MissingCryptoKeyException.java | 47 ++ .../crypto/VolatileCryptoKeyProvider.java | 69 ++ ...ensitivePropertyProviderFactoryTest.groovy | 81 ++ .../AESSensitivePropertyProviderTest.groovy | 471 +++++++++++ .../NiFiRegistryPropertiesGroovyTest.groovy | 121 +++ ...iRegistryPropertiesLoaderGroovyTest.groovy | 264 +++++++ .../ProtectedNiFiPropertiesGroovyTest.groovy | 739 ++++++++++++++++++ .../crypto/CryptoKeyLoaderGroovyTest.groovy | 121 +++ .../src/test/resources/conf/bootstrap.conf | 60 ++ ...bootstrap.unreadable_file_permissions.conf | 22 + .../conf/bootstrap.with_missing_key.conf | 60 ++ .../conf/bootstrap.with_missing_key_line.conf | 60 ++ .../resources/conf/nifi-registry.properties | 45 ++ ....with_additional_sensitive_keys.properties | 55 ++ ...e_props_fully_protected_aes_128.properties | 43 + ...nsitive_props_protected_aes_128.properties | 43 + ...rops_protected_aes_128_password.properties | 43 + ...nsitive_props_protected_aes_256.properties | 43 + ...rotected_aes_multiple_malformed.properties | 43 + ..._protected_aes_single_malformed.properties | 43 + ...nsitive_props_protected_unknown.properties | 43 + ...ith_sensitive_props_unprotected.properties | 41 + ...ve_props_unprotected_extra_line.properties | 42 + .../src/main/resources/conf/bootstrap.conf | 5 +- .../resources/conf/nifi-registry.properties | 3 + .../apache/nifi/registry/NiFiRegistry.java | 141 ++-- nifi-registry-web-api/pom.xml | 4 +- .../registry/NiFiRegistryApiApplication.java | 1 + .../NiFiRegistryMasterKeyFactory.java | 115 +++ .../nifi/registry/web/api/SecureLdapIT.java | 2 +- 47 files changed, 4551 insertions(+), 85 deletions(-) create mode 100644 nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/crypto/SensitivePropertyProviderConfiguration.java create mode 100644 nifi-registry-properties/src/main/java/org/apache/nifi/registry/properties/AESSensitivePropertyProvider.java create mode 100644 nifi-registry-properties/src/main/java/org/apache/nifi/registry/properties/AESSensitivePropertyProviderFactory.java create mode 100644 nifi-registry-properties/src/main/java/org/apache/nifi/registry/properties/MultipleSensitivePropertyProtectionException.java create mode 100644 nifi-registry-properties/src/main/java/org/apache/nifi/registry/properties/NiFiRegistryPropertiesLoader.java create mode 100644 nifi-registry-properties/src/main/java/org/apache/nifi/registry/properties/ProtectedNiFiRegistryProperties.java create mode 100644 nifi-registry-properties/src/main/java/org/apache/nifi/registry/properties/SensitivePropertyProtectionException.java create mode 100644 nifi-registry-properties/src/main/java/org/apache/nifi/registry/properties/SensitivePropertyProvider.java create mode 100644 nifi-registry-properties/src/main/java/org/apache/nifi/registry/properties/SensitivePropertyProviderFactory.java create mode 100644 nifi-registry-properties/src/main/java/org/apache/nifi/registry/security/crypto/BootstrapFileCryptoKeyProvider.java create mode 100644 nifi-registry-properties/src/main/java/org/apache/nifi/registry/security/crypto/CryptoKeyLoader.java create mode 100644 nifi-registry-properties/src/main/java/org/apache/nifi/registry/security/crypto/CryptoKeyProvider.java create mode 100644 nifi-registry-properties/src/main/java/org/apache/nifi/registry/security/crypto/MissingCryptoKeyException.java create mode 100644 nifi-registry-properties/src/main/java/org/apache/nifi/registry/security/crypto/VolatileCryptoKeyProvider.java create mode 100644 nifi-registry-properties/src/test/groovy/org/apache/nifi/registry/properties/AESSensitivePropertyProviderFactoryTest.groovy create mode 100644 nifi-registry-properties/src/test/groovy/org/apache/nifi/registry/properties/AESSensitivePropertyProviderTest.groovy create mode 100644 nifi-registry-properties/src/test/groovy/org/apache/nifi/registry/properties/NiFiRegistryPropertiesGroovyTest.groovy create mode 100644 nifi-registry-properties/src/test/groovy/org/apache/nifi/registry/properties/NiFiRegistryPropertiesLoaderGroovyTest.groovy create mode 100644 nifi-registry-properties/src/test/groovy/org/apache/nifi/registry/properties/ProtectedNiFiPropertiesGroovyTest.groovy create mode 100644 nifi-registry-properties/src/test/groovy/org/apache/nifi/security/crypto/CryptoKeyLoaderGroovyTest.groovy create mode 100644 nifi-registry-properties/src/test/resources/conf/bootstrap.conf create mode 100644 nifi-registry-properties/src/test/resources/conf/bootstrap.unreadable_file_permissions.conf create mode 100644 nifi-registry-properties/src/test/resources/conf/bootstrap.with_missing_key.conf create mode 100644 nifi-registry-properties/src/test/resources/conf/bootstrap.with_missing_key_line.conf create mode 100644 nifi-registry-properties/src/test/resources/conf/nifi-registry.properties create mode 100644 nifi-registry-properties/src/test/resources/conf/nifi-registry.with_additional_sensitive_keys.properties create mode 100644 nifi-registry-properties/src/test/resources/conf/nifi-registry.with_sensitive_props_fully_protected_aes_128.properties create mode 100644 nifi-registry-properties/src/test/resources/conf/nifi-registry.with_sensitive_props_protected_aes_128.properties create mode 100644 nifi-registry-properties/src/test/resources/conf/nifi-registry.with_sensitive_props_protected_aes_128_password.properties create mode 100644 nifi-registry-properties/src/test/resources/conf/nifi-registry.with_sensitive_props_protected_aes_256.properties create mode 100644 nifi-registry-properties/src/test/resources/conf/nifi-registry.with_sensitive_props_protected_aes_multiple_malformed.properties create mode 100644 nifi-registry-properties/src/test/resources/conf/nifi-registry.with_sensitive_props_protected_aes_single_malformed.properties create mode 100644 nifi-registry-properties/src/test/resources/conf/nifi-registry.with_sensitive_props_protected_unknown.properties create mode 100644 nifi-registry-properties/src/test/resources/conf/nifi-registry.with_sensitive_props_unprotected.properties create mode 100644 nifi-registry-properties/src/test/resources/conf/nifi-registry.with_sensitive_props_unprotected_extra_line.properties create mode 100644 nifi-registry-web-api/src/main/java/org/apache/nifi/registry/web/security/NiFiRegistryMasterKeyFactory.java diff --git a/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authentication/IdentityProviderFactory.java b/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authentication/IdentityProviderFactory.java index 720bd9cac..49bc1b5a0 100644 --- a/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authentication/IdentityProviderFactory.java +++ b/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authentication/IdentityProviderFactory.java @@ -19,6 +19,8 @@ import org.apache.commons.lang3.StringUtils; import org.apache.nifi.registry.extension.ExtensionManager; import org.apache.nifi.registry.properties.NiFiRegistryProperties; +import org.apache.nifi.registry.properties.SensitivePropertyProtectionException; +import org.apache.nifi.registry.properties.SensitivePropertyProvider; import org.apache.nifi.registry.security.authentication.annotation.IdentityProviderContext; import org.apache.nifi.registry.security.authentication.generated.IdentityProviders; import org.apache.nifi.registry.security.authentication.generated.Property; @@ -31,6 +33,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Primary; +import org.springframework.lang.Nullable; import org.xml.sax.SAXException; import javax.xml.XMLConstants; @@ -68,13 +71,18 @@ private static JAXBContext initializeJaxbContext() { private NiFiRegistryProperties properties; private ExtensionManager extensionManager; + private SensitivePropertyProvider sensitivePropertyProvider; private IdentityProvider identityProvider; private final Map identityProviders = new HashMap<>(); @Autowired - public IdentityProviderFactory(final NiFiRegistryProperties properties, final ExtensionManager extensionManager) { + public IdentityProviderFactory( + final NiFiRegistryProperties properties, + final ExtensionManager extensionManager, + @Nullable final SensitivePropertyProvider sensitivePropertyProvider) { this.properties = properties; this.extensionManager = extensionManager; + this.sensitivePropertyProvider = sensitivePropertyProvider; if (this.properties == null) { throw new IllegalStateException("NiFiRegistryProperties cannot be null"); @@ -90,9 +98,8 @@ public IdentityProvider getIdentityProvider(String identifier) { return identityProviders.get(identifier); } - @Primary @Bean -// @Bean("LoginIdentityProvider") + @Primary public IdentityProvider getIdentityProvider() throws Exception { if (identityProvider == null) { // look up the login identity provider to use @@ -186,7 +193,12 @@ private IdentityProviderConfigurationContext loadLoginIdentityProviderConfigurat final Map providerProperties = new HashMap<>(); for (final Property property : provider.getProperty()) { - providerProperties.put(property.getName(), property.getValue()); + if (!StringUtils.isBlank(property.getEncryption())) { + String decryptedValue = decryptValue(property.getValue(), property.getEncryption()); + providerProperties.put(property.getName(), decryptedValue); + } else { + providerProperties.put(property.getName(), property.getValue()); + } } return new StandardIdentityProviderConfigurationContext(provider.getIdentifier(), this, providerProperties); @@ -258,4 +270,22 @@ private void performFieldInjection(final IdentityProvider instance, final Class } } + private String decryptValue(String cipherText, String encryptionScheme) throws SensitivePropertyProtectionException { + if (sensitivePropertyProvider == null) { + throw new SensitivePropertyProtectionException("Sensitive Property Provider dependency was never wired, so protected" + + "properties cannot be decrypted. This usually indicates that a master key for this NiFi Registry was not " + + "detected and configured during the bootstrap startup sequence. Contact the system administrator."); + } + + if (!sensitivePropertyProvider.getIdentifierKey().equalsIgnoreCase(encryptionScheme)) { + throw new SensitivePropertyProtectionException("Identity Provider configuration XML was protected using " + + encryptionScheme + + ", but the configured Sensitive Property Provider supports " + + sensitivePropertyProvider.getIdentifierKey() + + ". Cannot configure this Identity Provider due to failing to decrypt protected configuration properties."); + } + + return sensitivePropertyProvider.unprotect(cipherText); + } + } diff --git a/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/AuthorizerFactory.java b/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/AuthorizerFactory.java index f94889f17..187ab971c 100644 --- a/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/AuthorizerFactory.java +++ b/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authorization/AuthorizerFactory.java @@ -19,6 +19,8 @@ import org.apache.commons.lang3.StringUtils; import org.apache.nifi.registry.extension.ExtensionManager; import org.apache.nifi.registry.properties.NiFiRegistryProperties; +import org.apache.nifi.registry.properties.SensitivePropertyProtectionException; +import org.apache.nifi.registry.properties.SensitivePropertyProvider; import org.apache.nifi.registry.provider.StandardProviderFactory; import org.apache.nifi.registry.security.authorization.annotation.AuthorizerContext; import org.apache.nifi.registry.security.authorization.exception.AuthorizationAccessException; @@ -34,6 +36,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.lang.Nullable; import org.xml.sax.SAXException; import javax.xml.XMLConstants; @@ -80,6 +83,7 @@ private static JAXBContext initializeJaxbContext() { private final NiFiRegistryProperties properties; private final ExtensionManager extensionManager; + private final SensitivePropertyProvider sensitivePropertyProvider; private Authorizer authorizer; private final Map userGroupProviders = new HashMap<>(); @@ -87,9 +91,14 @@ private static JAXBContext initializeJaxbContext() { private final Map authorizers = new HashMap<>(); @Autowired - public AuthorizerFactory(final NiFiRegistryProperties properties, final ExtensionManager extensionManager) { + public AuthorizerFactory( + final NiFiRegistryProperties properties, + final ExtensionManager extensionManager, + @Nullable final SensitivePropertyProvider sensitivePropertyProvider) { + this.properties = properties; this.extensionManager = extensionManager; + this.sensitivePropertyProvider = sensitivePropertyProvider; if (this.properties == null) { throw new IllegalStateException("NiFiRegistryProperties cannot be null"); @@ -233,7 +242,12 @@ private AuthorizerConfigurationContext loadAuthorizerConfiguration(final String final Map authorizerProperties = new HashMap<>(); for (final Prop property : properties) { - authorizerProperties.put(property.getName(), property.getValue()); + if (!StringUtils.isBlank(property.getEncryption())) { + String decryptedValue = decryptValue(property.getValue(), property.getEncryption()); + authorizerProperties.put(property.getName(), decryptedValue); + } else { + authorizerProperties.put(property.getName(), property.getValue()); + } } return new StandardAuthorizerConfigurationContext(identifier, authorizerProperties); } @@ -387,6 +401,24 @@ private void performFieldInjection(final Object instance, final Class authorizer } } + private String decryptValue(String cipherText, String encryptionScheme) throws SensitivePropertyProtectionException { + if (sensitivePropertyProvider == null) { + throw new SensitivePropertyProtectionException("Sensitive Property Provider dependency was never wired, so protected" + + "properties cannot be decrypted. This usually indicates that a master key for this NiFi Registry was not " + + "detected and configured during the bootstrap startup sequence. Contact the system administrator."); + } + + if (!sensitivePropertyProvider.getIdentifierKey().equalsIgnoreCase(encryptionScheme)) { + throw new SensitivePropertyProtectionException("Identity Provider configuration XML was protected using " + + encryptionScheme + + ", but the configured Sensitive Property Provider supports " + + sensitivePropertyProvider.getIdentifierKey() + + ". Cannot configure this Identity Provider due to failing to decrypt protected configuration properties."); + } + + return sensitivePropertyProvider.unprotect(cipherText); + } + /** * @return a default Authorizer to use when running unsecurely with no authorizer configured diff --git a/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/crypto/SensitivePropertyProviderConfiguration.java b/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/crypto/SensitivePropertyProviderConfiguration.java new file mode 100644 index 000000000..78594921b --- /dev/null +++ b/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/crypto/SensitivePropertyProviderConfiguration.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.registry.security.crypto; + +import org.apache.nifi.registry.properties.AESSensitivePropertyProvider; +import org.apache.nifi.registry.properties.SensitivePropertyProtectionException; +import org.apache.nifi.registry.properties.SensitivePropertyProvider; +import org.apache.nifi.registry.properties.SensitivePropertyProviderFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import javax.crypto.NoSuchPaddingException; +import java.security.NoSuchAlgorithmException; +import java.security.NoSuchProviderException; + +@Configuration +public class SensitivePropertyProviderConfiguration implements SensitivePropertyProviderFactory { + + private static final Logger logger = LoggerFactory.getLogger(SensitivePropertyProviderConfiguration.class); + + @Autowired(required = false) + private CryptoKeyProvider masterKeyProvider; + + /** + * @return a SensitivePropertyProvider initialized with the master key if present, + * or null if the master key is not present. + */ + @Bean + @Override + public SensitivePropertyProvider getProvider() { + if (masterKeyProvider == null || masterKeyProvider.isEmpty()) { + // This NiFi Registry was not configured with a master key, so the assumption is + // the optional Spring bean normally provided by this method will never be needed + return null; + } + + try { + // Note, this bean is intentionally NOT a singleton because we want the + // returned provider, which has a copy of the sensitive master key material + // to be reaped when it goes out of scope in order to decrease the time + // key material is held in memory. + String key = masterKeyProvider.getKey(); + return new AESSensitivePropertyProvider(masterKeyProvider.getKey()); + } catch (MissingCryptoKeyException | NoSuchAlgorithmException | NoSuchProviderException | NoSuchPaddingException e) { + logger.warn("Error creating AES Sensitive Property Provider", e); + throw new SensitivePropertyProtectionException("Error creating AES Sensitive Property Provider", e); + } + } + +} diff --git a/nifi-registry-framework/src/main/xsd/authorizers.xsd b/nifi-registry-framework/src/main/xsd/authorizers.xsd index 278ff0997..ed2a29366 100644 --- a/nifi-registry-framework/src/main/xsd/authorizers.xsd +++ b/nifi-registry-framework/src/main/xsd/authorizers.xsd @@ -50,6 +50,7 @@ + diff --git a/nifi-registry-jetty/src/main/java/org/apache/nifi/registry/jetty/JettyServer.java b/nifi-registry-jetty/src/main/java/org/apache/nifi/registry/jetty/JettyServer.java index 459a61b76..c63d59053 100644 --- a/nifi-registry-jetty/src/main/java/org/apache/nifi/registry/jetty/JettyServer.java +++ b/nifi-registry-jetty/src/main/java/org/apache/nifi/registry/jetty/JettyServer.java @@ -17,6 +17,7 @@ package org.apache.nifi.registry.jetty; import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.registry.security.crypto.CryptoKeyProvider; import org.apache.nifi.registry.properties.NiFiRegistryProperties; import org.eclipse.jetty.annotations.AnnotationConfiguration; import org.eclipse.jetty.server.Connector; @@ -72,17 +73,19 @@ public boolean accept(File pathname) { }; private final NiFiRegistryProperties properties; + private final CryptoKeyProvider masterKeyProvider; private final Server server; private WebAppContext webUiContext; private WebAppContext webApiContext; private WebAppContext webDocsContext; - public JettyServer(final NiFiRegistryProperties properties) { + public JettyServer(final NiFiRegistryProperties properties, final CryptoKeyProvider cryptoKeyProvider) { final QueuedThreadPool threadPool = new QueuedThreadPool(properties.getWebThreads()); threadPool.setName("NiFi Registry Web Server"); this.properties = properties; + this.masterKeyProvider = cryptoKeyProvider; this.server = new Server(threadPool); // enable the annotation based configuration to ensure the jsp container is initialized properly @@ -236,6 +239,7 @@ private void loadWars() throws IOException { webApiContext = loadWar(webApiWar, "/nifi-registry-api"); webApiContext.setAttribute("nifi-registry.properties", properties); + webApiContext.setAttribute("nifi-registry.key", masterKeyProvider); // there is an issue scanning the asm repackaged jar so narrow down what we are scanning webApiContext.setAttribute("org.eclipse.jetty.server.webapp.WebInfIncludeJarPattern", ".*/spring-[^/]*\\.jar$"); diff --git a/nifi-registry-properties/pom.xml b/nifi-registry-properties/pom.xml index 24c9aa3be..7e91d9355 100644 --- a/nifi-registry-properties/pom.xml +++ b/nifi-registry-properties/pom.xml @@ -22,10 +22,52 @@ nifi-registry-properties jar + + + + + org.codehaus.gmavenplus + gmavenplus-plugin + 1.5 + + + + addTestSources + testCompile + + + + + + + org.apache.commons commons-lang3 + + org.bouncycastle + bcprov-jdk15on + 1.55 + + + org.codehaus.groovy + groovy-all + 2.4.12 + test + + + cglib + cglib-nodep + 2.2.2 + test + + + org.slf4j + slf4j-simple + 1.7.12 + test + \ No newline at end of file diff --git a/nifi-registry-properties/src/main/java/org/apache/nifi/registry/properties/AESSensitivePropertyProvider.java b/nifi-registry-properties/src/main/java/org/apache/nifi/registry/properties/AESSensitivePropertyProvider.java new file mode 100644 index 000000000..b7d1d2e23 --- /dev/null +++ b/nifi-registry-properties/src/main/java/org/apache/nifi/registry/properties/AESSensitivePropertyProvider.java @@ -0,0 +1,265 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.registry.properties; + +import org.apache.commons.lang3.StringUtils; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.bouncycastle.util.encoders.Base64; +import org.bouncycastle.util.encoders.DecoderException; +import org.bouncycastle.util.encoders.EncoderException; +import org.bouncycastle.util.encoders.Hex; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.crypto.BadPaddingException; +import javax.crypto.Cipher; +import javax.crypto.IllegalBlockSizeException; +import javax.crypto.NoSuchPaddingException; +import javax.crypto.SecretKey; +import javax.crypto.spec.IvParameterSpec; +import javax.crypto.spec.SecretKeySpec; +import java.nio.charset.StandardCharsets; +import java.security.InvalidAlgorithmParameterException; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.security.NoSuchProviderException; +import java.security.SecureRandom; +import java.security.Security; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +public class AESSensitivePropertyProvider implements SensitivePropertyProvider { + private static final Logger logger = LoggerFactory.getLogger(AESSensitivePropertyProvider.class); + + private static final String IMPLEMENTATION_NAME = "AES Sensitive Property Provider"; + private static final String IMPLEMENTATION_KEY = "aes/gcm/"; + private static final String ALGORITHM = "AES/GCM/NoPadding"; + private static final String PROVIDER = "BC"; + private static final String DELIMITER = "||"; // "|" is not a valid Base64 character, so ensured not to be present in cipher text + private static final int IV_LENGTH = 12; + private static final int MIN_CIPHER_TEXT_LENGTH = IV_LENGTH * 4 / 3 + DELIMITER.length() + 1; + + private Cipher cipher; + private final SecretKey key; + + public AESSensitivePropertyProvider(String keyHex) throws NoSuchPaddingException, NoSuchAlgorithmException, NoSuchProviderException { + byte[] key = validateKey(keyHex); + + try { + Security.addProvider(new BouncyCastleProvider()); + cipher = Cipher.getInstance(ALGORITHM, PROVIDER); + // Only store the key if the cipher was initialized successfully + this.key = new SecretKeySpec(key, "AES"); + } catch (NoSuchAlgorithmException | NoSuchProviderException | NoSuchPaddingException e) { + logger.error("Encountered an error initializing the {}: {}", IMPLEMENTATION_NAME, e.getMessage()); + throw new SensitivePropertyProtectionException("Error initializing the protection cipher", e); + } + } + + private byte[] validateKey(String keyHex) { + if (keyHex == null || StringUtils.isBlank(keyHex)) { + throw new SensitivePropertyProtectionException("The key cannot be empty"); + } + keyHex = formatHexKey(keyHex); + if (!isHexKeyValid(keyHex)) { + throw new SensitivePropertyProtectionException("The key must be a valid hexadecimal key"); + } + byte[] key = Hex.decode(keyHex); + final List validKeyLengths = getValidKeyLengths(); + if (!validKeyLengths.contains(key.length * 8)) { + List validKeyLengthsAsStrings = validKeyLengths.stream().map(i -> Integer.toString(i)).collect(Collectors.toList()); + throw new SensitivePropertyProtectionException("The key (" + key.length * 8 + " bits) must be a valid length: " + StringUtils.join(validKeyLengthsAsStrings, ", ")); + } + return key; + } + + public AESSensitivePropertyProvider(byte[] key) throws NoSuchPaddingException, NoSuchAlgorithmException, NoSuchProviderException { + this(key == null ? "" : Hex.toHexString(key)); + } + + private static String formatHexKey(String input) { + if (input == null || StringUtils.isBlank(input)) { + return ""; + } + return input.replaceAll("[^0-9a-fA-F]", "").toLowerCase(); + } + + private static boolean isHexKeyValid(String key) { + if (key == null || StringUtils.isBlank(key)) { + return false; + } + // Key length is in "nibbles" (i.e. one hex char = 4 bits) + return getValidKeyLengths().contains(key.length() * 4) && key.matches("^[0-9a-fA-F]*$"); + } + + private static List getValidKeyLengths() { + List validLengths = new ArrayList<>(); + validLengths.add(128); + + try { + if (Cipher.getMaxAllowedKeyLength("AES") > 128) { + validLengths.add(192); + validLengths.add(256); + } else { + logger.warn("JCE Unlimited Strength Cryptography Jurisdiction policies are not available, so the max key length is 128 bits"); + } + } catch (NoSuchAlgorithmException e) { + logger.warn("Encountered an error determining the max key length", e); + } + + return validLengths; + } + + /** + * Returns the name of the underlying implementation. + * + * @return the name of this sensitive property provider + */ + @Override + public String getName() { + return IMPLEMENTATION_NAME; + } + + /** + * Returns the key used to identify the provider implementation in {@code nifi.properties}. + * + * @return the key to persist in the sibling property + */ + @Override + public String getIdentifierKey() { + return IMPLEMENTATION_KEY + getKeySize(Hex.toHexString(key.getEncoded())); + } + + private int getKeySize(String key) { + if (StringUtils.isBlank(key)) { + return 0; + } else { + // A key in hexadecimal format has one char per nibble (4 bits) + return formatHexKey(key).length() * 4; + } + } + + /** + * Returns the encrypted cipher text. + * + * @param unprotectedValue the sensitive value + * @return the value to persist in the {@code nifi.properties} file + * @throws SensitivePropertyProtectionException if there is an exception encrypting the value + */ + @Override + public String protect(String unprotectedValue) throws SensitivePropertyProtectionException { + if (unprotectedValue == null || unprotectedValue.trim().length() == 0) { + throw new IllegalArgumentException("Cannot encrypt an empty value"); + } + + // Generate IV + byte[] iv = generateIV(); + if (iv.length < IV_LENGTH) { + throw new IllegalArgumentException("The IV (" + iv.length + " bytes) must be at least " + IV_LENGTH + " bytes"); + } + + try { + // Initialize cipher for encryption + cipher.init(Cipher.ENCRYPT_MODE, this.key, new IvParameterSpec(iv)); + + byte[] plainBytes = unprotectedValue.getBytes(StandardCharsets.UTF_8); + byte[] cipherBytes = cipher.doFinal(plainBytes); + logger.info(getName() + " encrypted a sensitive value successfully"); + return base64Encode(iv) + DELIMITER + base64Encode(cipherBytes); + // return Base64.toBase64String(iv) + DELIMITER + Base64.toBase64String(cipherBytes); + } catch (BadPaddingException | IllegalBlockSizeException | EncoderException | InvalidAlgorithmParameterException | InvalidKeyException e) { + final String msg = "Error encrypting a protected value"; + logger.error(msg, e); + throw new SensitivePropertyProtectionException(msg, e); + } + } + + private String base64Encode(byte[] input) { + return Base64.toBase64String(input).replaceAll("=", ""); + } + + /** + * Generates a new random IV of 12 bytes using {@link SecureRandom}. + * + * @return the IV + */ + private byte[] generateIV() { + byte[] iv = new byte[IV_LENGTH]; + new SecureRandom().nextBytes(iv); + return iv; + } + + /** + * Returns the decrypted plaintext. + * + * @param protectedValue the cipher text read from the {@code nifi.properties} file + * @return the raw value to be used by the application + * @throws SensitivePropertyProtectionException if there is an error decrypting the cipher text + */ + @Override + public String unprotect(String protectedValue) throws SensitivePropertyProtectionException { + if (protectedValue == null || protectedValue.trim().length() < MIN_CIPHER_TEXT_LENGTH) { + throw new IllegalArgumentException("Cannot decrypt a cipher text shorter than " + MIN_CIPHER_TEXT_LENGTH + " chars"); + } + + if (!protectedValue.contains(DELIMITER)) { + throw new IllegalArgumentException("The cipher text does not contain the delimiter " + DELIMITER + " -- it should be of the form Base64(IV) || Base64(cipherText)"); + } + + protectedValue = protectedValue.trim(); + + final String IV_B64 = protectedValue.substring(0, protectedValue.indexOf(DELIMITER)); + byte[] iv = Base64.decode(IV_B64); + if (iv.length < IV_LENGTH) { + throw new IllegalArgumentException("The IV (" + iv.length + " bytes) must be at least " + IV_LENGTH + " bytes"); + } + + String CIPHERTEXT_B64 = protectedValue.substring(protectedValue.indexOf(DELIMITER) + 2); + + // Restore the = padding if necessary to reconstitute the GCM MAC check + if (CIPHERTEXT_B64.length() % 4 != 0) { + final int paddedLength = CIPHERTEXT_B64.length() + 4 - (CIPHERTEXT_B64.length() % 4); + CIPHERTEXT_B64 = StringUtils.rightPad(CIPHERTEXT_B64, paddedLength, '='); + } + + try { + byte[] cipherBytes = Base64.decode(CIPHERTEXT_B64); + + cipher.init(Cipher.DECRYPT_MODE, this.key, new IvParameterSpec(iv)); + byte[] plainBytes = cipher.doFinal(cipherBytes); + logger.debug(getName() + " decrypted a sensitive value successfully"); + return new String(plainBytes, StandardCharsets.UTF_8); + } catch (BadPaddingException | IllegalBlockSizeException | DecoderException | InvalidAlgorithmParameterException | InvalidKeyException e) { + final String msg = "Error decrypting a protected value"; + logger.error(msg, e); + throw new SensitivePropertyProtectionException(msg, e); + } + } + + public static int getIvLength() { + return IV_LENGTH; + } + + public static int getMinCipherTextLength() { + return MIN_CIPHER_TEXT_LENGTH; + } + + public static String getDelimiter() { + return DELIMITER; + } +} diff --git a/nifi-registry-properties/src/main/java/org/apache/nifi/registry/properties/AESSensitivePropertyProviderFactory.java b/nifi-registry-properties/src/main/java/org/apache/nifi/registry/properties/AESSensitivePropertyProviderFactory.java new file mode 100644 index 000000000..5c24a73c6 --- /dev/null +++ b/nifi-registry-properties/src/main/java/org/apache/nifi/registry/properties/AESSensitivePropertyProviderFactory.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.registry.properties; + +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.crypto.NoSuchPaddingException; +import java.security.NoSuchAlgorithmException; +import java.security.NoSuchProviderException; + +public class AESSensitivePropertyProviderFactory implements SensitivePropertyProviderFactory { + private static final Logger logger = LoggerFactory.getLogger(AESSensitivePropertyProviderFactory.class); + + private String keyHex; + + public AESSensitivePropertyProviderFactory(String keyHex) { + this.keyHex = keyHex; + } + + public SensitivePropertyProvider getProvider() throws SensitivePropertyProtectionException { + try { + if (keyHex != null && !StringUtils.isBlank(keyHex)) { + return new AESSensitivePropertyProvider(keyHex); + } else { + throw new SensitivePropertyProtectionException("The provider factory cannot generate providers without a key"); + } + } catch (NoSuchAlgorithmException | NoSuchProviderException | NoSuchPaddingException e) { + String msg = "Error creating AES Sensitive Property Provider"; + logger.warn(msg, e); + throw new SensitivePropertyProtectionException(msg, e); + } + } + + @Override + public String toString() { + return "SensitivePropertyProviderFactory for creating AESSensitivePropertyProviders"; + } +} diff --git a/nifi-registry-properties/src/main/java/org/apache/nifi/registry/properties/MultipleSensitivePropertyProtectionException.java b/nifi-registry-properties/src/main/java/org/apache/nifi/registry/properties/MultipleSensitivePropertyProtectionException.java new file mode 100644 index 000000000..df4047fea --- /dev/null +++ b/nifi-registry-properties/src/main/java/org/apache/nifi/registry/properties/MultipleSensitivePropertyProtectionException.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.registry.properties; + +import org.apache.commons.lang3.StringUtils; + +import java.util.Collection; +import java.util.HashSet; +import java.util.Set; + +public class MultipleSensitivePropertyProtectionException extends SensitivePropertyProtectionException { + + private Set failedKeys; + + /** + * Constructs a new throwable with {@code null} as its detail message. + * The cause is not initialized, and may subsequently be initialized by a + * call to {@link #initCause}. + *

+ *

The {@link #fillInStackTrace()} method is called to initialize + * the stack trace data in the newly created throwable. + */ + public MultipleSensitivePropertyProtectionException() { + } + + /** + * Constructs a new throwable with the specified detail message. The + * cause is not initialized, and may subsequently be initialized by + * a call to {@link #initCause}. + *

+ *

The {@link #fillInStackTrace()} method is called to initialize + * the stack trace data in the newly created throwable. + * + * @param message the detail message. The detail message is saved for + * later retrieval by the {@link #getMessage()} method. + */ + public MultipleSensitivePropertyProtectionException(String message) { + super(message); + } + + /** + * Constructs a new throwable with the specified detail message and + * cause.

Note that the detail message associated with + * {@code cause} is not automatically incorporated in + * this throwable's detail message. + *

+ *

The {@link #fillInStackTrace()} method is called to initialize + * the stack trace data in the newly created throwable. + * + * @param message the detail message (which is saved for later retrieval + * by the {@link #getMessage()} method). + * @param cause the cause (which is saved for later retrieval by the + * {@link #getCause()} method). (A {@code null} value is + * permitted, and indicates that the cause is nonexistent or + * unknown.) + * @since 1.4 + */ + public MultipleSensitivePropertyProtectionException(String message, Throwable cause) { + super(message, cause); + } + + /** + * Constructs a new throwable with the specified cause and a detail + * message of {@code (cause==null ? null : cause.toString())} (which + * typically contains the class and detail message of {@code cause}). + * This constructor is useful for throwables that are little more than + * wrappers for other throwables (for example, PrivilegedActionException). + *

+ *

The {@link #fillInStackTrace()} method is called to initialize + * the stack trace data in the newly created throwable. + * + * @param cause the cause (which is saved for later retrieval by the + * {@link #getCause()} method). (A {@code null} value is + * permitted, and indicates that the cause is nonexistent or + * unknown.) + * @since 1.4 + */ + public MultipleSensitivePropertyProtectionException(Throwable cause) { + super(cause); + } + + /** + * Constructs a new exception with the provided message and a unique set of the keys that caused the error. + * + * @param message the message + * @param failedKeys any failed keys + */ + public MultipleSensitivePropertyProtectionException(String message, Collection failedKeys) { + this(message, failedKeys, null); + } + + /** + * Constructs a new exception with the provided message and a unique set of the keys that caused the error. + * + * @param message the message + * @param failedKeys any failed keys + * @param cause the cause (which is saved for later retrieval by the + * {@link #getCause()} method). (A {@code null} value is + * permitted, and indicates that the cause is nonexistent or + * unknown.) + */ + public MultipleSensitivePropertyProtectionException(String message, Collection failedKeys, Throwable cause) { + super(message, cause); + this.failedKeys = new HashSet<>(failedKeys); + } + + public Set getFailedKeys() { + return this.failedKeys; + } + + @Override + public String toString() { + return "SensitivePropertyProtectionException for [" + StringUtils.join(this.failedKeys, ", ") + "]: " + getLocalizedMessage(); + } +} diff --git a/nifi-registry-properties/src/main/java/org/apache/nifi/registry/properties/NiFiRegistryPropertiesLoader.java b/nifi-registry-properties/src/main/java/org/apache/nifi/registry/properties/NiFiRegistryPropertiesLoader.java new file mode 100644 index 000000000..5ceffd175 --- /dev/null +++ b/nifi-registry-properties/src/main/java/org/apache/nifi/registry/properties/NiFiRegistryPropertiesLoader.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.registry.properties; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.crypto.Cipher; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.security.NoSuchAlgorithmException; + +public class NiFiRegistryPropertiesLoader { + + private static final Logger logger = LoggerFactory.getLogger(NiFiRegistryPropertiesLoader.class); + + private static final String RELATIVE_PATH = "conf/nifi-registry.properties"; + + private String keyHex; + + // Future enhancement: allow for external registration of new providers + private static SensitivePropertyProviderFactory sensitivePropertyProviderFactory; + + /** + * Returns an instance of the loader configured with the key. + *

+ *

+ * NOTE: This method is used reflectively by the process which starts NiFi + * so changes to it must be made in conjunction with that mechanism.

+ * + * @param keyHex the key used to encrypt any sensitive properties + * @return the configured loader + */ + public static NiFiRegistryPropertiesLoader withKey(String keyHex) { + NiFiRegistryPropertiesLoader loader = new NiFiRegistryPropertiesLoader(); + loader.setKeyHex(keyHex); + return loader; + } + + /** + * Sets the hexadecimal key used to unprotect properties encrypted with + * {@link AESSensitivePropertyProvider}. If the key has already been set, + * calling this method will throw a {@link RuntimeException}. + * + * @param keyHex the key in hexadecimal format + */ + public void setKeyHex(String keyHex) { + if (this.keyHex == null || this.keyHex.trim().isEmpty()) { + this.keyHex = keyHex; + } else { + throw new RuntimeException("Cannot overwrite an existing key"); + } + } + + private static String getDefaultProviderKey() { + try { + return "aes/gcm/" + (Cipher.getMaxAllowedKeyLength("AES") > 128 ? "256" : "128"); + } catch (NoSuchAlgorithmException e) { + return "aes/gcm/128"; + } + } + + private void initializeSensitivePropertyProviderFactory() { + sensitivePropertyProviderFactory = new AESSensitivePropertyProviderFactory(keyHex); + } + + private SensitivePropertyProvider getSensitivePropertyProvider() { + initializeSensitivePropertyProviderFactory(); + return sensitivePropertyProviderFactory.getProvider(); + } + + /** + * Returns a {@link ProtectedNiFiRegistryProperties} instance loaded from the + * serialized form in the file. Responsible for actually reading from disk + * and deserializing the properties. Returns a protected instance to allow + * for decryption operations. + * + * @param file the file containing serialized properties + * @return the ProtectedNiFiProperties instance + */ + ProtectedNiFiRegistryProperties readProtectedPropertiesFromDisk(File file) { + if (file == null || !file.exists() || !file.canRead()) { + String path = (file == null ? "missing file" : file.getAbsolutePath()); + logger.error("Cannot read from '{}' -- file is missing or not readable", path); + throw new IllegalArgumentException("NiFi Registry properties file missing or unreadable"); + } + + final NiFiRegistryProperties rawProperties = new NiFiRegistryProperties(); + try (final FileReader reader = new FileReader(file)) { + rawProperties.load(reader); + logger.info("Loaded {} properties from {}", rawProperties.size(), file.getAbsolutePath()); + ProtectedNiFiRegistryProperties protectedNiFiRegistryProperties = new ProtectedNiFiRegistryProperties(rawProperties); + return protectedNiFiRegistryProperties; + } catch (final IOException ioe) { + logger.error("Cannot load properties file due to " + ioe.getLocalizedMessage()); + throw new RuntimeException("Cannot load properties file due to " + ioe.getLocalizedMessage(), ioe); + } + } + + /** + * Returns an instance of {@link NiFiRegistryProperties} loaded from the provided + * {@link File}. If any properties are protected, will attempt to use the appropriate + * {@link SensitivePropertyProvider} to unprotect them transparently. + * + * @param file the File containing the serialized properties + * @return the NiFiProperties instance + */ + public NiFiRegistryProperties load(File file) { + ProtectedNiFiRegistryProperties protectedNiFiRegistryProperties = readProtectedPropertiesFromDisk(file); + if (protectedNiFiRegistryProperties.hasProtectedKeys()) { + protectedNiFiRegistryProperties.addSensitivePropertyProvider(getSensitivePropertyProvider()); + } + + return protectedNiFiRegistryProperties.getUnprotectedProperties(); + } + + /** + * Returns an instance of {@link NiFiRegistryProperties}. The path must not be empty. + * + * @param path the path of the serialized properties file + * @return the NiFiRegistryProperties instance + * @see NiFiRegistryPropertiesLoader#load(File) + */ + public NiFiRegistryProperties load(String path) { + if (path != null && !path.trim().isEmpty()) { + return load(new File(path)); + } else { + logger.error("Cannot read from '{}' -- path is null or empty", path); + throw new IllegalArgumentException("NiFi Registry properties file path empty or null"); + } + } + +} diff --git a/nifi-registry-properties/src/main/java/org/apache/nifi/registry/properties/ProtectedNiFiRegistryProperties.java b/nifi-registry-properties/src/main/java/org/apache/nifi/registry/properties/ProtectedNiFiRegistryProperties.java new file mode 100644 index 000000000..5debc4a1b --- /dev/null +++ b/nifi-registry-properties/src/main/java/org/apache/nifi/registry/properties/ProtectedNiFiRegistryProperties.java @@ -0,0 +1,528 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.registry.properties; + +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import static java.util.Arrays.asList; + +/** + * Wrapper class of {@link NiFiRegistryProperties} for intermediate phase when + * {@link NiFiRegistryPropertiesLoader} loads the raw properties file and performs + * unprotection activities before returning an instance of {@link NiFiRegistryProperties}. + */ +class ProtectedNiFiRegistryProperties { + private static final Logger logger = LoggerFactory.getLogger(ProtectedNiFiRegistryProperties.class); + + private NiFiRegistryProperties properties; + + private Map localProviderCache = new HashMap<>(); + + // Additional "sensitive" property key + public static final String ADDITIONAL_SENSITIVE_PROPERTIES_KEY = "nifi.registry.sensitive.props.additional.keys"; + + // Default list of "sensitive" property keys + public static final List DEFAULT_SENSITIVE_PROPERTIES = new ArrayList<>(asList( + NiFiRegistryProperties.SECURITY_KEY_PASSWD, + NiFiRegistryProperties.SECURITY_KEYSTORE_PASSWD, + NiFiRegistryProperties.SECURITY_TRUSTSTORE_PASSWD)); + + public ProtectedNiFiRegistryProperties() { + this(null); + } + + /** + * Creates an instance containing the provided {@link NiFiRegistryProperties}. + * + * @param props the NiFiProperties to contain + */ + public ProtectedNiFiRegistryProperties(NiFiRegistryProperties props) { + if (props == null) { + props = new NiFiRegistryProperties(); + } + this.properties = props; + logger.debug("Loaded {} properties (including {} protection schemes) into ProtectedNiFiProperties", + getPropertyKeysIncludingProtectionSchemes().size(), getProtectedPropertyKeys().size()); + } + + /** + * Retrieves the property value for the given property key. + * + * @param key the key of property value to lookup + * @return value of property at given key or null if not found + */ + // @Override + public String getProperty(String key) { + return getInternalNiFiProperties().getProperty(key); + } + + /** + * Returns the internal representation of the {@link NiFiRegistryProperties} -- protected + * or not as determined by the current state. No guarantee is made to the + * protection state of these properties. If the internal reference is null, a new + * {@link NiFiRegistryProperties} instance is created. + * + * @return the internal properties + */ + NiFiRegistryProperties getInternalNiFiProperties() { + if (this.properties == null) { + this.properties = new NiFiRegistryProperties(); + } + + return this.properties; + } + + /** + * Returns the number of properties in the NiFiRegistryProperties, + * excluding protection scheme properties. + * + *

+ * Example: + *

+ * key: E(value, key) + * key.protected: aes/gcm/256 + * key2: value2 + *

+ * would return size 2 + * + * @return the count of real properties + */ + int size() { + return getPropertyKeysExcludingProtectionSchemes().size(); + } + + /** + * Returns the complete set of property keys in the NiFiRegistryProperties, + * including any protection keys (i.e. 'x.y.z.protected'). + * + * @return the set of property keys + */ + Set getPropertyKeysIncludingProtectionSchemes() { + return getInternalNiFiProperties().getPropertyKeys(); + } + + /** + * Returns the set of property keys in the NiFiRegistryProperties, + * excluding any protection keys (i.e. 'x.y.z.protected'). + * + * @return the set of property keys + */ + Set getPropertyKeysExcludingProtectionSchemes() { + Set filteredKeys = getPropertyKeysIncludingProtectionSchemes(); + filteredKeys.removeIf(p -> p.endsWith(".protected")); + return filteredKeys; + } + + /** + * Splits a single string containing multiple property keys into a List. + * + * Delimited by ',' or ';' and ignores leading and trailing whitespace around delimiter. + * + * @param multipleProperties a single String containing multiple properties, i.e. + * "nifi.registry.property.1; nifi.registry.property.2, nifi.registry.property.3" + * @return a List containing the split and trimmed properties + */ + private static List splitMultipleProperties(String multipleProperties) { + if (multipleProperties == null || multipleProperties.trim().isEmpty()) { + return new ArrayList<>(0); + } else { + List properties = new ArrayList<>(asList(multipleProperties.split("\\s*[,;]\\s*"))); + for (int i = 0; i < properties.size(); i++) { + properties.set(i, properties.get(i).trim()); + } + return properties; + } + } + + /** + * Returns a list of the keys identifying "sensitive" properties. + * + * There is a default list, and additional keys can be provided in the + * {@code nifi.registry.sensitive.props.additional.keys} property in {@code nifi-registry.properties}. + * + * @return the list of sensitive property keys + */ + public List getSensitivePropertyKeys() { + String additionalPropertiesString = getProperty(ADDITIONAL_SENSITIVE_PROPERTIES_KEY); + if (additionalPropertiesString == null || additionalPropertiesString.trim().isEmpty()) { + return DEFAULT_SENSITIVE_PROPERTIES; + } else { + List additionalProperties = splitMultipleProperties(additionalPropertiesString); + /* Remove this key if it was accidentally provided as a sensitive key + * because we cannot protect it and read from it + */ + if (additionalProperties.contains(ADDITIONAL_SENSITIVE_PROPERTIES_KEY)) { + logger.warn("The key '{}' contains itself. This is poor practice and should be removed", ADDITIONAL_SENSITIVE_PROPERTIES_KEY); + additionalProperties.remove(ADDITIONAL_SENSITIVE_PROPERTIES_KEY); + } + additionalProperties.addAll(DEFAULT_SENSITIVE_PROPERTIES); + return additionalProperties; + } + } + + /** + * Returns a list of the keys identifying "sensitive" properties. There is a default list, + * and additional keys can be provided in the {@code nifi.sensitive.props.additional.keys} property in {@code nifi.properties}. + * + * @return the list of sensitive property keys + */ + public List getPopulatedSensitivePropertyKeys() { + List allSensitiveKeys = getSensitivePropertyKeys(); + return allSensitiveKeys.stream().filter(k -> StringUtils.isNotBlank(getProperty(k))).collect(Collectors.toList()); + } + + /** + * Returns true if any sensitive keys are protected. + * + * @return true if any key is protected; false otherwise + */ + public boolean hasProtectedKeys() { + List sensitiveKeys = getSensitivePropertyKeys(); + for (String k : sensitiveKeys) { + if (isPropertyProtected(k)) { + return true; + } + } + return false; + } + + /** + * Returns a Map of the keys identifying "sensitive" properties that are currently protected and the "protection" key for each. + * + * This may or may not include all properties marked as sensitive. + * + * @return the Map of protected property keys and the protection identifier for each + */ + public Map getProtectedPropertyKeys() { + List sensitiveKeys = getSensitivePropertyKeys(); + + Map traditionalProtectedProperties = new HashMap<>(); + for (String key : sensitiveKeys) { + String protection = getProperty(getProtectionKey(key)); + if (StringUtils.isNotBlank(protection) && StringUtils.isNotBlank(getProperty(key))) { + traditionalProtectedProperties.put(key, protection); + } + } + + return traditionalProtectedProperties; + } + + /** + * Returns the unique set of all protection schemes currently in use for this instance. + * + * @return the set of protection schemes + */ + public Set getProtectionSchemes() { + return new HashSet<>(getProtectedPropertyKeys().values()); + } + + /** + * Returns a percentage of the total number of populated properties marked as sensitive that are currently protected. + * + * @return the percent of sensitive properties marked as protected + */ + public int getPercentOfSensitivePropertiesProtected() { + return (int) Math.round(getProtectedPropertyKeys().size() / ((double) getPopulatedSensitivePropertyKeys().size()) * 100); + } + + /** + * Returns true if the property identified by this key is considered sensitive in this instance of {@code NiFiProperties}. + * Some properties are sensitive by default, while others can be specified by + * {@link ProtectedNiFiRegistryProperties#ADDITIONAL_SENSITIVE_PROPERTIES_KEY}. + * + * @param key the key + * @return true if it is sensitive + * @see ProtectedNiFiRegistryProperties#getSensitivePropertyKeys() + */ + public boolean isPropertySensitive(String key) { + // If the explicit check for ADDITIONAL_SENSITIVE_PROPERTIES_KEY is not here, this could loop infinitely + return key != null && !key.equals(ADDITIONAL_SENSITIVE_PROPERTIES_KEY) && getSensitivePropertyKeys().contains(key.trim()); + } + + /** + * Returns true if the property identified by this key is considered protected in this instance of {@code NiFiProperties}. + * The property value is protected if the key is sensitive and the sibling key of key.protected is present. + * + * @param key the key + * @return true if it is currently marked as protected + * @see ProtectedNiFiRegistryProperties#getSensitivePropertyKeys() + */ + public boolean isPropertyProtected(String key) { + return key != null && isPropertySensitive(key) && !StringUtils.isBlank(getProperty(getProtectionKey(key))); + } + + /** + * Returns the sibling property key which specifies the protection scheme for this key. + *

+ * Example: + *

+ * nifi.registry.sensitive.key=ABCXYZ + * nifi.registry.sensitive.key.protected=aes/gcm/256 + *

+ * nifi.registry.sensitive.key -> nifi.sensitive.key.protected + * + * @param key the key identifying the sensitive property + * @return the key identifying the protection scheme for the sensitive property + */ + public static String getProtectionKey(String key) { + if (key == null || key.isEmpty()) { + throw new IllegalArgumentException("Cannot find protection key for null key"); + } + + return key + ".protected"; + } + + /** + * Returns the unprotected {@link NiFiRegistryProperties} instance. If none of the + * properties loaded are marked as protected, it will simply pass through the + * internal instance. If any are protected, it will drop the protection scheme keys + * and translate each protected value (encrypted, HSM-retrieved, etc.) into the raw + * value and store it under the original key. + *

+ * If any property fails to unprotect, it will save that key and continue. After + * attempting all properties, it will throw an exception containing all failed + * properties. This is necessary because the order is not enforced, so all failed + * properties should be gathered together. + * + * @return the NiFiRegistryProperties instance with all raw values + * @throws SensitivePropertyProtectionException if there is a problem unprotecting one or more keys + */ + public NiFiRegistryProperties getUnprotectedProperties() throws SensitivePropertyProtectionException { + if (hasProtectedKeys()) { + logger.debug("There are {} protected properties of {} sensitive properties ({}%)", + getProtectedPropertyKeys().size(), + getPopulatedSensitivePropertyKeys().size(), + getPercentOfSensitivePropertiesProtected()); + + NiFiRegistryProperties unprotectedProperties = new NiFiRegistryProperties(); + + Set failedKeys = new HashSet<>(); + + for (String key : getPropertyKeysExcludingProtectionSchemes()) { + /* Three kinds of keys + * 1. protection schemes -- skip + * 2. protected keys -- unprotect and copy + * 3. normal keys -- copy over + */ + if (key.endsWith(".protected")) { + // Do nothing + } else if (isPropertyProtected(key)) { + try { + unprotectedProperties.setProperty(key, unprotectValue(key, getProperty(key))); + } catch (SensitivePropertyProtectionException e) { + logger.warn("Failed to unprotect '{}'", key, e); + failedKeys.add(key); + } + } else { + unprotectedProperties.setProperty(key, getProperty(key)); + } + } + + if (!failedKeys.isEmpty()) { + if (failedKeys.size() > 1) { + logger.warn("Combining {} failed keys [{}] into single exception", failedKeys.size(), StringUtils.join(failedKeys, ", ")); + throw new MultipleSensitivePropertyProtectionException("Failed to unprotect keys", failedKeys); + } else { + throw new SensitivePropertyProtectionException("Failed to unprotect key " + failedKeys.iterator().next()); + } + } + + return unprotectedProperties; + } else { + logger.debug("No protected properties"); + return getInternalNiFiProperties(); + } + } + + /** + * Registers a new {@link SensitivePropertyProvider}. This method will throw a {@link UnsupportedOperationException} if a provider is already registered for the protection scheme. + * + * @param sensitivePropertyProvider the provider + */ + void addSensitivePropertyProvider(SensitivePropertyProvider sensitivePropertyProvider) { + if (sensitivePropertyProvider == null) { + throw new IllegalArgumentException("Cannot add null SensitivePropertyProvider"); + } + + if (getSensitivePropertyProviders().containsKey(sensitivePropertyProvider.getIdentifierKey())) { + throw new UnsupportedOperationException("Cannot overwrite existing sensitive property provider registered for " + sensitivePropertyProvider.getIdentifierKey()); + } + + getSensitivePropertyProviders().put(sensitivePropertyProvider.getIdentifierKey(), sensitivePropertyProvider); + } + + private String getDefaultProtectionScheme() { + if (!getSensitivePropertyProviders().isEmpty()) { + List schemes = new ArrayList<>(getSensitivePropertyProviders().keySet()); + Collections.sort(schemes); + return schemes.get(0); + } else { + throw new IllegalStateException("No registered protection schemes"); + } + } + + /** + * Returns a new instance of {@link NiFiRegistryProperties} with all populated sensitive values protected by the default protection scheme. + * + * Plain non-sensitive values are copied directly. + * + * @return the protected properties in a {@link NiFiRegistryProperties} object + * @throws IllegalStateException if no protection schemes are registered + */ + NiFiRegistryProperties protectPlainProperties() { + try { + return protectPlainProperties(getDefaultProtectionScheme()); + } catch (IllegalStateException e) { + final String msg = "Cannot protect properties with default scheme if no protection schemes are registered"; + logger.warn(msg); + throw new IllegalStateException(msg, e); + } + } + + /** + * Returns a new instance of {@link NiFiRegistryProperties} with all populated sensitive values protected by the provided protection scheme. + * + * Plain non-sensitive values are copied directly. + * + * @param protectionScheme the identifier key of the {@link SensitivePropertyProvider} to use + * @return the protected properties in a {@link NiFiRegistryProperties} object + */ + NiFiRegistryProperties protectPlainProperties(String protectionScheme) { + SensitivePropertyProvider spp = getSensitivePropertyProvider(protectionScheme); + + NiFiRegistryProperties protectedProperties = new NiFiRegistryProperties(); + + // Copy over the plain keys + Set plainKeys = getPropertyKeysExcludingProtectionSchemes(); + plainKeys.removeAll(getSensitivePropertyKeys()); + for (String key : plainKeys) { + protectedProperties.setProperty(key, getInternalNiFiProperties().getProperty(key)); + } + + // Add the protected keys and the protection schemes + for (String key : getSensitivePropertyKeys()) { + final String plainValue = getProperty(key); + if (plainValue != null && !plainValue.trim().isEmpty()) { + final String protectedValue = spp.protect(plainValue); + protectedProperties.setProperty(key, protectedValue); + protectedProperties.setProperty(getProtectionKey(key), protectionScheme); + } + } + + return protectedProperties; + } + + /** + * Returns the number of properties that are marked as protected in the provided {@link NiFiRegistryProperties} instance + * without requiring external creation of a {@link ProtectedNiFiRegistryProperties} instance. + * + * @param plainProperties the instance to count protected properties + * @return the number of protected properties + */ + public static int countProtectedProperties(NiFiRegistryProperties plainProperties) { + return new ProtectedNiFiRegistryProperties(plainProperties).getProtectedPropertyKeys().size(); + } + + /** + * Returns the number of properties that are marked as sensitive in the provided {@link NiFiRegistryProperties} instance + * without requiring external creation of a {@link ProtectedNiFiRegistryProperties} instance. + * + * @param plainProperties the instance to count sensitive properties + * @return the number of sensitive properties + */ + public static int countSensitiveProperties(NiFiRegistryProperties plainProperties) { + return new ProtectedNiFiRegistryProperties(plainProperties).getSensitivePropertyKeys().size(); + } + + @Override + public String toString() { + final Set providers = getSensitivePropertyProviders().keySet(); + return new StringBuilder("ProtectedNiFiProperties instance with ") + .append(getPropertyKeysIncludingProtectionSchemes().size()) + .append(" properties (") + .append(getProtectedPropertyKeys().size()) + .append(" protected) and ") + .append(providers.size()) + .append(" sensitive property providers: ") + .append(StringUtils.join(providers, ", ")) + .toString(); + } + + /** + * Returns the local provider cache (null-safe) as a Map of protection schemes -> implementations. + * + * @return the map + */ + private Map getSensitivePropertyProviders() { + if (localProviderCache == null) { + localProviderCache = new HashMap<>(); + } + + return localProviderCache; + } + + private SensitivePropertyProvider getSensitivePropertyProvider(String protectionScheme) { + if (isProviderAvailable(protectionScheme)) { + return getSensitivePropertyProviders().get(protectionScheme); + } else { + throw new SensitivePropertyProtectionException("No provider available for " + protectionScheme); + } + } + + private boolean isProviderAvailable(String protectionScheme) { + return getSensitivePropertyProviders().containsKey(protectionScheme); + } + + /** + * If the value is protected, unprotects it and returns it. If not, returns the original value. + * + * @param key the retrieved property key + * @param retrievedValue the retrieved property value + * @return the unprotected value + */ + private String unprotectValue(String key, String retrievedValue) { + // Checks if the key is sensitive and marked as protected + if (isPropertyProtected(key)) { + final String protectionScheme = getProperty(getProtectionKey(key)); + + // No provider registered for this scheme, so just return the value + if (!isProviderAvailable(protectionScheme)) { + logger.warn("No provider available for {} so passing the protected {} value back", protectionScheme, key); + return retrievedValue; + } + + try { + SensitivePropertyProvider sensitivePropertyProvider = getSensitivePropertyProvider(protectionScheme); + return sensitivePropertyProvider.unprotect(retrievedValue); + } catch (SensitivePropertyProtectionException e) { + throw new SensitivePropertyProtectionException("Error unprotecting value for " + key, e.getCause()); + } + } + return retrievedValue; + } +} diff --git a/nifi-registry-properties/src/main/java/org/apache/nifi/registry/properties/SensitivePropertyProtectionException.java b/nifi-registry-properties/src/main/java/org/apache/nifi/registry/properties/SensitivePropertyProtectionException.java new file mode 100644 index 000000000..2ffa9022f --- /dev/null +++ b/nifi-registry-properties/src/main/java/org/apache/nifi/registry/properties/SensitivePropertyProtectionException.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.registry.properties; + +public class SensitivePropertyProtectionException extends RuntimeException { + /** + * Constructs a new throwable with {@code null} as its detail message. + * The cause is not initialized, and may subsequently be initialized by a + * call to {@link #initCause}. + *

+ *

The {@link #fillInStackTrace()} method is called to initialize + * the stack trace data in the newly created throwable. + */ + public SensitivePropertyProtectionException() { + } + + /** + * Constructs a new throwable with the specified detail message. The + * cause is not initialized, and may subsequently be initialized by + * a call to {@link #initCause}. + *

+ *

The {@link #fillInStackTrace()} method is called to initialize + * the stack trace data in the newly created throwable. + * + * @param message the detail message. The detail message is saved for + * later retrieval by the {@link #getMessage()} method. + */ + public SensitivePropertyProtectionException(String message) { + super(message); + } + + /** + * Constructs a new throwable with the specified detail message and + * cause.

Note that the detail message associated with + * {@code cause} is not automatically incorporated in + * this throwable's detail message. + *

+ *

The {@link #fillInStackTrace()} method is called to initialize + * the stack trace data in the newly created throwable. + * + * @param message the detail message (which is saved for later retrieval + * by the {@link #getMessage()} method). + * @param cause the cause (which is saved for later retrieval by the + * {@link #getCause()} method). (A {@code null} value is + * permitted, and indicates that the cause is nonexistent or + * unknown.) + */ + public SensitivePropertyProtectionException(String message, Throwable cause) { + super(message, cause); + } + + /** + * Constructs a new throwable with the specified cause and a detail + * message of {@code (cause==null ? null : cause.toString())} (which + * typically contains the class and detail message of {@code cause}). + * This constructor is useful for throwables that are little more than + * wrappers for other throwables (for example, PrivilegedActionException). + *

+ *

The {@link #fillInStackTrace()} method is called to initialize + * the stack trace data in the newly created throwable. + * + * @param cause the cause (which is saved for later retrieval by the + * {@link #getCause()} method). (A {@code null} value is + * permitted, and indicates that the cause is nonexistent or + * unknown.) + */ + public SensitivePropertyProtectionException(Throwable cause) { + super(cause); + } + + @Override + public String toString() { + return "SensitivePropertyProtectionException: " + getLocalizedMessage(); + } +} diff --git a/nifi-registry-properties/src/main/java/org/apache/nifi/registry/properties/SensitivePropertyProvider.java b/nifi-registry-properties/src/main/java/org/apache/nifi/registry/properties/SensitivePropertyProvider.java new file mode 100644 index 000000000..c0dd43c6b --- /dev/null +++ b/nifi-registry-properties/src/main/java/org/apache/nifi/registry/properties/SensitivePropertyProvider.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.registry.properties; + +public interface SensitivePropertyProvider { + + /** + * Returns the name of the underlying implementation. + * + * @return the name of this sensitive property provider + */ + String getName(); + + /** + * Returns the key used to identify the provider implementation in {@code nifi.properties}. + * + * @return the key to persist in the sibling property + */ + String getIdentifierKey(); + + /** + * Returns the "protected" form of this value. This is a form which can safely be persisted in the {@code nifi.properties} file without compromising the value. + * An encryption-based provider would return a cipher text, while a remote-lookup provider could return a unique ID to retrieve the secured value. + * + * @param unprotectedValue the sensitive value + * @return the value to persist in the {@code nifi.properties} file + */ + String protect(String unprotectedValue) throws SensitivePropertyProtectionException; + + /** + * Returns the "unprotected" form of this value. This is the raw sensitive value which is used by the application logic. + * An encryption-based provider would decrypt a cipher text and return the plaintext, while a remote-lookup provider could retrieve the secured value. + * + * @param protectedValue the protected value read from the {@code nifi.properties} file + * @return the raw value to be used by the application + */ + String unprotect(String protectedValue) throws SensitivePropertyProtectionException; +} diff --git a/nifi-registry-properties/src/main/java/org/apache/nifi/registry/properties/SensitivePropertyProviderFactory.java b/nifi-registry-properties/src/main/java/org/apache/nifi/registry/properties/SensitivePropertyProviderFactory.java new file mode 100644 index 000000000..c9d4313e8 --- /dev/null +++ b/nifi-registry-properties/src/main/java/org/apache/nifi/registry/properties/SensitivePropertyProviderFactory.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.registry.properties; + +public interface SensitivePropertyProviderFactory { + + SensitivePropertyProvider getProvider(); + +} diff --git a/nifi-registry-properties/src/main/java/org/apache/nifi/registry/security/crypto/BootstrapFileCryptoKeyProvider.java b/nifi-registry-properties/src/main/java/org/apache/nifi/registry/security/crypto/BootstrapFileCryptoKeyProvider.java new file mode 100644 index 000000000..191b5e28b --- /dev/null +++ b/nifi-registry-properties/src/main/java/org/apache/nifi/registry/security/crypto/BootstrapFileCryptoKeyProvider.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.registry.security.crypto; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; + +/** + * An implementation of {@link CryptoKeyProvider} that loads the key from disk every time it is needed. + * + * The persistence-backing of the key is in the bootstrap.conf file, which must be provided to the + * constructor of this class. + * + * As key access for sensitive value decryption is only used a few times during server initialization, + * this implementation trades efficiency for security by only keeping the key in memory with an + * in-scope reference for a brief period of time (assuming callers do not maintain an in-scope reference). + * + * @see CryptoKeyProvider + */ +public class BootstrapFileCryptoKeyProvider implements CryptoKeyProvider { + + private static final Logger logger = LoggerFactory.getLogger(BootstrapFileCryptoKeyProvider.class); + + private final String bootstrapFile; + + /** + * Construct a new instance backed by the contents of a bootstrap.conf file. + * + * @param bootstrapFilePath The path to the bootstrap.conf file for this instance of NiFi Registry. + * Must not be null. + */ + public BootstrapFileCryptoKeyProvider(final String bootstrapFilePath) { + if (bootstrapFilePath == null) { + throw new IllegalArgumentException(BootstrapFileCryptoKeyProvider.class.getSimpleName() + " cannot be initialized with null bootstrap file path."); + } + this.bootstrapFile = bootstrapFilePath; + } + + /** + * @return The bootstrap file path that backs this provider instance. + */ + public String getBootstrapFile() { + return bootstrapFile; + } + + @Override + public String getKey() throws MissingCryptoKeyException { + try { + return CryptoKeyLoader.extractKeyFromBootstrapFile(this.bootstrapFile); + } catch (IOException ioe) { + final String errMsg = "Loading the master crypto key from bootstrap file '" + bootstrapFile + "' failed due to IOException."; + logger.warn(errMsg); + throw new MissingCryptoKeyException(errMsg, ioe); + } + + } + + @Override + public String toString() { + return "BootstrapFileCryptoKeyProvider{" + + "bootstrapFile='" + bootstrapFile + '\'' + + '}'; + } + +} diff --git a/nifi-registry-properties/src/main/java/org/apache/nifi/registry/security/crypto/CryptoKeyLoader.java b/nifi-registry-properties/src/main/java/org/apache/nifi/registry/security/crypto/CryptoKeyLoader.java new file mode 100644 index 000000000..483741815 --- /dev/null +++ b/nifi-registry-properties/src/main/java/org/apache/nifi/registry/security/crypto/CryptoKeyLoader.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.registry.security.crypto; + +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Optional; +import java.util.Random; +import java.util.stream.Stream; + +public class CryptoKeyLoader { + + private static final Logger logger = LoggerFactory.getLogger(CryptoKeyLoader.class); + + private static final String BOOTSTRAP_KEY_PREFIX = "nifi.registry.bootstrap.sensitive.key="; + + /** + * Returns the key (if any) used to encrypt sensitive properties. + * The key extracted from the bootstrap.conf file at the specified location. + * + * @param bootstrapPath the path to the bootstrap file + * @return the key in hexadecimal format, or {@link CryptoKeyProvider#EMPTY_KEY} if the key is null or empty + * @throws IOException if the file is not readable + */ + public static String extractKeyFromBootstrapFile(String bootstrapPath) throws IOException { + File expectedBootstrapFile; + if (StringUtils.isBlank(bootstrapPath)) { + logger.error("Cannot read from bootstrap.conf file to extract encryption key; location not specified"); + throw new IOException("Cannot read from bootstrap.conf without file location"); + } else { + expectedBootstrapFile = new File(bootstrapPath); + } + + if (expectedBootstrapFile.exists() && expectedBootstrapFile.canRead()) { + try (Stream stream = Files.lines(Paths.get(expectedBootstrapFile.getAbsolutePath()))) { + Optional keyLine = stream.filter(l -> l.startsWith(BOOTSTRAP_KEY_PREFIX)).findFirst(); + if (keyLine.isPresent()) { + String keyValue = keyLine.get().split("=", 2)[1]; + return checkHexKey(keyValue); + } else { + logger.warn("No encryption key present in the bootstrap.conf file at {}", expectedBootstrapFile.getAbsolutePath()); + return CryptoKeyProvider.EMPTY_KEY; + } + } catch (IOException e) { + logger.error("Cannot read from bootstrap.conf file at {} to extract encryption key", expectedBootstrapFile.getAbsolutePath()); + throw new IOException("Cannot read from bootstrap.conf", e); + } + } else { + logger.error("Cannot read from bootstrap.conf file at {} to extract encryption key -- file is missing or permissions are incorrect", expectedBootstrapFile.getAbsolutePath()); + throw new IOException("Cannot read from bootstrap.conf"); + } + } + + /** + * Returns the key (if any) used to encrypt sensitive properties. + * The key extracted from the key file at the specified location. + * + * @param keyfilePath the path to the file containing the password/key + * @param securelyDeleteKeyfileOnSuccess If true, this method has the additional + * side-effect of overwriting the contents + * of the key file (with random bytes) and + * deleting it after successfully reading the key. + * If the key is not read from the file, secure + * deletion is not attempted even if this flag is set to true. + * @return the key in hexadecimal format, or {@link CryptoKeyProvider#EMPTY_KEY} if the key is null or empty + * @throws IOException if the file is not readable + */ + public static String extractKeyFromKeyFile(String keyfilePath, boolean securelyDeleteKeyfileOnSuccess) + throws IOException, IllegalArgumentException { + + if (StringUtils.isBlank(keyfilePath)) { + logger.error("Cannot read from password file to extract encryption key; location not specified"); + throw new IOException("Cannot read from password file without file location"); + } + + // Slurp in the contents of the file: + logger.info("Loading crypto key from file: {}", keyfilePath); + byte[] encoded = Files.readAllBytes(Paths.get(keyfilePath)); + String key = new String(encoded, StandardCharsets.UTF_8); + if (0 == key.length()) + throw new IllegalArgumentException("Key in keyfile " + keyfilePath + " yielded an empty key"); + + if (securelyDeleteKeyfileOnSuccess) { + // Overwrite the contents of the file (to avoid littering file system unlinked with key material): + logger.info("Now overwriting file '{}' with random bytes ", keyfilePath); + File password_file = new File(keyfilePath); + FileWriter overwriter = new FileWriter(password_file,false); + + // Construe a random pad: + Random r = new Random(); + StringBuffer sb = new StringBuffer(); + // Note on correctness: this pad is longer, but equally sufficient. + while(sb.length() < encoded.length){ + sb.append(Integer.toHexString(r.nextInt())); + } + String pad = sb.toString(); + logger.debug("Overwriting key material with pad: " + pad); + overwriter.write(pad); + overwriter.close(); + + logger.info("Removing/unlinking file: "+ keyfilePath); + password_file.delete(); + } + + return checkHexKey(key); + } + + private static String checkHexKey(String input) { + if (input == null || input.trim().isEmpty()) { + return CryptoKeyProvider.EMPTY_KEY; + } + return input; + } + + + +} diff --git a/nifi-registry-properties/src/main/java/org/apache/nifi/registry/security/crypto/CryptoKeyProvider.java b/nifi-registry-properties/src/main/java/org/apache/nifi/registry/security/crypto/CryptoKeyProvider.java new file mode 100644 index 000000000..bab8d7c7f --- /dev/null +++ b/nifi-registry-properties/src/main/java/org/apache/nifi/registry/security/crypto/CryptoKeyProvider.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.registry.security.crypto; + +/** + * A simple interface that wraps a key that can be used for encryption and decryption. + * This allows for more flexibility with the lifecycle of keys and how other classes + * can declare dependencies for keys, by depending on a CryptoKeyProvider that will provided + * at runtime. + */ +public interface CryptoKeyProvider { + + /** + * A string literal that indicates the contents of a key are empty. + * Can also be used in contexts that a null key is undesirable. + */ + String EMPTY_KEY = ""; + + /** + * @return The crypto key known to this CryptoKeyProvider instance in hexadecimal format, or + * {@link #EMPTY_KEY} if the key is empty. + * @throws MissingCryptoKeyException if the key cannot be provided or determined for any reason. + * If the key is known to be empty, {@link #EMPTY_KEY} will be returned and a + * CryptoKeyMissingException will not be thrown + */ + String getKey() throws MissingCryptoKeyException; + + /** + * @return A boolean indicating if the key value held by this CryptoKeyProvider is empty, + * such as 'null' or empty string. + */ + default boolean isEmpty() { + String key; + try { + key = getKey(); + } catch (MissingCryptoKeyException e) { + return true; + } + return EMPTY_KEY.equals(key); + } + + /** + * A string representation of this CryptoKeyProvider instance. + *

+ *

+ * Note: Implementations of this interface should take care not to leak sensitive + * key material in any strings they emmit, including in the toString implementation. + * + * @return A string representation of this CryptoKeyProvider instance. + */ + @Override + public String toString(); + +} diff --git a/nifi-registry-properties/src/main/java/org/apache/nifi/registry/security/crypto/MissingCryptoKeyException.java b/nifi-registry-properties/src/main/java/org/apache/nifi/registry/security/crypto/MissingCryptoKeyException.java new file mode 100644 index 000000000..dbc3752c9 --- /dev/null +++ b/nifi-registry-properties/src/main/java/org/apache/nifi/registry/security/crypto/MissingCryptoKeyException.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.registry.security.crypto; + +/** + * An exception type used by a {@link CryptoKeyProvider} when a request for the key + * cannot be fulfilled for any reason. + * + * @see CryptoKeyProvider + */ +public class MissingCryptoKeyException extends Exception { + + public MissingCryptoKeyException() { + super(); + } + + public MissingCryptoKeyException(String message) { + super(message); + } + + public MissingCryptoKeyException(String message, Throwable cause) { + super(message, cause); + } + + public MissingCryptoKeyException(Throwable cause) { + super(cause); + } + + protected MissingCryptoKeyException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } + +} diff --git a/nifi-registry-properties/src/main/java/org/apache/nifi/registry/security/crypto/VolatileCryptoKeyProvider.java b/nifi-registry-properties/src/main/java/org/apache/nifi/registry/security/crypto/VolatileCryptoKeyProvider.java new file mode 100644 index 000000000..1fd451ac4 --- /dev/null +++ b/nifi-registry-properties/src/main/java/org/apache/nifi/registry/security/crypto/VolatileCryptoKeyProvider.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.registry.security.crypto; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * An implementation of {@link CryptoKeyProvider} that loads the key from disk every time it is needed. + * + * The persistence-backing of the key is in the bootstrap.conf file, which must be provided to the + * constructor of this class. + * + * As key access for sensitive value decryption is only used a few times during server initialization, + * this implementation trades efficiency for security by only keeping the key in memory with an + * in-scope reference for a brief period of time (assuming callers do not maintain an in-scope reference). + * + * @see CryptoKeyProvider + */ +public final class VolatileCryptoKeyProvider implements CryptoKeyProvider { + + private static final Logger logger = LoggerFactory.getLogger(VolatileCryptoKeyProvider.class); + + private static final VolatileCryptoKeyProvider EMPTY = new VolatileCryptoKeyProvider(EMPTY_KEY); + + private final String key; + + /** + * Construct a new instance with an in-memory key. + * + * @param key The contents of the key in hexadecimal format. + * Must not be null. + */ + public VolatileCryptoKeyProvider(final String key) { + if (key == null) { + throw new IllegalArgumentException(VolatileCryptoKeyProvider.class.getSimpleName() + " cannot be initialized with a null key."); + } + this.key = key; + } + + @Override + public String getKey() throws MissingCryptoKeyException { + if (key == null) { + throw new MissingCryptoKeyException("Key is null."); + } + return key; + } + + @Override + public String toString() { + return "VolatileCryptoKeyProvider{" + + "key='[PROTECTED]'" + + '}'; + } +} diff --git a/nifi-registry-properties/src/test/groovy/org/apache/nifi/registry/properties/AESSensitivePropertyProviderFactoryTest.groovy b/nifi-registry-properties/src/test/groovy/org/apache/nifi/registry/properties/AESSensitivePropertyProviderFactoryTest.groovy new file mode 100644 index 000000000..0d1d5e29b --- /dev/null +++ b/nifi-registry-properties/src/test/groovy/org/apache/nifi/registry/properties/AESSensitivePropertyProviderFactoryTest.groovy @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.registry.properties + +import org.bouncycastle.jce.provider.BouncyCastleProvider +import org.junit.* +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +import javax.crypto.Cipher +import java.security.Security + +@RunWith(JUnit4.class) +class AESSensitivePropertyProviderFactoryTest extends GroovyTestCase { + private static final Logger logger = LoggerFactory.getLogger(AESSensitivePropertyProviderFactoryTest.class) + + private static final String KEY_HEX_128 = "0123456789ABCDEFFEDCBA9876543210" + private static final String KEY_HEX_256 = KEY_HEX_128 * 2 + + @BeforeClass + public static void setUpOnce() throws Exception { + Security.addProvider(new BouncyCastleProvider()) + + logger.metaClass.methodMissing = { String name, args -> + logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}") + } + } + + @Before + public void setUp() throws Exception { + } + + @After + public void tearDown() throws Exception { + } + + @Test + public void testShouldGetProviderWithKey() throws Exception { + // Arrange + SensitivePropertyProviderFactory factory = new AESSensitivePropertyProviderFactory(KEY_HEX_128) + + // Act + SensitivePropertyProvider provider = factory.getProvider() + + // Assert + assert provider instanceof AESSensitivePropertyProvider + assert provider.@key + assert provider.@cipher + } + + @Test + public void testShouldGetProviderWith256BitKey() throws Exception { + // Arrange + Assume.assumeTrue("JCE unlimited strength crypto policy must be installed for this test", Cipher.getMaxAllowedKeyLength("AES") > 128) + SensitivePropertyProviderFactory factory = new AESSensitivePropertyProviderFactory(KEY_HEX_256) + + // Act + SensitivePropertyProvider provider = factory.getProvider() + + // Assert + assert provider instanceof AESSensitivePropertyProvider + assert provider.@key + assert provider.@cipher + } +} \ No newline at end of file diff --git a/nifi-registry-properties/src/test/groovy/org/apache/nifi/registry/properties/AESSensitivePropertyProviderTest.groovy b/nifi-registry-properties/src/test/groovy/org/apache/nifi/registry/properties/AESSensitivePropertyProviderTest.groovy new file mode 100644 index 000000000..98fdd9b76 --- /dev/null +++ b/nifi-registry-properties/src/test/groovy/org/apache/nifi/registry/properties/AESSensitivePropertyProviderTest.groovy @@ -0,0 +1,471 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.registry.properties + +import org.bouncycastle.jce.provider.BouncyCastleProvider +import org.bouncycastle.util.encoders.DecoderException +import org.bouncycastle.util.encoders.Hex +import org.junit.* +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +import javax.crypto.Cipher +import javax.crypto.spec.IvParameterSpec +import javax.crypto.spec.SecretKeySpec +import java.nio.charset.StandardCharsets +import java.security.SecureRandom +import java.security.Security + +@RunWith(JUnit4.class) +class AESSensitivePropertyProviderTest extends GroovyTestCase { + private static final Logger logger = LoggerFactory.getLogger(AESSensitivePropertyProviderTest.class) + + private static final String KEY_128_HEX = "0123456789ABCDEFFEDCBA9876543210" + private static final String KEY_256_HEX = KEY_128_HEX * 2 + private static final int IV_LENGTH = AESSensitivePropertyProvider.getIvLength() + + private static final List KEY_SIZES = getAvailableKeySizes() + + private static final SecureRandom secureRandom = new SecureRandom() + + private static final Base64.Encoder encoder = Base64.encoder + private static final Base64.Decoder decoder = Base64.decoder + + @BeforeClass + static void setUpOnce() throws Exception { + Security.addProvider(new BouncyCastleProvider()) + + logger.metaClass.methodMissing = { String name, args -> + logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}") + } + } + + @Before + void setUp() throws Exception { + + } + + @After + void tearDown() throws Exception { + + } + + private static Cipher getCipher(boolean encrypt = true, int keySize = 256, byte[] iv = [0x00] * IV_LENGTH) { + Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding") + String key = getKeyOfSize(keySize) + cipher.init((encrypt ? Cipher.ENCRYPT_MODE : Cipher.DECRYPT_MODE) as int, new SecretKeySpec(Hex.decode(key), "AES"), new IvParameterSpec(iv)) + logger.setup("Initialized a cipher in ${encrypt ? "encrypt" : "decrypt"} mode with a key of length ${keySize} bits") + cipher + } + + private static String getKeyOfSize(int keySize = 256) { + switch (keySize) { + case 128: + return KEY_128_HEX + case 192: + case 256: + if (Cipher.getMaxAllowedKeyLength("AES") < keySize) { + throw new IllegalArgumentException("The JCE unlimited strength cryptographic jurisdiction policies are not installed, so the max key size is 128 bits") + } + return KEY_256_HEX[0..<(keySize / 4)] + default: + throw new IllegalArgumentException("Key size ${keySize} bits is not valid") + } + } + + private static List getAvailableKeySizes() { + if (Cipher.getMaxAllowedKeyLength("AES") > 128) { + [128, 192, 256] + } else { + [128] + } + } + + private static String manipulateString(String input, int start = 0, int end = input?.length()) { + if ((input[start..end] as List).unique().size() == 1) { + throw new IllegalArgumentException("Can't manipulate a String where the entire range is identical [${input[start..end]}]") + } + List shuffled = input[start..end] as List + Collections.shuffle(shuffled) + String reconstituted = input[0.. CIPHER_TEXTS = KEY_SIZES.collectEntries { int keySize -> + SensitivePropertyProvider spp = new AESSensitivePropertyProvider(Hex.decode(getKeyOfSize(keySize))) + logger.info("Initialized ${spp.name} with key size ${keySize}") + [(keySize): spp.protect(PLAINTEXT)] + } + CIPHER_TEXTS.each { ks, ct -> logger.info("Encrypted for ${ks} length key: ${ct}") } + + // Assert + + // The IV generation is part of #protect, so the expected cipher text values must be generated after #protect has run + Map decryptionCiphers = CIPHER_TEXTS.collectEntries { int keySize, String cipherText -> + // The 12 byte IV is the first 16 Base64-encoded characters of the "complete" cipher text + byte[] iv = decoder.decode(cipherText[0..<16]) + [(keySize): getCipher(false, keySize, iv)] + } + Map plaintexts = decryptionCiphers.collectEntries { Map.Entry e -> + String cipherTextWithoutIVAndDelimiter = CIPHER_TEXTS[e.key][18..-1] + String plaintext = new String(e.value.doFinal(decoder.decode(cipherTextWithoutIVAndDelimiter)), StandardCharsets.UTF_8) + [(e.key): plaintext] + } + CIPHER_TEXTS.each { key, ct -> logger.expected("Cipher text for ${key} length key: ${ct}") } + + assert plaintexts.every { int ks, String pt -> pt == PLAINTEXT } + } + + @Test + void testShouldHandleProtectEmptyValue() throws Exception { + final List EMPTY_PLAINTEXTS = ["", " ", null] + + // Act + KEY_SIZES.collectEntries { int keySize -> + SensitivePropertyProvider spp = new AESSensitivePropertyProvider(Hex.decode(getKeyOfSize(keySize))) + logger.info("Initialized ${spp.name} with key size ${keySize}") + EMPTY_PLAINTEXTS.each { String emptyPlaintext -> + def msg = shouldFail(IllegalArgumentException) { + spp.protect(emptyPlaintext) + } + logger.expected("${msg} for keySize ${keySize} and plaintext [${emptyPlaintext}]") + + // Assert + assert msg == "Cannot encrypt an empty value" + } + } + } + + @Test + void testShouldUnprotectValue() throws Exception { + // Arrange + final String PLAINTEXT = "This is a plaintext value" + + Map encryptionCiphers = KEY_SIZES.collectEntries { int keySize -> + byte[] iv = new byte[IV_LENGTH] + secureRandom.nextBytes(iv) + [(keySize): getCipher(true, keySize, iv)] + } + + Map CIPHER_TEXTS = encryptionCiphers.collectEntries { Map.Entry e -> + String iv = encoder.encodeToString(e.value.getIV()) + String cipherText = encoder.encodeToString(e.value.doFinal(PLAINTEXT.getBytes(StandardCharsets.UTF_8))) + [(e.key): "${iv}||${cipherText}"] + } + CIPHER_TEXTS.each { key, ct -> logger.expected("Cipher text for ${key} length key: ${ct}") } + + // Act + Map plaintexts = CIPHER_TEXTS.collectEntries { int keySize, String cipherText -> + SensitivePropertyProvider spp = new AESSensitivePropertyProvider(Hex.decode(getKeyOfSize(keySize))) + logger.info("Initialized ${spp.name} with key size ${keySize}") + [(keySize): spp.unprotect(cipherText)] + } + plaintexts.each { ks, pt -> logger.info("Decrypted for ${ks} length key: ${pt}") } + + // Assert + assert plaintexts.every { int ks, String pt -> pt == PLAINTEXT } + } + + /** + * Tests inputs where the entire String is empty/blank space/{@code null}. + * + * @throws Exception + */ + @Test + void testShouldHandleUnprotectEmptyValue() throws Exception { + // Arrange + final List EMPTY_CIPHER_TEXTS = ["", " ", null] + + // Act + KEY_SIZES.each { int keySize -> + SensitivePropertyProvider spp = new AESSensitivePropertyProvider(Hex.decode(getKeyOfSize(keySize))) + logger.info("Initialized ${spp.name} with key size ${keySize}") + EMPTY_CIPHER_TEXTS.each { String emptyCipherText -> + def msg = shouldFail(IllegalArgumentException) { + spp.unprotect(emptyCipherText) + } + logger.expected("${msg} for keySize ${keySize} and cipher text [${emptyCipherText}]") + + // Assert + assert msg == "Cannot decrypt a cipher text shorter than ${AESSensitivePropertyProvider.minCipherTextLength} chars".toString() + } + } + } + + @Test + void testShouldUnprotectValueWithWhitespace() throws Exception { + // Arrange + final String PLAINTEXT = "This is a plaintext value" + + Map encryptionCiphers = KEY_SIZES.collectEntries { int keySize -> + byte[] iv = new byte[IV_LENGTH] + secureRandom.nextBytes(iv) + [(keySize): getCipher(true, keySize, iv)] + } + + Map CIPHER_TEXTS = encryptionCiphers.collectEntries { Map.Entry e -> + String iv = encoder.encodeToString(e.value.getIV()) + String cipherText = encoder.encodeToString(e.value.doFinal(PLAINTEXT.getBytes(StandardCharsets.UTF_8))) + [(e.key): "${iv}||${cipherText}"] + } + CIPHER_TEXTS.each { key, ct -> logger.expected("Cipher text for ${key} length key: ${ct}") } + + // Act + Map plaintexts = CIPHER_TEXTS.collectEntries { int keySize, String cipherText -> + SensitivePropertyProvider spp = new AESSensitivePropertyProvider(Hex.decode(getKeyOfSize(keySize))) + logger.info("Initialized ${spp.name} with key size ${keySize}") + [(keySize): spp.unprotect("\t" + cipherText + "\n")] + } + plaintexts.each { ks, pt -> logger.info("Decrypted for ${ks} length key: ${pt}") } + + // Assert + assert plaintexts.every { int ks, String pt -> pt == PLAINTEXT } + } + + @Test + void testShouldHandleUnprotectMalformedValue() throws Exception { + // Arrange + final String PLAINTEXT = "This is a plaintext value" + + // Act + KEY_SIZES.each { int keySize -> + SensitivePropertyProvider spp = new AESSensitivePropertyProvider(Hex.decode(getKeyOfSize(keySize))) + logger.info("Initialized ${spp.name} with key size ${keySize}") + String cipherText = spp.protect(PLAINTEXT) + // Swap two characters in the cipher text + final String MALFORMED_CIPHER_TEXT = manipulateString(cipherText, 25, 28) + logger.info("Manipulated ${cipherText} to\n${MALFORMED_CIPHER_TEXT.padLeft(163)}") + + def msg = shouldFail(SensitivePropertyProtectionException) { + spp.unprotect(MALFORMED_CIPHER_TEXT) + } + logger.expected("${msg} for keySize ${keySize} and cipher text [${MALFORMED_CIPHER_TEXT}]") + + // Assert + assert msg == "Error decrypting a protected value" + } + } + + @Test + void testShouldHandleUnprotectMissingIV() throws Exception { + // Arrange + final String PLAINTEXT = "This is a plaintext value" + + // Act + KEY_SIZES.each { int keySize -> + SensitivePropertyProvider spp = new AESSensitivePropertyProvider(Hex.decode(getKeyOfSize(keySize))) + logger.info("Initialized ${spp.name} with key size ${keySize}") + String cipherText = spp.protect(PLAINTEXT) + // Remove the IV from the "complete" cipher text + final String MISSING_IV_CIPHER_TEXT = cipherText[18..-1] + logger.info("Manipulated ${cipherText} to\n${MISSING_IV_CIPHER_TEXT.padLeft(163)}") + + def msg = shouldFail(IllegalArgumentException) { + spp.unprotect(MISSING_IV_CIPHER_TEXT) + } + logger.expected("${msg} for keySize ${keySize} and cipher text [${MISSING_IV_CIPHER_TEXT}]") + + // Remove the IV from the "complete" cipher text but keep the delimiter + final String MISSING_IV_CIPHER_TEXT_WITH_DELIMITER = cipherText[16..-1] + logger.info("Manipulated ${cipherText} to\n${MISSING_IV_CIPHER_TEXT_WITH_DELIMITER.padLeft(163)}") + + def msgWithDelimiter = shouldFail(DecoderException) { + spp.unprotect(MISSING_IV_CIPHER_TEXT_WITH_DELIMITER) + } + logger.expected("${msgWithDelimiter} for keySize ${keySize} and cipher text [${MISSING_IV_CIPHER_TEXT_WITH_DELIMITER}]") + + // Assert + assert msg == "The cipher text does not contain the delimiter || -- it should be of the form Base64(IV) || Base64(cipherText)" + + // Assert + assert msgWithDelimiter =~ "unable to decode base64 string" + } + } + + /** + * Tests inputs which have a valid IV and delimiter but no "cipher text". + * + * @throws Exception + */ + @Test + void testShouldHandleUnprotectEmptyCipherText() throws Exception { + // Arrange + final String IV_AND_DELIMITER = "${encoder.encodeToString("Bad IV value".getBytes(StandardCharsets.UTF_8))}||" + logger.info("IV and delimiter: ${IV_AND_DELIMITER}") + + final List EMPTY_CIPHER_TEXTS = ["", " ", "\n"].collect { "${IV_AND_DELIMITER}${it}" } + + // Act + KEY_SIZES.each { int keySize -> + SensitivePropertyProvider spp = new AESSensitivePropertyProvider(Hex.decode(getKeyOfSize(keySize))) + logger.info("Initialized ${spp.name} with key size ${keySize}") + EMPTY_CIPHER_TEXTS.each { String emptyCipherText -> + def msg = shouldFail(IllegalArgumentException) { + spp.unprotect(emptyCipherText) + } + logger.expected("${msg} for keySize ${keySize} and cipher text [${emptyCipherText}]") + + // Assert + assert msg == "Cannot decrypt a cipher text shorter than ${AESSensitivePropertyProvider.minCipherTextLength} chars".toString() + } + } + } + + @Test + void testShouldHandleUnprotectMalformedIV() throws Exception { + // Arrange + final String PLAINTEXT = "This is a plaintext value" + + // Act + KEY_SIZES.each { int keySize -> + SensitivePropertyProvider spp = new AESSensitivePropertyProvider(Hex.decode(getKeyOfSize(keySize))) + logger.info("Initialized ${spp.name} with key size ${keySize}") + String cipherText = spp.protect(PLAINTEXT) + // Swap two characters in the IV + final String MALFORMED_IV_CIPHER_TEXT = manipulateString(cipherText, 8, 11) + logger.info("Manipulated ${cipherText} to\n${MALFORMED_IV_CIPHER_TEXT.padLeft(163)}") + + def msg = shouldFail(SensitivePropertyProtectionException) { + spp.unprotect(MALFORMED_IV_CIPHER_TEXT) + } + logger.expected("${msg} for keySize ${keySize} and cipher text [${MALFORMED_IV_CIPHER_TEXT}]") + + // Assert + assert msg == "Error decrypting a protected value" + } + } + + @Test + void testShouldGetIdentifierKeyWithDifferentMaxKeyLengths() throws Exception { + // Arrange + def keys = getAvailableKeySizes().collectEntries { int keySize -> + [(keySize): getKeyOfSize(keySize)] + } + logger.info("Keys: ${keys}") + + // Act + keys.each { int size, String key -> + String identifierKey = new AESSensitivePropertyProvider(key).getIdentifierKey() + logger.info("Identifier key: ${identifierKey} for size ${size}") + + // Assert + assert identifierKey =~ /aes\/gcm\/${size}/ + } + } + + @Test + void testShouldNotAllowEmptyKey() throws Exception { + // Arrange + final String INVALID_KEY = "" + + // Act + def msg = shouldFail(SensitivePropertyProtectionException) { + AESSensitivePropertyProvider spp = new AESSensitivePropertyProvider(INVALID_KEY) + } + + // Assert + assert msg == "The key cannot be empty" + } + + @Test + void testShouldNotAllowIncorrectlySizedKey() throws Exception { + // Arrange + final String INVALID_KEY = "Z" * 31 + + // Act + def msg = shouldFail(SensitivePropertyProtectionException) { + AESSensitivePropertyProvider spp = new AESSensitivePropertyProvider(INVALID_KEY) + } + + // Assert + assert msg == "The key must be a valid hexadecimal key" + } + + @Test + void testShouldNotAllowInvalidKey() throws Exception { + // Arrange + final String INVALID_KEY = "Z" * 32 + + // Act + def msg = shouldFail(SensitivePropertyProtectionException) { + AESSensitivePropertyProvider spp = new AESSensitivePropertyProvider(INVALID_KEY) + } + + // Assert + assert msg == "The key must be a valid hexadecimal key" + } + + /** + * This test is to ensure internal consistency and allow for encrypting value for various property files + */ + @Test + void testShouldEncryptArbitraryValues() { + // Arrange + def values = ["thisIsABadPassword", "thisIsABadSensitiveKeyPassword", "thisIsABadKeystorePassword", "thisIsABadKeyPassword", "thisIsABadTruststorePassword", "This is an encrypted banner message", "nififtw!"] + + String key = "2C576A9585DB862F5ECBEE5B4FFFCCA1" //getKeyOfSize(128) + // key = "0" * 64 + + SensitivePropertyProvider spp = new AESSensitivePropertyProvider(key) + + // Act + def encryptedValues = values.collect { String v -> + def encryptedValue = spp.protect(v) + logger.info("${v} -> ${encryptedValue}") + def (String iv, String cipherText) = encryptedValue.tokenize("||") + logger.info("Normal Base64 encoding would be ${encoder.encodeToString(decoder.decode(iv))}||${encoder.encodeToString(decoder.decode(cipherText))}") + encryptedValue + } + + // Assert + assert values == encryptedValues.collect { spp.unprotect(it) } + } + + /** + * This test is to ensure external compatibility in case someone encodes the encrypted value with Base64 and does not remove the padding + */ + @Test + void testShouldDecryptPaddedValueWith256BitKey() { + // Arrange + Assume.assumeTrue("JCE unlimited strength crypto policy must be installed for this test", Cipher.getMaxAllowedKeyLength("AES") > 128) + + final String EXPECTED_VALUE = getKeyOfSize(256) // "thisIsABadKeyPassword" + String cipherText = "aYDkDKys1ENr3gp+||sTBPpMlIvHcOLTGZlfWct8r9RY8BuDlDkoaYmGJ/9m9af9tZIVzcnDwvYQAaIKxRGF7vI2yrY7Xd6x9GTDnWGiGiRXlaP458BBMMgfzH2O8" + String unpaddedCipherText = cipherText.replaceAll("=", "") + + String key = "AAAABBBBCCCCDDDDEEEEFFFF00001111" * 2 // getKeyOfSize(256) + + SensitivePropertyProvider spp = new AESSensitivePropertyProvider(key) + + // Act + String rawValue = spp.unprotect(cipherText) + logger.info("Decrypted ${cipherText} to ${rawValue}") + String rawUnpaddedValue = spp.unprotect(unpaddedCipherText) + logger.info("Decrypted ${unpaddedCipherText} to ${rawUnpaddedValue}") + + // Assert + assert rawValue == EXPECTED_VALUE + assert rawUnpaddedValue == EXPECTED_VALUE + } +} diff --git a/nifi-registry-properties/src/test/groovy/org/apache/nifi/registry/properties/NiFiRegistryPropertiesGroovyTest.groovy b/nifi-registry-properties/src/test/groovy/org/apache/nifi/registry/properties/NiFiRegistryPropertiesGroovyTest.groovy new file mode 100644 index 000000000..0c403cd5f --- /dev/null +++ b/nifi-registry-properties/src/test/groovy/org/apache/nifi/registry/properties/NiFiRegistryPropertiesGroovyTest.groovy @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.registry.properties + +import org.junit.* +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +@RunWith(JUnit4.class) +class NiFiRegistryPropertiesGroovyTest extends GroovyTestCase { + private static final Logger logger = LoggerFactory.getLogger(NiFiRegistryPropertiesGroovyTest.class) + + @BeforeClass + static void setUpOnce() throws Exception { + logger.metaClass.methodMissing = { String name, args -> + logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}") + } + } + + @Before + void setUp() throws Exception { + } + + @After + void tearDown() throws Exception { + } + + @AfterClass + static void tearDownOnce() { + } + + private static NiFiRegistryProperties loadFromFile(String propertiesFilePath) { + String filePath + try { + filePath = NiFiRegistryPropertiesGroovyTest.class.getResource(propertiesFilePath).toURI().getPath() + } catch (URISyntaxException ex) { + throw new RuntimeException("Cannot load properties file due to " + + ex.getLocalizedMessage(), ex) + } + + NiFiRegistryProperties properties = new NiFiRegistryProperties() + FileReader reader = new FileReader(filePath) + + try { + properties.load(reader) + logger.info("Loaded {} properties from {}", properties.size(), filePath) + + return properties + } catch (final Exception ex) { + logger.error("Cannot load properties file due to " + ex.getLocalizedMessage()) + throw new RuntimeException("Cannot load properties file due to " + + ex.getLocalizedMessage(), ex) + } + } + + @Test + void testConstructorShouldCreateNewInstance() throws Exception { + // Arrange + + // Act + NiFiRegistryProperties NiFiRegistryProperties = new NiFiRegistryProperties() + logger.info("NiFiRegistryProperties has ${NiFiRegistryProperties.size()} properties: ${NiFiRegistryProperties.getPropertyKeys()}") + + // Assert + assert NiFiRegistryProperties.size() == 0 + assert NiFiRegistryProperties.getPropertyKeys() == [] as Set + } + + @Test + void testConstructorShouldAcceptDefaultProperties() throws Exception { + // Arrange + Properties rawProperties = new Properties() + rawProperties.setProperty("key", "value") + logger.info("rawProperties has ${rawProperties.size()} properties: ${rawProperties.stringPropertyNames()}") + assert rawProperties.size() == 1 + + // Act + NiFiRegistryProperties NiFiRegistryProperties = new NiFiRegistryProperties(rawProperties) + logger.info("NiFiRegistryProperties has ${NiFiRegistryProperties.size()} properties: ${NiFiRegistryProperties.getPropertyKeys()}") + + // Assert + assert NiFiRegistryProperties.size() == 1 + assert NiFiRegistryProperties.getPropertyKeys() == ["key"] as Set + } + + @Test + void testShouldAllowMultipleInstances() throws Exception { + // Arrange + + // Act + NiFiRegistryProperties properties = new NiFiRegistryProperties() + properties.setProperty("key", "value") + logger.info("niFiProperties has ${properties.size()} properties: ${properties.getPropertyKeys()}") + NiFiRegistryProperties emptyProperties = new NiFiRegistryProperties() + logger.info("emptyProperties has ${emptyProperties.size()} properties: ${emptyProperties.getPropertyKeys()}") + + // Assert + assert properties.size() == 1 + assert properties.getPropertyKeys() == ["key"] as Set + + assert emptyProperties.size() == 0 + assert emptyProperties.getPropertyKeys() == [] as Set + } + +} diff --git a/nifi-registry-properties/src/test/groovy/org/apache/nifi/registry/properties/NiFiRegistryPropertiesLoaderGroovyTest.groovy b/nifi-registry-properties/src/test/groovy/org/apache/nifi/registry/properties/NiFiRegistryPropertiesLoaderGroovyTest.groovy new file mode 100644 index 000000000..58c8087d5 --- /dev/null +++ b/nifi-registry-properties/src/test/groovy/org/apache/nifi/registry/properties/NiFiRegistryPropertiesLoaderGroovyTest.groovy @@ -0,0 +1,264 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.registry.properties + +import org.bouncycastle.jce.provider.BouncyCastleProvider +import org.junit.After +import org.junit.AfterClass +import org.junit.Before +import org.junit.BeforeClass +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +import javax.crypto.Cipher +import java.security.Security + +@RunWith(JUnit4.class) +class NiFiRegistryPropertiesLoaderGroovyTest extends GroovyTestCase { + + private static final Logger logger = LoggerFactory.getLogger(NiFiRegistryPropertiesLoaderGroovyTest.class) + + private static final String KEYSTORE_PASSWORD_KEY = NiFiRegistryProperties.SECURITY_KEYSTORE_PASSWD + private static final String KEY_PASSWORD_KEY = NiFiRegistryProperties.SECURITY_KEY_PASSWD + private static final String TRUSTSTORE_PASSWORD_KEY = NiFiRegistryProperties.SECURITY_TRUSTSTORE_PASSWD + + private static final String KEY_HEX_128 = "0123456789ABCDEFFEDCBA9876543210" + private static final String KEY_HEX_256 = KEY_HEX_128 * 2 + private static final String KEY_HEX = Cipher.getMaxAllowedKeyLength("AES") < 256 ? KEY_HEX_128 : KEY_HEX_256 + + private static final String PASSWORD_KEY_HEX_128 = "2C576A9585DB862F5ECBEE5B4FFFCCA1" + + @BeforeClass + public static void setUpOnce() throws Exception { + Security.addProvider(new BouncyCastleProvider()) + + logger.metaClass.methodMissing = { String name, args -> + logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}") + } + } + + @Before + public void setUp() throws Exception { + } + + @After + public void tearDown() throws Exception { + // Clear the sensitive property providers between runs + NiFiRegistryPropertiesLoader.@sensitivePropertyProviderFactory = null + } + + @AfterClass + public static void tearDownOnce() { + } + + @Test + public void testConstructorShouldCreateNewInstance() throws Exception { + // Arrange + + // Act + NiFiRegistryPropertiesLoader propertiesLoader = new NiFiRegistryPropertiesLoader() + + // Assert + assert !propertiesLoader.@keyHex + } + + @Test + public void testShouldCreateInstanceWithKey() throws Exception { + // Arrange + + // Act + NiFiRegistryPropertiesLoader propertiesLoader = NiFiRegistryPropertiesLoader.withKey(KEY_HEX) + + // Assert + assert propertiesLoader.@keyHex == KEY_HEX + } + + @Test + public void testConstructorShouldCreateMultipleInstances() throws Exception { + // Arrange + NiFiRegistryPropertiesLoader propertiesLoader1 = NiFiRegistryPropertiesLoader.withKey(KEY_HEX) + + // Act + NiFiRegistryPropertiesLoader propertiesLoader2 = new NiFiRegistryPropertiesLoader() + + // Assert + assert propertiesLoader1.@keyHex == KEY_HEX + assert !propertiesLoader2.@keyHex + } + + @Test + public void testShouldGetDefaultProviderKey() throws Exception { + // Arrange + final String expectedProviderKey = "aes/gcm/${Cipher.getMaxAllowedKeyLength("AES") > 128 ? 256 : 128}" + logger.info("Expected provider key: ${expectedProviderKey}") + + // Act + String defaultKey = NiFiRegistryPropertiesLoader.getDefaultProviderKey() + logger.info("Default key: ${defaultKey}") + // Assert + assert defaultKey == expectedProviderKey + } + + @Test + public void testShouldInitializeSensitivePropertyProviderFactory() throws Exception { + // Arrange + NiFiRegistryPropertiesLoader propertiesLoader = new NiFiRegistryPropertiesLoader() + + // Act + propertiesLoader.initializeSensitivePropertyProviderFactory() + + // Assert + assert propertiesLoader.@sensitivePropertyProviderFactory + } + + @Test + public void testShouldLoadUnprotectedPropertiesFromFile() throws Exception { + // Arrange + File unprotectedFile = new File("src/test/resources/conf/nifi-registry.properties") + NiFiRegistryPropertiesLoader propertiesLoader = new NiFiRegistryPropertiesLoader() + + // Act + NiFiRegistryProperties properties = propertiesLoader.load(unprotectedFile) + + // Assert + assert properties.size() > 0 + + // Ensure it is not a ProtectedNiFiProperties + assert properties instanceof NiFiRegistryProperties + } + + @Test + public void testShouldNotLoadUnprotectedPropertiesFromNullFile() throws Exception { + // Arrange + NiFiRegistryPropertiesLoader propertiesLoader = new NiFiRegistryPropertiesLoader() + + // Act + def msg = shouldFail(IllegalArgumentException) { + NiFiRegistryProperties properties = propertiesLoader.load(null as File) + } + logger.info(msg) + + // Assert + assert msg == "NiFi Registry properties file missing or unreadable" + } + + @Test + public void testShouldNotLoadUnprotectedPropertiesFromMissingFile() throws Exception { + // Arrange + File missingFile = new File("src/test/resources/conf/nifi-registry.missing.properties") + assert !missingFile.exists() + + NiFiRegistryPropertiesLoader propertiesLoader = new NiFiRegistryPropertiesLoader() + + // Act + def msg = shouldFail(IllegalArgumentException) { + NiFiRegistryProperties properties = propertiesLoader.load(missingFile) + } + logger.info(msg) + + // Assert + assert msg == "NiFi Registry properties file missing or unreadable" + } + + @Test + public void testShouldLoadUnprotectedPropertiesFromPath() throws Exception { + // Arrange + File unprotectedFile = new File("src/test/resources/conf/nifi-registry.properties") + NiFiRegistryPropertiesLoader propertiesLoader = new NiFiRegistryPropertiesLoader() + + // Act + NiFiRegistryProperties properties = propertiesLoader.load(unprotectedFile.path) + + // Assert + assert properties.size() > 0 + + // Ensure it is not a ProtectedNiFiProperties + assert properties instanceof NiFiRegistryProperties + } + + @Test + public void testShouldLoadUnprotectedPropertiesFromProtectedFile() throws Exception { + // Arrange + File protectedFile = new File("src/test/resources/conf/nifi-registry.with_sensitive_props_protected_aes_128.properties") + NiFiRegistryPropertiesLoader propertiesLoader = NiFiRegistryPropertiesLoader.withKey(KEY_HEX_128) + + final def EXPECTED_PLAIN_VALUES = [ + (KEYSTORE_PASSWORD_KEY): "thisIsABadPassword", + (KEY_PASSWORD_KEY): "thisIsABadPassword", + ] + + // This method is covered in tests above, so safe to use here to retrieve protected properties + ProtectedNiFiRegistryProperties protectedNiFiProperties = propertiesLoader.readProtectedPropertiesFromDisk(protectedFile) + int totalKeysCount = protectedNiFiProperties.getPropertyKeysIncludingProtectionSchemes().size() + int protectedKeysCount = protectedNiFiProperties.getProtectedPropertyKeys().size() + logger.info("Read ${totalKeysCount} total properties (${protectedKeysCount} protected) from ${protectedFile.canonicalPath}") + + // Act + NiFiRegistryProperties properties = propertiesLoader.load(protectedFile) + + // Assert + assert properties.size() == totalKeysCount - protectedKeysCount + + // Ensure that any key marked as protected above is different in this instance + protectedNiFiProperties.getProtectedPropertyKeys().keySet().each { String key -> + String plainValue = properties.getProperty(key) + String protectedValue = protectedNiFiProperties.getProperty(key) + + logger.info("Checking that [${protectedValue}] -> [${plainValue}] == [${EXPECTED_PLAIN_VALUES[key]}]") + + assert plainValue == EXPECTED_PLAIN_VALUES[key] + assert plainValue != protectedValue + assert plainValue.length() <= protectedValue.length() + } + + // Ensure it is not a ProtectedNiFiProperties + assert properties instanceof NiFiRegistryProperties + } + + @Test + public void testShouldUpdateKeyInFactory() throws Exception { + // Arrange + File originalKeyFile = new File("src/test/resources/conf/nifi-registry.with_sensitive_props_protected_aes_128.properties") + File passwordKeyFile = new File("src/test/resources/conf/nifi-registry.with_sensitive_props_protected_aes_128_password.properties") + NiFiRegistryPropertiesLoader propertiesLoader = NiFiRegistryPropertiesLoader.withKey(KEY_HEX_128) + + NiFiRegistryProperties properties = propertiesLoader.load(originalKeyFile) + logger.info("Read ${properties.size()} total properties from ${originalKeyFile.canonicalPath}") + + // Act + NiFiRegistryPropertiesLoader passwordNiFiRegistryPropertiesLoader = NiFiRegistryPropertiesLoader.withKey(PASSWORD_KEY_HEX_128) + + NiFiRegistryProperties passwordProperties = passwordNiFiRegistryPropertiesLoader.load(passwordKeyFile) + logger.info("Read ${passwordProperties.size()} total properties from ${passwordKeyFile.canonicalPath}") + + // Assert + assert properties.size() == passwordProperties.size() + + + def readPropertiesAndValues = properties.getPropertyKeys().collectEntries { + [(it): properties.getProperty(it)] + } + def readPasswordPropertiesAndValues = passwordProperties.getPropertyKeys().collectEntries { + [(it): passwordProperties.getProperty(it)] + } + + assert readPropertiesAndValues == readPasswordPropertiesAndValues + } +} diff --git a/nifi-registry-properties/src/test/groovy/org/apache/nifi/registry/properties/ProtectedNiFiPropertiesGroovyTest.groovy b/nifi-registry-properties/src/test/groovy/org/apache/nifi/registry/properties/ProtectedNiFiPropertiesGroovyTest.groovy new file mode 100644 index 000000000..86c7fb402 --- /dev/null +++ b/nifi-registry-properties/src/test/groovy/org/apache/nifi/registry/properties/ProtectedNiFiPropertiesGroovyTest.groovy @@ -0,0 +1,739 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.registry.properties + +import org.bouncycastle.jce.provider.BouncyCastleProvider +import org.junit.After +import org.junit.AfterClass +import org.junit.Assume +import org.junit.Before +import org.junit.BeforeClass +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +import javax.crypto.Cipher +import java.security.Security + +@RunWith(JUnit4.class) +class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase { + private static final Logger logger = LoggerFactory.getLogger(ProtectedNiFiPropertiesGroovyTest.class) + + private static final String KEYSTORE_PASSWORD_KEY = NiFiRegistryProperties.SECURITY_KEYSTORE_PASSWD + private static final String KEY_PASSWORD_KEY = NiFiRegistryProperties.SECURITY_KEY_PASSWD + private static final String TRUSTSTORE_PASSWORD_KEY = NiFiRegistryProperties.SECURITY_TRUSTSTORE_PASSWD + + private static final def DEFAULT_SENSITIVE_PROPERTIES = [ + KEYSTORE_PASSWORD_KEY, + KEY_PASSWORD_KEY, + TRUSTSTORE_PASSWORD_KEY + ] + + private static final String KEY_HEX_128 = "0123456789ABCDEFFEDCBA9876543210" + private static final String KEY_HEX_256 = KEY_HEX_128 * 2 + private static final String KEY_HEX = Cipher.getMaxAllowedKeyLength("AES") < 256 ? KEY_HEX_128 : KEY_HEX_256 + + @BeforeClass + static void setUpOnce() throws Exception { + Security.addProvider(new BouncyCastleProvider()) + + logger.metaClass.methodMissing = { String name, args -> + logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}") + } + } + + @Before + void setUp() throws Exception { + } + + @After + void tearDown() throws Exception { + } + + @AfterClass + static void tearDownOnce() { + } + + private static ProtectedNiFiRegistryProperties loadFromResourceFile(String propertiesFilePath) { + return loadFromResourceFile(propertiesFilePath, KEY_HEX) + } + + private static ProtectedNiFiRegistryProperties loadFromResourceFile(String propertiesFilePath, String keyHex) { + File file = fileForResource(propertiesFilePath) + + if (file == null || !file.exists() || !file.canRead()) { + String path = (file == null ? "missing file" : file.getAbsolutePath()) + logger.error("Cannot read from '{}' -- file is missing or not readable", path) + throw new IllegalArgumentException("NiFi Registry properties file missing or unreadable") + } + + NiFiRegistryProperties properties = new NiFiRegistryProperties() + FileReader reader = new FileReader(file) + + try { + properties.load(reader) + logger.info("Loaded {} properties from {}", properties.size(), file.getAbsolutePath()) + + ProtectedNiFiRegistryProperties protectedNiFiProperties = new ProtectedNiFiRegistryProperties(properties) + + // If it has protected keys, inject the SPP + if (protectedNiFiProperties.hasProtectedKeys()) { + protectedNiFiProperties.addSensitivePropertyProvider(new AESSensitivePropertyProvider(keyHex)) + } + + return protectedNiFiProperties + } catch (final Exception ex) { + logger.error("Cannot load properties file due to " + ex.getLocalizedMessage()) + throw new RuntimeException("Cannot load properties file due to " + + ex.getLocalizedMessage(), ex) + } + } + + private static File fileForResource(String resourcePath) { + String filePath + try { + URL resourceURL = ProtectedNiFiPropertiesGroovyTest.class.getResource(resourcePath) + if (!resourceURL) { + throw new RuntimeException("File ${resourcePath} not found in class resources, cannot load.") + } + filePath = resourceURL.toURI().getPath() + } catch (URISyntaxException ex) { + throw new RuntimeException("Cannot load resource file due to " + + ex.getLocalizedMessage(), ex) + } + File file = new File(filePath) + return file + } + + @Test + void testShouldDetectIfPropertyIsSensitive() throws Exception { + // Arrange + final String INSENSITIVE_PROPERTY_KEY = "nifi.registry.web.http.port" + final String SENSITIVE_PROPERTY_KEY = "nifi.registry.security.keystorePasswd" + + ProtectedNiFiRegistryProperties properties = loadFromResourceFile("/conf/nifi-registry.properties") + + // Act + boolean bannerIsSensitive = properties.isPropertySensitive(INSENSITIVE_PROPERTY_KEY) + logger.info("${INSENSITIVE_PROPERTY_KEY} is ${bannerIsSensitive ? "SENSITIVE" : "NOT SENSITIVE"}") + boolean passwordIsSensitive = properties.isPropertySensitive(SENSITIVE_PROPERTY_KEY) + logger.info("${SENSITIVE_PROPERTY_KEY} is ${passwordIsSensitive ? "SENSITIVE" : "NOT SENSITIVE"}") + + // Assert + assert !bannerIsSensitive + assert passwordIsSensitive + } + + @Test + void testShouldGetDefaultSensitiveProperties() throws Exception { + // Arrange + logger.info("${DEFAULT_SENSITIVE_PROPERTIES.size()} default sensitive properties: ${DEFAULT_SENSITIVE_PROPERTIES.join(", ")}") + + ProtectedNiFiRegistryProperties properties = loadFromResourceFile("/conf/nifi-registry.properties") + + // Act + List defaultSensitiveProperties = properties.getSensitivePropertyKeys() + logger.info("${defaultSensitiveProperties.size()} default sensitive properties: ${defaultSensitiveProperties.join(", ")}") + + // Assert + assert defaultSensitiveProperties.size() == DEFAULT_SENSITIVE_PROPERTIES.size() + assert defaultSensitiveProperties.containsAll(DEFAULT_SENSITIVE_PROPERTIES) + } + + @Test + void testShouldGetAdditionalSensitiveProperties() throws Exception { + // Arrange + def completeSensitiveProperties = DEFAULT_SENSITIVE_PROPERTIES + ["nifi.registry.web.http.port", "nifi.registry.web.http.host"] + logger.info("${completeSensitiveProperties.size()} total sensitive properties: ${completeSensitiveProperties.join(", ")}") + + ProtectedNiFiRegistryProperties properties = loadFromResourceFile("/conf/nifi-registry.with_additional_sensitive_keys.properties") + + // Act + List retrievedSensitiveProperties = properties.getSensitivePropertyKeys() + logger.info("${retrievedSensitiveProperties.size()} retrieved sensitive properties: ${retrievedSensitiveProperties.join(", ")}") + + // Assert + assert retrievedSensitiveProperties.size() == completeSensitiveProperties.size() + assert retrievedSensitiveProperties.containsAll(completeSensitiveProperties) + } + + @Test + void testGetAdditionalSensitivePropertiesShouldNotIncludeSelf() throws Exception { + // Arrange + def completeSensitiveProperties = DEFAULT_SENSITIVE_PROPERTIES + ["nifi.registry.web.http.port", "nifi.registry.web.http.host"] + logger.info("${completeSensitiveProperties.size()} total sensitive properties: ${completeSensitiveProperties.join(", ")}") + + ProtectedNiFiRegistryProperties properties = loadFromResourceFile("/conf/nifi-registry.with_additional_sensitive_keys.properties") + + // Act + List retrievedSensitiveProperties = properties.getSensitivePropertyKeys() + logger.info("${retrievedSensitiveProperties.size()} retrieved sensitive properties: ${retrievedSensitiveProperties.join(", ")}") + + // Assert + assert retrievedSensitiveProperties.size() == completeSensitiveProperties.size() + assert retrievedSensitiveProperties.containsAll(completeSensitiveProperties) + } + + /** + * In the default (no protection enabled) scenario, a call to retrieve a sensitive property should return the raw value transparently. + * @throws Exception + */ + @Test + void testShouldGetUnprotectedValueOfSensitiveProperty() throws Exception { + // Arrange + final String expectedKeystorePassword = "thisIsABadKeystorePassword" + + ProtectedNiFiRegistryProperties properties = loadFromResourceFile("/conf/nifi-registry.with_sensitive_props_unprotected.properties") + + boolean isSensitive = properties.isPropertySensitive(KEYSTORE_PASSWORD_KEY) + boolean isProtected = properties.isPropertyProtected(KEYSTORE_PASSWORD_KEY) + logger.info("The property is ${isSensitive ? "sensitive" : "not sensitive"} and ${isProtected ? "protected" : "not protected"}") + + // Act + String retrievedKeystorePassword = properties.getProperty(KEYSTORE_PASSWORD_KEY) + logger.info("${KEYSTORE_PASSWORD_KEY}: ${retrievedKeystorePassword}") + + // Assert + assert retrievedKeystorePassword == expectedKeystorePassword + assert isSensitive + assert !isProtected + } + + /** + * In the default (no protection enabled) scenario, a call to retrieve a sensitive property (which is empty) should return the raw value transparently. + * @throws Exception + */ + @Test + void testShouldGetEmptyUnprotectedValueOfSensitiveProperty() throws Exception { + // Arrange + final String expectedTruststorePassword = "" + + ProtectedNiFiRegistryProperties properties = loadFromResourceFile("/conf/nifi-registry.with_sensitive_props_unprotected.properties") + + boolean isSensitive = properties.isPropertySensitive(TRUSTSTORE_PASSWORD_KEY) + boolean isProtected = properties.isPropertyProtected(TRUSTSTORE_PASSWORD_KEY) + logger.info("The property is ${isSensitive ? "sensitive" : "not sensitive"} and ${isProtected ? "protected" : "not protected"}") + + // Act + NiFiRegistryProperties unprotectedProperties = properties.getUnprotectedProperties() + String retrievedTruststorePassword = unprotectedProperties.getProperty(TRUSTSTORE_PASSWORD_KEY) + logger.info("${TRUSTSTORE_PASSWORD_KEY}: ${retrievedTruststorePassword}") + + // Assert + assert retrievedTruststorePassword == expectedTruststorePassword + assert isSensitive + assert !isProtected + } + + /** + * The new model no longer needs to maintain the protected state -- it is used as a wrapper/decorator during load to unprotect the sensitive properties and then return an instance of raw properties. + * + * @throws Exception + */ + @Test + void testShouldGetUnprotectedValueOfSensitivePropertyWhenProtected() throws Exception { + // Arrange + final String expectedKeystorePassword = "thisIsABadPassword" + + ProtectedNiFiRegistryProperties properties = loadFromResourceFile("/conf/nifi-registry.with_sensitive_props_protected_aes_128.properties", KEY_HEX_128) + + boolean isSensitive = properties.isPropertySensitive(KEYSTORE_PASSWORD_KEY) + boolean isProtected = properties.isPropertyProtected(KEYSTORE_PASSWORD_KEY) + logger.info("The property is ${isSensitive ? "sensitive" : "not sensitive"} and ${isProtected ? "protected" : "not protected"}") + + // Act + NiFiRegistryProperties unprotectedProperties = properties.getUnprotectedProperties() + String retrievedKeystorePassword = unprotectedProperties.getProperty(KEYSTORE_PASSWORD_KEY) + logger.info("${KEYSTORE_PASSWORD_KEY}: ${retrievedKeystorePassword}") + + // Assert + assert retrievedKeystorePassword == expectedKeystorePassword + assert isSensitive + assert isProtected + } + + /** + * In the protection enabled scenario, a call to retrieve a sensitive property should handle if the property is protected with an unknown protection scheme. + * @throws Exception + */ + @Test + void testGetValueOfSensitivePropertyShouldHandleUnknownProtectionScheme() throws Exception { + // Arrange + + // Raw properties + Properties rawProperties = new Properties() + rawProperties.load(new FileReader(fileForResource("/conf/nifi-registry.with_sensitive_props_protected_unknown.properties"))) + final String expectedKeystorePassword = rawProperties.getProperty(KEYSTORE_PASSWORD_KEY) + logger.info("Raw value for ${KEYSTORE_PASSWORD_KEY}: ${expectedKeystorePassword}") + + ProtectedNiFiRegistryProperties properties = loadFromResourceFile("/conf/nifi-registry.with_sensitive_props_protected_unknown.properties") + + boolean isSensitive = properties.isPropertySensitive(KEYSTORE_PASSWORD_KEY) + boolean isProtected = properties.isPropertyProtected(KEYSTORE_PASSWORD_KEY) + + // While the value is "protected", the scheme is not registered, so treat it as raw + logger.info("The property is ${isSensitive ? "sensitive" : "not sensitive"} and ${isProtected ? "protected" : "not protected"}") + + // Act + NiFiRegistryProperties unprotectedProperties = properties.getUnprotectedProperties() + String retrievedKeystorePassword = unprotectedProperties.getProperty(KEYSTORE_PASSWORD_KEY) + logger.info("${KEYSTORE_PASSWORD_KEY}: ${retrievedKeystorePassword}") + + // Assert + assert retrievedKeystorePassword == expectedKeystorePassword + assert isSensitive + assert isProtected + } + + /** + * In the protection enabled scenario, a call to retrieve a sensitive property should handle if the property is unable to be unprotected due to a malformed value. + * @throws Exception + */ + @Test + void testGetValueOfSensitivePropertyShouldHandleSingleMalformedValue() throws Exception { + // Arrange + ProtectedNiFiRegistryProperties properties = loadFromResourceFile("/conf/nifi-registry.with_sensitive_props_protected_aes_single_malformed.properties", KEY_HEX_128) + boolean isSensitive = properties.isPropertySensitive(KEYSTORE_PASSWORD_KEY) + boolean isProtected = properties.isPropertyProtected(KEYSTORE_PASSWORD_KEY) + logger.info("The property is ${isSensitive ? "sensitive" : "not sensitive"} and ${isProtected ? "protected" : "not protected"}") + + // Act + def msg = shouldFail(SensitivePropertyProtectionException) { + NiFiRegistryProperties unprotectedProperties = properties.getUnprotectedProperties() + String retrievedKeystorePassword = unprotectedProperties.getProperty(KEYSTORE_PASSWORD_KEY) + logger.info("${KEYSTORE_PASSWORD_KEY}: ${retrievedKeystorePassword}") + } + logger.info(msg) + + // Assert + assert msg =~ "Failed to unprotect key ${KEYSTORE_PASSWORD_KEY}" + assert isSensitive + assert isProtected + } + + /** + * In the protection enabled scenario, a call to retrieve a sensitive property should handle if the property is unable to be unprotected due to a malformed value. + * @throws Exception + */ + @Test + void testGetValueOfSensitivePropertyShouldHandleMultipleMalformedValues() throws Exception { + // Arrange + + // Raw properties + ProtectedNiFiRegistryProperties properties = + loadFromResourceFile("/conf/nifi-registry.with_sensitive_props_protected_aes_multiple_malformed.properties", KEY_HEX_128) + + // Iterate over the protected keys and track the ones that fail to decrypt + SensitivePropertyProvider spp = new AESSensitivePropertyProvider(KEY_HEX_128) + Set malformedKeys = properties.getProtectedPropertyKeys() + .findAll { String key, String scheme -> scheme == spp.identifierKey } + .keySet() + .findAll { String key -> + try { + spp.unprotect(properties.getProperty(key)) + return false + } catch (SensitivePropertyProtectionException e) { + logger.expected("Caught a malformed value for ${key}") + return true + } + } + + logger.expected("Malformed keys: ${malformedKeys.join(", ")}") + + // Act + def e = groovy.test.GroovyAssert.shouldFail(SensitivePropertyProtectionException) { + NiFiRegistryProperties unprotectedProperties = properties.getUnprotectedProperties() + String retrievedKeystorePassword = unprotectedProperties.getProperty(KEYSTORE_PASSWORD_KEY) + logger.info("${KEYSTORE_PASSWORD_KEY}: ${retrievedKeystorePassword}") + } + logger.expected(e.getMessage()) + + // Assert + assert e instanceof MultipleSensitivePropertyProtectionException + assert e.getMessage() =~ "Failed to unprotect keys" + assert e.getFailedKeys() == malformedKeys + + } + + /** + * In the protection enabled scenario, a call to retrieve a sensitive property should handle if the internal cache of providers is empty. + * @throws Exception + */ + @Test + void testGetValueOfSensitivePropertyShouldHandleInvalidatedInternalCache() throws Exception { + // Arrange + ProtectedNiFiRegistryProperties properties = + loadFromResourceFile("/conf/nifi-registry.with_sensitive_props_protected_aes_128.properties", KEY_HEX_128) + final String expectedKeystorePassword = properties.getProperty(KEYSTORE_PASSWORD_KEY) + logger.info("Read raw value from properties: ${expectedKeystorePassword}") + + // Overwrite the internal cache + properties.localProviderCache = [:] + + boolean isSensitive = properties.isPropertySensitive(KEYSTORE_PASSWORD_KEY) + boolean isProtected = properties.isPropertyProtected(KEYSTORE_PASSWORD_KEY) + logger.info("The property is ${isSensitive ? "sensitive" : "not sensitive"} and ${isProtected ? "protected" : "not protected"}") + + // Act + NiFiRegistryProperties unprotectedProperties = properties.getUnprotectedProperties() + String retrievedKeystorePassword = unprotectedProperties.getProperty(KEYSTORE_PASSWORD_KEY) + logger.info("${KEYSTORE_PASSWORD_KEY}: ${retrievedKeystorePassword}") + + // Assert + assert retrievedKeystorePassword == expectedKeystorePassword + assert isSensitive + assert isProtected + } + + @Test + void testShouldDetectIfPropertyIsProtected() throws Exception { + // Arrange + final String unprotectedPropertyKey = TRUSTSTORE_PASSWORD_KEY + final String protectedPropertyKey = KEYSTORE_PASSWORD_KEY + + ProtectedNiFiRegistryProperties properties = + loadFromResourceFile("/conf/nifi-registry.with_sensitive_props_protected_aes_128.properties", KEY_HEX_128) + + // Act + boolean unprotectedPasswordIsSensitive = properties.isPropertySensitive(unprotectedPropertyKey) + boolean unprotectedPasswordIsProtected = properties.isPropertyProtected(unprotectedPropertyKey) + logger.info("${unprotectedPropertyKey} is ${unprotectedPasswordIsSensitive ? "SENSITIVE" : "NOT SENSITIVE"}") + logger.info("${unprotectedPropertyKey} is ${unprotectedPasswordIsProtected ? "PROTECTED" : "NOT PROTECTED"}") + boolean protectedPasswordIsSensitive = properties.isPropertySensitive(protectedPropertyKey) + boolean protectedPasswordIsProtected = properties.isPropertyProtected(protectedPropertyKey) + logger.info("${protectedPropertyKey} is ${protectedPasswordIsSensitive ? "SENSITIVE" : "NOT SENSITIVE"}") + logger.info("${protectedPropertyKey} is ${protectedPasswordIsProtected ? "PROTECTED" : "NOT PROTECTED"}") + + // Assert + assert unprotectedPasswordIsSensitive + assert !unprotectedPasswordIsProtected + + assert protectedPasswordIsSensitive + assert protectedPasswordIsProtected + } + + @Test + void testShouldDetectIfPropertyWithEmptyProtectionSchemeIsProtected() throws Exception { + // Arrange + final String unprotectedPropertyKey = KEY_PASSWORD_KEY + ProtectedNiFiRegistryProperties properties = + loadFromResourceFile("/conf/nifi-registry.with_sensitive_props_unprotected_extra_line.properties") + + // Act + boolean unprotectedPasswordIsSensitive = properties.isPropertySensitive(unprotectedPropertyKey) + boolean unprotectedPasswordIsProtected = properties.isPropertyProtected(unprotectedPropertyKey) + logger.info("${unprotectedPropertyKey} is ${unprotectedPasswordIsSensitive ? "SENSITIVE" : "NOT SENSITIVE"}") + logger.info("${unprotectedPropertyKey} is ${unprotectedPasswordIsProtected ? "PROTECTED" : "NOT PROTECTED"}") + + // Assert + assert unprotectedPasswordIsSensitive + assert !unprotectedPasswordIsProtected + } + + @Test + void testShouldGetPercentageOfSensitivePropertiesProtected_0() throws Exception { + // Arrange + ProtectedNiFiRegistryProperties properties = loadFromResourceFile("/conf/nifi-registry.properties") + + logger.info("Sensitive property keys: ${properties.getSensitivePropertyKeys()}") + logger.info("Protected property keys: ${properties.getProtectedPropertyKeys().keySet()}") + + // Act + double percentProtected = properties.getPercentOfSensitivePropertiesProtected() + logger.info("${percentProtected}% (${properties.getProtectedPropertyKeys().size()} of ${properties.getPopulatedSensitivePropertyKeys().size()}) protected") + + // Assert + assert percentProtected == 0.0 + } + + @Test + void testShouldGetPercentageOfSensitivePropertiesProtected_75() throws Exception { + // Arrange + ProtectedNiFiRegistryProperties properties = + loadFromResourceFile("/conf/nifi-registry.with_sensitive_props_protected_aes_128.properties", KEY_HEX_128) + + logger.info("Sensitive property keys: ${properties.getSensitivePropertyKeys()}") + logger.info("Protected property keys: ${properties.getProtectedPropertyKeys().keySet()}") + + // Act + double percentProtected = properties.getPercentOfSensitivePropertiesProtected() + logger.info("${percentProtected}% (${properties.getProtectedPropertyKeys().size()} of ${properties.getPopulatedSensitivePropertyKeys().size()}) protected") + + // Assert + assert percentProtected == 67.0 + } + + @Test + void testShouldGetPercentageOfSensitivePropertiesProtected_100() throws Exception { + // Arrange + ProtectedNiFiRegistryProperties properties = + loadFromResourceFile("/conf/nifi-registry.with_sensitive_props_fully_protected_aes_128.properties", KEY_HEX_128) + + logger.info("Sensitive property keys: ${properties.getSensitivePropertyKeys()}") + logger.info("Protected property keys: ${properties.getProtectedPropertyKeys().keySet()}") + + // Act + double percentProtected = properties.getPercentOfSensitivePropertiesProtected() + logger.info("${percentProtected}% (${properties.getProtectedPropertyKeys().size()} of ${properties.getPopulatedSensitivePropertyKeys().size()}) protected") + + // Assert + assert percentProtected == 100.0 + } + + @Test + void testInstanceWithNoProtectedPropertiesShouldNotLoadSPP() throws Exception { + // Arrange + ProtectedNiFiRegistryProperties properties = loadFromResourceFile("/conf/nifi-registry.properties") + assert properties.@localProviderCache?.isEmpty() + + logger.info("Has protected properties: ${properties.hasProtectedKeys()}") + assert !properties.hasProtectedKeys() + + // Act + Map localCache = properties.@localProviderCache + logger.info("Internal cache ${localCache} has ${localCache.size()} providers loaded") + + // Assert + assert localCache.isEmpty() + } + + @Test + void testShouldAddSensitivePropertyProvider() throws Exception { + // Arrange + ProtectedNiFiRegistryProperties properties = new ProtectedNiFiRegistryProperties() + assert properties.getSensitivePropertyProviders().isEmpty() + + SensitivePropertyProvider mockProvider = + [unprotect : { String input -> + logger.mock("Mock call to #unprotect(${input})") + input.reverse() + }, + getIdentifierKey: { -> "mockProvider" }] as SensitivePropertyProvider + + // Act + properties.addSensitivePropertyProvider(mockProvider) + + // Assert + assert properties.getSensitivePropertyProviders().size() == 1 + } + + @Test + void testShouldNotAddNullSensitivePropertyProvider() throws Exception { + // Arrange + ProtectedNiFiRegistryProperties properties = new ProtectedNiFiRegistryProperties() + assert properties.@localProviderCache?.isEmpty() + + // Act + def msg = shouldFail(IllegalArgumentException) { + properties.addSensitivePropertyProvider(null) + } + logger.info(msg) + + // Assert + assert properties.getSensitivePropertyProviders().size() == 0 + assert msg == "Cannot add null SensitivePropertyProvider" + } + + @Test + void testShouldNotAllowOverwriteOfProvider() throws Exception { + // Arrange + ProtectedNiFiRegistryProperties properties = new ProtectedNiFiRegistryProperties() + assert properties.getSensitivePropertyProviders().isEmpty() + + SensitivePropertyProvider mockProvider = + [unprotect : { String input -> + logger.mock("Mock call to 1#unprotect(${input})") + input.reverse() + }, + getIdentifierKey: { -> "mockProvider" }] as SensitivePropertyProvider + properties.addSensitivePropertyProvider(mockProvider) + assert properties.getSensitivePropertyProviders().size() == 1 + + SensitivePropertyProvider mockProvider2 = + [unprotect : { String input -> + logger.mock("Mock call to 2#unprotect(${input})") + input.reverse() + }, + getIdentifierKey: { -> "mockProvider" }] as SensitivePropertyProvider + + // Act + def msg = shouldFail(UnsupportedOperationException) { + properties.addSensitivePropertyProvider(mockProvider2) + } + logger.info(msg) + + // Assert + assert msg == "Cannot overwrite existing sensitive property provider registered for mockProvider" + assert properties.getSensitivePropertyProviders().size() == 1 + } + + @Test + void testGetUnprotectedPropertiesShouldReturnInternalInstanceWhenNoneProtected() { + // Arrange + ProtectedNiFiRegistryProperties protectedNiFiProperties = loadFromResourceFile("/conf/nifi-registry.properties") + logger.info("Loaded ${protectedNiFiProperties.size()} properties from conf/nifi.properties") + + int hashCode = protectedNiFiProperties.internalNiFiProperties.hashCode() + logger.info("Hash code of internal instance: ${hashCode}") + + // Act + NiFiRegistryProperties unprotectedNiFiProperties = protectedNiFiProperties.getUnprotectedProperties() + logger.info("Unprotected ${unprotectedNiFiProperties.size()} properties") + + // Assert + assert unprotectedNiFiProperties.size() == protectedNiFiProperties.size() + assert unprotectedNiFiProperties.getPropertyKeys().every { + !unprotectedNiFiProperties.getProperty(it).endsWith(".protected") + } + logger.info("Hash code from returned unprotected instance: ${unprotectedNiFiProperties.hashCode()}") + assert unprotectedNiFiProperties.hashCode() == hashCode + } + + @Test + void testGetUnprotectedPropertiesShouldDecryptProtectedProperties() { + // Arrange + ProtectedNiFiRegistryProperties protectedNiFiProperties = + loadFromResourceFile("/conf/nifi-registry.with_sensitive_props_protected_aes_128.properties", KEY_HEX_128) + + int protectedPropertyCount = protectedNiFiProperties.getProtectedPropertyKeys().size() + int protectionSchemeCount = protectedNiFiProperties + .getPropertyKeysIncludingProtectionSchemes() + .findAll { it.endsWith(".protected") } + .size() + int expectedUnprotectedPropertyCount = protectedNiFiProperties.size() + + String protectedProps = protectedNiFiProperties + .getProtectedPropertyKeys() + .collectEntries { + [(it.key): protectedNiFiProperties.getProperty(it.key)] + }.entrySet() + .join("\n") + + logger.info("Detected ${protectedPropertyCount} protected properties and ${protectionSchemeCount} protection scheme properties") + logger.info("Protected properties: \n${protectedProps}") + + logger.info("Expected unprotected property count: ${expectedUnprotectedPropertyCount}") + + int hashCode = protectedNiFiProperties.internalNiFiProperties.hashCode() + logger.info("Hash code of internal instance: ${hashCode}") + + // Act + NiFiRegistryProperties unprotectedNiFiProperties = protectedNiFiProperties.getUnprotectedProperties() + logger.info("Unprotected ${unprotectedNiFiProperties.size()} properties") + + // Assert + assert unprotectedNiFiProperties.size() == expectedUnprotectedPropertyCount + assert unprotectedNiFiProperties.getPropertyKeys().every { + !unprotectedNiFiProperties.getProperty(it).endsWith(".protected") + } + logger.info("Hash code from returned unprotected instance: ${unprotectedNiFiProperties.hashCode()}") + assert unprotectedNiFiProperties.hashCode() != hashCode + } + + @Test + void testGetUnprotectedPropertiesShouldDecryptProtectedPropertiesWith256Bit() { + // Arrange + Assume.assumeTrue("JCE unlimited strength crypto policy must be installed for this test", Cipher.getMaxAllowedKeyLength("AES") > 128) + ProtectedNiFiRegistryProperties protectedNiFiProperties = + loadFromResourceFile("/conf/nifi-registry.with_sensitive_props_protected_aes_256.properties", KEY_HEX_256) + + int protectedPropertyCount = protectedNiFiProperties.getProtectedPropertyKeys().size() + int protectionSchemeCount = protectedNiFiProperties + .getPropertyKeysIncludingProtectionSchemes() + .findAll { it.endsWith(".protected") } + .size() + int expectedUnprotectedPropertyCount = protectedNiFiProperties.size() + + String protectedProps = protectedNiFiProperties + .getProtectedPropertyKeys() + .collectEntries { + [(it.key): protectedNiFiProperties.getProperty(it.key)] + }.entrySet() + .join("\n") + + logger.info("Detected ${protectedPropertyCount} protected properties and ${protectionSchemeCount} protection scheme properties") + logger.info("Protected properties: \n${protectedProps}") + + logger.info("Expected unprotected property count: ${expectedUnprotectedPropertyCount}") + + int hashCode = protectedNiFiProperties.internalNiFiProperties.hashCode() + logger.info("Hash code of internal instance: ${hashCode}") + + // Act + NiFiRegistryProperties unprotectedNiFiProperties = protectedNiFiProperties.getUnprotectedProperties() + logger.info("Unprotected ${unprotectedNiFiProperties.size()} properties") + + // Assert + assert unprotectedNiFiProperties.size() == expectedUnprotectedPropertyCount + assert unprotectedNiFiProperties.getPropertyKeys().every { + !unprotectedNiFiProperties.getProperty(it).endsWith(".protected") + } + logger.info("Hash code from returned unprotected instance: ${unprotectedNiFiProperties.hashCode()}") + assert unprotectedNiFiProperties.hashCode() != hashCode + } + + @Test + void testShouldCalculateSize() { + // Arrange + NiFiRegistryProperties rawProperties = [key: "protectedValue", "key.protected": "scheme", "key2": "value2"] as NiFiRegistryProperties + ProtectedNiFiRegistryProperties protectedNiFiProperties = new ProtectedNiFiRegistryProperties(rawProperties) + logger.info("Raw properties (${rawProperties.size()}): ${rawProperties.keySet().join(", ")}") + + // Act + int protectedSize = protectedNiFiProperties.size() + logger.info("Protected properties (${protectedNiFiProperties.size()}): " + + "${protectedNiFiProperties.getPropertyKeysExcludingProtectionSchemes().join(", ")}") + + // Assert + assert protectedSize == rawProperties.size() - 1 + } + + @Test + void testGetPropertyKeysShouldMatchSize() { + // Arrange + NiFiRegistryProperties rawProperties = [key: "protectedValue", "key.protected": "scheme", "key2": "value2"] as NiFiRegistryProperties + ProtectedNiFiRegistryProperties protectedNiFiProperties = new ProtectedNiFiRegistryProperties(rawProperties) + logger.info("Raw properties (${rawProperties.size()}): ${rawProperties.keySet().join(", ")}") + + // Act + def filteredKeys = protectedNiFiProperties.getPropertyKeysExcludingProtectionSchemes() + logger.info("Protected properties (${protectedNiFiProperties.size()}): ${filteredKeys.join(", ")}") + + // Assert + assert protectedNiFiProperties.size() == rawProperties.size() - 1 + assert filteredKeys == rawProperties.keySet() - "key.protected" + } + + @Test + void testShouldGetPropertyKeysIncludingProtectionSchemes() { + // Arrange + NiFiRegistryProperties rawProperties = [key: "protectedValue", "key.protected": "scheme", "key2": "value2"] as NiFiRegistryProperties + ProtectedNiFiRegistryProperties protectedNiFiProperties = new ProtectedNiFiRegistryProperties(rawProperties) + logger.info("Raw properties (${rawProperties.size()}): ${rawProperties.keySet().join(", ")}") + + // Act + def allKeys = protectedNiFiProperties.getPropertyKeysIncludingProtectionSchemes() + logger.info("Protected properties with schemes (${allKeys.size()}): ${allKeys.join(", ")}") + + // Assert + assert allKeys.size() == rawProperties.size() + assert allKeys == rawProperties.keySet() + } + +} diff --git a/nifi-registry-properties/src/test/groovy/org/apache/nifi/security/crypto/CryptoKeyLoaderGroovyTest.groovy b/nifi-registry-properties/src/test/groovy/org/apache/nifi/security/crypto/CryptoKeyLoaderGroovyTest.groovy new file mode 100644 index 000000000..4f69682fe --- /dev/null +++ b/nifi-registry-properties/src/test/groovy/org/apache/nifi/security/crypto/CryptoKeyLoaderGroovyTest.groovy @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.security.crypto + +import org.apache.nifi.registry.security.crypto.CryptoKeyLoader +import org.apache.nifi.registry.security.crypto.CryptoKeyProvider +import org.bouncycastle.jce.provider.BouncyCastleProvider +import org.junit.BeforeClass +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +import java.nio.file.Files +import java.nio.file.attribute.PosixFilePermission +import java.security.Security + +@RunWith(JUnit4.class) +class CryptoKeyLoaderGroovyTest extends GroovyTestCase { + + private static final Logger logger = LoggerFactory.getLogger(CryptoKeyLoaderGroovyTest.class) + + private static final String KEY_HEX_128 = "0123456789ABCDEFFEDCBA9876543210" + private static final String KEY_HEX_256 = KEY_HEX_128 * 2 + + @BeforeClass + public static void setUpOnce() throws Exception { + Security.addProvider(new BouncyCastleProvider()) + + logger.metaClass.methodMissing = { String name, args -> + logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}") + } + } + + @Test + public void testShouldExtractKeyFromBootstrapFile() throws Exception { + // Arrange + final String expectedKey = KEY_HEX_256 + + // Act + String key = CryptoKeyLoader.extractKeyFromBootstrapFile("src/test/resources/conf/bootstrap.conf") + + // Assert + assert key == expectedKey + } + + @Test + public void testShouldNotExtractKeyFromBootstrapFileWithoutKeyLine() throws Exception { + // Arrange + + // Act + String key = CryptoKeyLoader.extractKeyFromBootstrapFile("src/test/resources/conf/bootstrap.with_missing_key_line.conf") + + // Assert + assert key == CryptoKeyProvider.EMPTY_KEY + } + + @Test + public void testShouldNotExtractKeyFromBootstrapFileWithoutKey() throws Exception { + // Arrange + + // Act + String key = CryptoKeyLoader.extractKeyFromBootstrapFile("src/test/resources/conf/bootstrap.with_missing_key.conf") + + // Assert + assert key == CryptoKeyProvider.EMPTY_KEY + } + + @Test + public void testShouldNotExtractKeyFromMissingBootstrapFile() throws Exception { + // Arrange + + // Act + def msg = shouldFail(IOException) { + CryptoKeyLoader.extractKeyFromBootstrapFile("src/test/resources/conf/bootstrap.missing.conf") + } + logger.info(msg) + + // Assert + assert msg == "Cannot read from bootstrap.conf" + } + + @Test + public void testShouldNotExtractKeyFromUnreadableBootstrapFile() throws Exception { + // Arrange + File unreadableFile = new File("src/test/resources/conf/bootstrap.unreadable_file_permissions.conf") + Set originalPermissions = Files.getPosixFilePermissions(unreadableFile.toPath()) + Files.setPosixFilePermissions(unreadableFile.toPath(), [] as Set) + try { + assert !unreadableFile.canRead() + + // Act + def msg = shouldFail(IOException) { + CryptoKeyLoader.extractKeyFromBootstrapFile("src/test/resources/conf/bootstrap.unreadable_file_permissions.conf") + } + logger.info(msg) + + // Assert + assert msg == "Cannot read from bootstrap.conf" + } finally { + // Clean up to allow for indexing, etc. + Files.setPosixFilePermissions(unreadableFile.toPath(), originalPermissions) + } + } + +} diff --git a/nifi-registry-properties/src/test/resources/conf/bootstrap.conf b/nifi-registry-properties/src/test/resources/conf/bootstrap.conf new file mode 100644 index 000000000..4321bca7b --- /dev/null +++ b/nifi-registry-properties/src/test/resources/conf/bootstrap.conf @@ -0,0 +1,60 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Java command to use when running nifi-registry +java=java + +# Username to use when running nifi-registry. This value will be ignored on Windows. +run.as= + +# Configure where nifi-registry's lib and conf directories live +lib.dir=./lib +conf.dir=./conf + +# How long to wait after telling nifi-registry to shutdown before explicitly killing the Process +graceful.shutdown.seconds=20 + +# Disable JSR 199 so that we can use JSP's without running a JDK +java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true + +# JVM memory settings +java.arg.2=-Xms512m +java.arg.3=-Xmx512m + +# Enable Remote Debugging +#java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000 + +java.arg.4=-Djava.net.preferIPv4Stack=true + +# allowRestrictedHeaders is required for Cluster/Node communications to work properly +java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true +java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol + +# Java 7 and below have issues with Code Cache. The following lines allow us to run well even with +# many classes loaded in the JVM. +#java.arg.7=-XX:ReservedCodeCacheSize=256m +#java.arg.8=-XX:CodeCacheFlushingMinimumFreeSpace=10m +#java.arg.9=-XX:+UseCodeCacheFlushing +#java.arg.11=-XX:PermSize=128M +#java.arg.12=-XX:MaxPermSize=128M + +# The G1GC is still considered experimental but has proven to be very advantageous in providing great +# performance without significant "stop-the-world" delays. +#java.arg.10=-XX:+UseG1GC + +# Master key in hexadecimal format for encrypted sensitive configuration values +nifi.registry.bootstrap.sensitive.key=0123456789ABCDEFFEDCBA98765432100123456789ABCDEFFEDCBA9876543210 \ No newline at end of file diff --git a/nifi-registry-properties/src/test/resources/conf/bootstrap.unreadable_file_permissions.conf b/nifi-registry-properties/src/test/resources/conf/bootstrap.unreadable_file_permissions.conf new file mode 100644 index 000000000..30436353a --- /dev/null +++ b/nifi-registry-properties/src/test/resources/conf/bootstrap.unreadable_file_permissions.conf @@ -0,0 +1,22 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The POSIX file permissions for this file are emptied (i.e., chmod 000) during test and then reverted +# See org.apache.nifi.registry.properties.NiFiRegistryPropertiesLoaderGroovyTest#testShouldNotExtractKeyFromUnreadableBootstrapFile + +# Master key in hexadecimal format for encrypted sensitive configuration values +nifi.registry.bootstrap.sensitive.key=0123456789ABCDEFFEDCBA98765432100123456789ABCDEFFEDCBA9876543210 \ No newline at end of file diff --git a/nifi-registry-properties/src/test/resources/conf/bootstrap.with_missing_key.conf b/nifi-registry-properties/src/test/resources/conf/bootstrap.with_missing_key.conf new file mode 100644 index 000000000..7317ab029 --- /dev/null +++ b/nifi-registry-properties/src/test/resources/conf/bootstrap.with_missing_key.conf @@ -0,0 +1,60 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Java command to use when running nifi-registry +java=java + +# Username to use when running nifi-registry. This value will be ignored on Windows. +run.as= + +# Configure where nifi-registry's lib and conf directories live +lib.dir=./lib +conf.dir=./conf + +# How long to wait after telling nifi-registry to shutdown before explicitly killing the Process +graceful.shutdown.seconds=20 + +# Disable JSR 199 so that we can use JSP's without running a JDK +java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true + +# JVM memory settings +java.arg.2=-Xms512m +java.arg.3=-Xmx512m + +# Enable Remote Debugging +#java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000 + +java.arg.4=-Djava.net.preferIPv4Stack=true + +# allowRestrictedHeaders is required for Cluster/Node communications to work properly +java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true +java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol + +# Java 7 and below have issues with Code Cache. The following lines allow us to run well even with +# many classes loaded in the JVM. +#java.arg.7=-XX:ReservedCodeCacheSize=256m +#java.arg.8=-XX:CodeCacheFlushingMinimumFreeSpace=10m +#java.arg.9=-XX:+UseCodeCacheFlushing +#java.arg.11=-XX:PermSize=128M +#java.arg.12=-XX:MaxPermSize=128M + +# The G1GC is still considered experimental but has proven to be very advantageous in providing great +# performance without significant "stop-the-world" delays. +#java.arg.10=-XX:+UseG1GC + +# Master key in hexadecimal format for encrypted sensitive configuration values +nifi.registry.bootstrap.sensitive.key= \ No newline at end of file diff --git a/nifi-registry-properties/src/test/resources/conf/bootstrap.with_missing_key_line.conf b/nifi-registry-properties/src/test/resources/conf/bootstrap.with_missing_key_line.conf new file mode 100644 index 000000000..6ccdaaf04 --- /dev/null +++ b/nifi-registry-properties/src/test/resources/conf/bootstrap.with_missing_key_line.conf @@ -0,0 +1,60 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Java command to use when running nifi-registry +java=java + +# Username to use when running nifi-registry. This value will be ignored on Windows. +run.as= + +# Configure where nifi-registry's lib and conf directories live +lib.dir=./lib +conf.dir=./conf + +# How long to wait after telling nifi-registry to shutdown before explicitly killing the Process +graceful.shutdown.seconds=20 + +# Disable JSR 199 so that we can use JSP's without running a JDK +java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true + +# JVM memory settings +java.arg.2=-Xms512m +java.arg.3=-Xmx512m + +# Enable Remote Debugging +#java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000 + +java.arg.4=-Djava.net.preferIPv4Stack=true + +# allowRestrictedHeaders is required for Cluster/Node communications to work properly +java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true +java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol + +# Java 7 and below have issues with Code Cache. The following lines allow us to run well even with +# many classes loaded in the JVM. +#java.arg.7=-XX:ReservedCodeCacheSize=256m +#java.arg.8=-XX:CodeCacheFlushingMinimumFreeSpace=10m +#java.arg.9=-XX:+UseCodeCacheFlushing +#java.arg.11=-XX:PermSize=128M +#java.arg.12=-XX:MaxPermSize=128M + +# The G1GC is still considered experimental but has proven to be very advantageous in providing great +# performance without significant "stop-the-world" delays. +#java.arg.10=-XX:+UseG1GC + +# Master key in hexadecimal format for encrypted sensitive configuration values +# nifi.registry.bootstrap.sensitive.key is intentionally absent from this file \ No newline at end of file diff --git a/nifi-registry-properties/src/test/resources/conf/nifi-registry.properties b/nifi-registry-properties/src/test/resources/conf/nifi-registry.properties new file mode 100644 index 000000000..a7efedbee --- /dev/null +++ b/nifi-registry-properties/src/test/resources/conf/nifi-registry.properties @@ -0,0 +1,45 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# web properties # +nifi.registry.web.war.directory=./target/lib +nifi.registry.web.http.host= +nifi.registry.web.http.port=8080 +nifi.registry.web.https.host= +nifi.registry.web.https.port= +nifi.registry.web.jetty.working.directory=./target/work/jetty +nifi.registry.web.jetty.threads=1 + +# security properties # +nifi.registry.security.keystore= +nifi.registry.security.keystoreType= +nifi.registry.security.keystorePasswd= +nifi.registry.security.keyPasswd= +nifi.registry.security.truststore= +nifi.registry.security.truststoreType= +nifi.registry.security.truststorePasswd= +nifi.registry.security.needClientAuth= +nifi.registry.security.authorizers.configuration.file= +nifi.registry.security.authorizer= +nifi.registry.security.identity.providers.configuration.file= +nifi.registry.security.identity.provider= + +# kerberos properties +nifi.registry.kerberos.krb5.file=/path/to/krb5.conf +nifi.registry.kerberos.spnego.authentication.expiration=12 hours +nifi.registry.kerberos.spnego.principal=HTTP/localhost@LOCALHOST +nifi.registry.kerberos.spnego.keytab.location=/path/to/keytab diff --git a/nifi-registry-properties/src/test/resources/conf/nifi-registry.with_additional_sensitive_keys.properties b/nifi-registry-properties/src/test/resources/conf/nifi-registry.with_additional_sensitive_keys.properties new file mode 100644 index 000000000..5afb3dd23 --- /dev/null +++ b/nifi-registry-properties/src/test/resources/conf/nifi-registry.with_additional_sensitive_keys.properties @@ -0,0 +1,55 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# web properties # +nifi.registry.web.war.directory=./target/lib +nifi.registry.web.http.host= +nifi.registry.web.http.port=8080 +nifi.registry.web.https.host= +nifi.registry.web.https.port= +nifi.registry.web.jetty.working.directory=./target/work/jetty +nifi.registry.web.jetty.threads=1 + +# security properties # +nifi.registry.security.keystore= +nifi.registry.security.keystoreType= +nifi.registry.security.keystorePasswd= +nifi.registry.security.keyPasswd= +nifi.registry.security.truststore= +nifi.registry.security.truststoreType= +nifi.registry.security.truststorePasswd= +nifi.registry.security.needClientAuth= +nifi.registry.security.authorizers.configuration.file= +nifi.registry.security.authorizer= +nifi.registry.security.identity.providers.configuration.file= +nifi.registry.security.identity.provider= + +# providers properties # +nifi.registry.providers.configuration.file= + +# database properties +nifi.registry.db.directory=./target/db +nifi.registry.db.url.append= + +# kerberos properties # +nifi.registry.kerberos.krb5.file=/path/to/krb5.conf +nifi.registry.kerberos.spnego.authentication.expiration=12 hours +nifi.registry.kerberos.spnego.principal=HTTP/localhost@LOCALHOST +nifi.registry.kerberos.spnego.keytab.location=/path/to/keytab + +# security properties # +nifi.registry.sensitive.props.additional.keys=nifi.registry.web.http.port, nifi.registry.web.http.host, nifi.registry.sensitive.props.additional.keys diff --git a/nifi-registry-properties/src/test/resources/conf/nifi-registry.with_sensitive_props_fully_protected_aes_128.properties b/nifi-registry-properties/src/test/resources/conf/nifi-registry.with_sensitive_props_fully_protected_aes_128.properties new file mode 100644 index 000000000..90eb64fc4 --- /dev/null +++ b/nifi-registry-properties/src/test/resources/conf/nifi-registry.with_sensitive_props_fully_protected_aes_128.properties @@ -0,0 +1,43 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# web properties # +nifi.registry.web.war.directory=./target/lib +nifi.registry.web.http.host= +nifi.registry.web.http.port=8080 +nifi.registry.web.https.host= +nifi.registry.web.https.port= +nifi.registry.web.jetty.working.directory=./target/work/jetty +nifi.registry.web.jetty.threads=1 + +# security properties # +nifi.registry.security.keystore=/path/to/keystore.jks +nifi.registry.security.keystoreType=JKS +nifi.registry.security.keystorePasswd=6WUpex+VZiN05LXu||joWJMuoSzYniEC7IAoingTimlG7+RGk8I2irl/WTlIuMcg +nifi.registry.security.keystorePasswd.protected=aes/gcm/128 +nifi.registry.security.keyPasswd=6WUpex+VZiN05LXu||joWJMuoSzYniEC7IAoingTimlG7+RGk8I2irl/WTlIuMcg +nifi.registry.security.keyPasswd.protected=aes/gcm/128 +nifi.registry.security.truststore= +nifi.registry.security.truststoreType= +nifi.registry.security.truststorePasswd= +nifi.registry.security.needClientAuth= +nifi.registry.security.authorizers.configuration.file= +nifi.registry.security.authorizer= +nifi.registry.security.identity.providers.configuration.file= +nifi.registry.security.identity.provider= + +nifi.registry.sensitive.props.additional.keys= diff --git a/nifi-registry-properties/src/test/resources/conf/nifi-registry.with_sensitive_props_protected_aes_128.properties b/nifi-registry-properties/src/test/resources/conf/nifi-registry.with_sensitive_props_protected_aes_128.properties new file mode 100644 index 000000000..6ecd28192 --- /dev/null +++ b/nifi-registry-properties/src/test/resources/conf/nifi-registry.with_sensitive_props_protected_aes_128.properties @@ -0,0 +1,43 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# web properties # +nifi.registry.web.war.directory=./target/lib +nifi.registry.web.http.host= +nifi.registry.web.http.port=8080 +nifi.registry.web.https.host= +nifi.registry.web.https.port= +nifi.registry.web.jetty.working.directory=./target/work/jetty +nifi.registry.web.jetty.threads=1 + +# security properties # +nifi.registry.security.keystore=/path/to/keystore.jks +nifi.registry.security.keystoreType=JKS +nifi.registry.security.keystorePasswd=6WUpex+VZiN05LXu||joWJMuoSzYniEC7IAoingTimlG7+RGk8I2irl/WTlIuMcg +nifi.registry.security.keystorePasswd.protected=aes/gcm/128 +nifi.registry.security.keyPasswd=6WUpex+VZiN05LXu||joWJMuoSzYniEC7IAoingTimlG7+RGk8I2irl/WTlIuMcg +nifi.registry.security.keyPasswd.protected=aes/gcm/128 +nifi.registry.security.truststore= +nifi.registry.security.truststoreType= +nifi.registry.security.truststorePasswd= +nifi.registry.security.needClientAuth= +nifi.registry.security.authorizers.configuration.file= +nifi.registry.security.authorizer= +nifi.registry.security.identity.providers.configuration.file= +nifi.registry.security.identity.provider= + +nifi.registry.sensitive.props.additional.keys=nifi.registry.web.http.port diff --git a/nifi-registry-properties/src/test/resources/conf/nifi-registry.with_sensitive_props_protected_aes_128_password.properties b/nifi-registry-properties/src/test/resources/conf/nifi-registry.with_sensitive_props_protected_aes_128_password.properties new file mode 100644 index 000000000..a3b272d99 --- /dev/null +++ b/nifi-registry-properties/src/test/resources/conf/nifi-registry.with_sensitive_props_protected_aes_128_password.properties @@ -0,0 +1,43 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# web properties # +nifi.registry.web.war.directory=./target/lib +nifi.registry.web.http.host= +nifi.registry.web.http.port=8080 +nifi.registry.web.https.host= +nifi.registry.web.https.port= +nifi.registry.web.jetty.working.directory=./target/work/jetty +nifi.registry.web.jetty.threads=1 + +# security properties # +nifi.registry.security.keystore=/path/to/keystore.jks +nifi.registry.security.keystoreType=JKS +nifi.registry.security.keystorePasswd=oa6Aaz5tlFprPuKt||IlVgftF2VqvBIambkP5HVDbRoyKzZl8wwKSw4O9tjHTALA +nifi.registry.security.keystorePasswd.protected=aes/gcm/128 +nifi.registry.security.keyPasswd=oa6Aaz5tlFprPuKt||IlVgftF2VqvBIambkP5HVDbRoyKzZl8wwKSw4O9tjHTALA +nifi.registry.security.keyPasswd.protected=aes/gcm/128 +nifi.registry.security.truststore= +nifi.registry.security.truststoreType= +nifi.registry.security.truststorePasswd= +nifi.registry.security.needClientAuth= +nifi.registry.security.authorizers.configuration.file= +nifi.registry.security.authorizer= +nifi.registry.security.identity.providers.configuration.file= +nifi.registry.security.identity.provider= + +nifi.registry.sensitive.props.additional.keys=nifi.registry.web.http.port diff --git a/nifi-registry-properties/src/test/resources/conf/nifi-registry.with_sensitive_props_protected_aes_256.properties b/nifi-registry-properties/src/test/resources/conf/nifi-registry.with_sensitive_props_protected_aes_256.properties new file mode 100644 index 000000000..97aaba013 --- /dev/null +++ b/nifi-registry-properties/src/test/resources/conf/nifi-registry.with_sensitive_props_protected_aes_256.properties @@ -0,0 +1,43 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# web properties # +nifi.registry.web.war.directory=./target/lib +nifi.registry.web.http.host= +nifi.registry.web.http.port=8080 +nifi.registry.web.https.host= +nifi.registry.web.https.port= +nifi.registry.web.jetty.working.directory=./target/work/jetty +nifi.registry.web.jetty.threads=1 + +# security properties # +nifi.registry.security.keystore=/path/to/keystore.jks +nifi.registry.security.keystoreType=JKS +nifi.registry.security.keystorePasswd=oBjT92hIGRElIGOh||MZ6uYuWNBrOA6usq/Jt3DaD2e4otNirZDytac/w/KFe0HOkrJR03vcbo +nifi.registry.security.keystorePasswd.protected=aes/gcm/256 +nifi.registry.security.keyPasswd=ac/BaE35SL/esLiJ||+ULRvRLYdIDA2VqpE0eQXDEMjaLBMG2kbKOdOwBk/hGebDKlVg== +nifi.registry.security.keyPasswd.protected=aes/gcm/256 +nifi.registry.security.truststore= +nifi.registry.security.truststoreType= +nifi.registry.security.truststorePasswd= +nifi.registry.security.needClientAuth= +nifi.registry.security.authorizers.configuration.file= +nifi.registry.security.authorizer= +nifi.registry.security.identity.providers.configuration.file= +nifi.registry.security.identity.provider= + +nifi.registry.sensitive.props.additional.keys=nifi.registry.web.http.port diff --git a/nifi-registry-properties/src/test/resources/conf/nifi-registry.with_sensitive_props_protected_aes_multiple_malformed.properties b/nifi-registry-properties/src/test/resources/conf/nifi-registry.with_sensitive_props_protected_aes_multiple_malformed.properties new file mode 100644 index 000000000..d408df06b --- /dev/null +++ b/nifi-registry-properties/src/test/resources/conf/nifi-registry.with_sensitive_props_protected_aes_multiple_malformed.properties @@ -0,0 +1,43 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# web properties # +nifi.registry.web.war.directory=./target/lib +nifi.registry.web.http.host= +nifi.registry.web.http.port=8080 +nifi.registry.web.https.host= +nifi.registry.web.https.port= +nifi.registry.web.jetty.working.directory=./target/work/jetty +nifi.registry.web.jetty.threads=1 + +# security properties # +nifi.registry.security.keystore=/path/to/keystore.jks +nifi.registry.security.keystoreType=JKS +nifi.registry.security.keystorePasswd=6WUpex+VZiN05LXu||thisIsAnIntentionallyMalformedCipherValue +nifi.registry.security.keystorePasswd.protected=aes/gcm/128 +nifi.registry.security.keyPasswd=6WUpex+VZiN05LXu||thisIsAnIntentionallyMalformedCipherValue +nifi.registry.security.keyPasswd.protected=aes/gcm/128 +nifi.registry.security.truststore= +nifi.registry.security.truststoreType= +nifi.registry.security.truststorePasswd= +nifi.registry.security.needClientAuth= +nifi.registry.security.authorizers.configuration.file= +nifi.registry.security.authorizer= +nifi.registry.security.identity.providers.configuration.file= +nifi.registry.security.identity.provider= + +nifi.registry.sensitive.props.additional.keys= diff --git a/nifi-registry-properties/src/test/resources/conf/nifi-registry.with_sensitive_props_protected_aes_single_malformed.properties b/nifi-registry-properties/src/test/resources/conf/nifi-registry.with_sensitive_props_protected_aes_single_malformed.properties new file mode 100644 index 000000000..8552f9e2a --- /dev/null +++ b/nifi-registry-properties/src/test/resources/conf/nifi-registry.with_sensitive_props_protected_aes_single_malformed.properties @@ -0,0 +1,43 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# web properties # +nifi.registry.web.war.directory=./target/lib +nifi.registry.web.http.host= +nifi.registry.web.http.port=8080 +nifi.registry.web.https.host= +nifi.registry.web.https.port= +nifi.registry.web.jetty.working.directory=./target/work/jetty +nifi.registry.web.jetty.threads=1 + +# security properties # +nifi.registry.security.keystore=/path/to/keystore.jks +nifi.registry.security.keystoreType=JKS +nifi.registry.security.keystorePasswd=6WUpex+VZiN05LXu||thisIsAnIntentionallyMalformedCipherValue +nifi.registry.security.keystorePasswd.protected=aes/gcm/128 +nifi.registry.security.keyPasswd=6WUpex+VZiN05LXu||joWJMuoSzYniEC7IAoingTimlG7+RGk8I2irl/WTlIuMcg +nifi.registry.security.keyPasswd.protected=aes/gcm/128 +nifi.registry.security.truststore= +nifi.registry.security.truststoreType= +nifi.registry.security.truststorePasswd= +nifi.registry.security.needClientAuth= +nifi.registry.security.authorizers.configuration.file= +nifi.registry.security.authorizer= +nifi.registry.security.identity.providers.configuration.file= +nifi.registry.security.identity.provider= + +nifi.registry.sensitive.props.additional.keys= diff --git a/nifi-registry-properties/src/test/resources/conf/nifi-registry.with_sensitive_props_protected_unknown.properties b/nifi-registry-properties/src/test/resources/conf/nifi-registry.with_sensitive_props_protected_unknown.properties new file mode 100644 index 000000000..8bd6f4fbb --- /dev/null +++ b/nifi-registry-properties/src/test/resources/conf/nifi-registry.with_sensitive_props_protected_unknown.properties @@ -0,0 +1,43 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# web properties # +nifi.registry.web.war.directory=./target/lib +nifi.registry.web.http.host= +nifi.registry.web.http.port=8080 +nifi.registry.web.https.host= +nifi.registry.web.https.port= +nifi.registry.web.jetty.working.directory=./target/work/jetty +nifi.registry.web.jetty.threads=1 + +# security properties # +nifi.registry.security.keystore=/path/to/keystore.jks +nifi.registry.security.keystoreType=JKS +nifi.registry.security.keystorePasswd=oBjT92hIGRElIGOh||MZ6uYuWNBrOA6usq/Jt3DaD2e4otNirZDytac/w/KFe0HOkrJR03vcbo +nifi.registry.security.keystorePasswd.protected=unknown +nifi.registry.security.keyPasswd=ac/BaE35SL/esLiJ||+ULRvRLYdIDA2VqpE0eQXDEMjaLBMG2kbKOdOwBk/hGebDKlVg== +nifi.registry.security.keyPasswd.protected=unknown +nifi.registry.security.truststore= +nifi.registry.security.truststoreType= +nifi.registry.security.truststorePasswd= +nifi.registry.security.needClientAuth= +nifi.registry.security.authorizers.configuration.file= +nifi.registry.security.authorizer= +nifi.registry.security.identity.providers.configuration.file= +nifi.registry.security.identity.provider= + +nifi.registry.sensitive.props.additional.keys=nifi.registry.web.http.port, nifi.registry.web.http.host diff --git a/nifi-registry-properties/src/test/resources/conf/nifi-registry.with_sensitive_props_unprotected.properties b/nifi-registry-properties/src/test/resources/conf/nifi-registry.with_sensitive_props_unprotected.properties new file mode 100644 index 000000000..b0f9f4041 --- /dev/null +++ b/nifi-registry-properties/src/test/resources/conf/nifi-registry.with_sensitive_props_unprotected.properties @@ -0,0 +1,41 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# web properties # +nifi.registry.web.war.directory=./target/lib +nifi.registry.web.http.host= +nifi.registry.web.http.port=8080 +nifi.registry.web.https.host= +nifi.registry.web.https.port= +nifi.registry.web.jetty.working.directory=./target/work/jetty +nifi.registry.web.jetty.threads=1 + +# security properties # +nifi.registry.security.keystore=path/to/keystore.jks +nifi.registry.security.keystoreType=JKS +nifi.registry.security.keystorePasswd=thisIsABadKeystorePassword +nifi.registry.security.keyPasswd=thisIsABadKeyPassword +nifi.registry.security.truststore= +nifi.registry.security.truststoreType= +nifi.registry.security.truststorePasswd= +nifi.registry.security.needClientAuth= +nifi.registry.security.authorizers.configuration.file= +nifi.registry.security.authorizer= +nifi.registry.security.identity.providers.configuration.file= +nifi.registry.security.identity.provider= + +nifi.registry.sensitive.props.additional.keys=nifi.registry.web.http.port, nifi.registry.web.http.host diff --git a/nifi-registry-properties/src/test/resources/conf/nifi-registry.with_sensitive_props_unprotected_extra_line.properties b/nifi-registry-properties/src/test/resources/conf/nifi-registry.with_sensitive_props_unprotected_extra_line.properties new file mode 100644 index 000000000..34b80a311 --- /dev/null +++ b/nifi-registry-properties/src/test/resources/conf/nifi-registry.with_sensitive_props_unprotected_extra_line.properties @@ -0,0 +1,42 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# web properties # +nifi.registry.web.war.directory=./target/lib +nifi.registry.web.http.host= +nifi.registry.web.http.port=8080 +nifi.registry.web.https.host= +nifi.registry.web.https.port= +nifi.registry.web.jetty.working.directory=./target/work/jetty +nifi.registry.web.jetty.threads=1 + +# security properties # +nifi.registry.security.keystore=path/to/keystore.jks +nifi.registry.security.keystoreType=JKS +nifi.registry.security.keystorePasswd=thisIsABadKeystorePassword +nifi.registry.security.keyPasswd=thisIsABadKeyPassword +nifi.registry.security.keyPasswd.protected= +nifi.registry.security.truststore= +nifi.registry.security.truststoreType= +nifi.registry.security.truststorePasswd= +nifi.registry.security.needClientAuth= +nifi.registry.security.authorizers.configuration.file= +nifi.registry.security.authorizer= +nifi.registry.security.identity.providers.configuration.file= +nifi.registry.security.identity.provider= + +nifi.registry.sensitive.props.additional.keys=nifi.registry.web.http.port, nifi.registry.web.http.host diff --git a/nifi-registry-resources/src/main/resources/conf/bootstrap.conf b/nifi-registry-resources/src/main/resources/conf/bootstrap.conf index bdbd1085a..7317ab029 100644 --- a/nifi-registry-resources/src/main/resources/conf/bootstrap.conf +++ b/nifi-registry-resources/src/main/resources/conf/bootstrap.conf @@ -54,4 +54,7 @@ java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol # The G1GC is still considered experimental but has proven to be very advantageous in providing great # performance without significant "stop-the-world" delays. -#java.arg.10=-XX:+UseG1GC \ No newline at end of file +#java.arg.10=-XX:+UseG1GC + +# Master key in hexadecimal format for encrypted sensitive configuration values +nifi.registry.bootstrap.sensitive.key= \ No newline at end of file diff --git a/nifi-registry-resources/src/main/resources/conf/nifi-registry.properties b/nifi-registry-resources/src/main/resources/conf/nifi-registry.properties index 28577dc93..5b6dae28d 100644 --- a/nifi-registry-resources/src/main/resources/conf/nifi-registry.properties +++ b/nifi-registry-resources/src/main/resources/conf/nifi-registry.properties @@ -36,6 +36,9 @@ nifi.registry.security.authorizer=${nifi.registry.security.authorizer} nifi.registry.security.identity.providers.configuration.file=${nifi.registry.security.identity.providers.configuration.file} nifi.registry.security.identity.provider=${nifi.registry.security.identity.provider} +# sensitive property protection properties # +# nifi.registry.sensitive.props.additional.keys= + # providers properties # nifi.registry.providers.configuration.file=${nifi.registry.providers.configuration.file} diff --git a/nifi-registry-runtime/src/main/java/org/apache/nifi/registry/NiFiRegistry.java b/nifi-registry-runtime/src/main/java/org/apache/nifi/registry/NiFiRegistry.java index 5877b8b48..deb2a8daf 100644 --- a/nifi-registry-runtime/src/main/java/org/apache/nifi/registry/NiFiRegistry.java +++ b/nifi-registry-runtime/src/main/java/org/apache/nifi/registry/NiFiRegistry.java @@ -17,24 +17,25 @@ package org.apache.nifi.registry; import org.apache.nifi.registry.jetty.JettyServer; +import org.apache.nifi.registry.security.crypto.BootstrapFileCryptoKeyProvider; +import org.apache.nifi.registry.security.crypto.CryptoKeyProvider; +import org.apache.nifi.registry.security.crypto.CryptoKeyLoader; +import org.apache.nifi.registry.security.crypto.MissingCryptoKeyException; import org.apache.nifi.registry.properties.NiFiRegistryProperties; +import org.apache.nifi.registry.properties.NiFiRegistryPropertiesLoader; +import org.apache.nifi.registry.properties.SensitivePropertyProtectionException; +import org.apache.nifi.registry.security.crypto.VolatileCryptoKeyProvider; import org.apache.nifi.registry.util.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.bridge.SLF4JBridgeHandler; import java.io.File; -import java.io.FileReader; -import java.io.FileWriter; import java.io.IOException; import java.lang.reflect.InvocationTargetException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.Random; import java.util.concurrent.TimeUnit; /** @@ -47,11 +48,14 @@ public class NiFiRegistry { public static final String BOOTSTRAP_PORT_PROPERTY = "nifi.registry.bootstrap.listen.port"; + public static final String REGISTRY_BOOTSTRAP_FILE_LOCATION = "conf/bootstrap.conf"; + public static final String REGISTRY_PROPERTIES_FILE_LOCATION = "conf/nifi-registry.properties"; + private final JettyServer server; private final BootstrapListener bootstrapListener; private volatile boolean shutdown = false; - public NiFiRegistry(final NiFiRegistryProperties properties) + public NiFiRegistry(final NiFiRegistryProperties properties, CryptoKeyProvider masterKeyProvider) throws ClassNotFoundException, IOException, NoSuchMethodException, InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException { Thread.setDefaultUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() { @@ -104,7 +108,7 @@ public void run() { SLF4JBridgeHandler.install(); final long startTime = System.nanoTime(); - server = new JettyServer(properties); + server = new JettyServer(properties, masterKeyProvider); if (shutdown) { LOGGER.info("NiFi Registry has been shutdown via NiFi Registry Bootstrap. Will not start Controller"); @@ -146,84 +150,80 @@ protected void shutdownHook() { public static void main(String[] args) { LOGGER.info("Launching NiFi Registry..."); - final NiFiRegistryProperties properties = new NiFiRegistryProperties(); - try (final FileReader reader = new FileReader("conf/nifi-registry.properties")) { - properties.load(reader); - } catch (final IOException ioe) { - throw new RuntimeException("Unable to load properties: " + ioe, ioe); + final CryptoKeyProvider masterKeyProvider; + final NiFiRegistryProperties properties; + try { + masterKeyProvider = createMasterKeyProvider(args); + properties = initializeProperties(masterKeyProvider); + } catch (final IllegalArgumentException iae) { + throw new RuntimeException("Unable to load properties: " + iae, iae); } try { - new NiFiRegistry(properties); + new NiFiRegistry(properties, masterKeyProvider); } catch (final Throwable t) { LOGGER.error("Failure to launch NiFi Registry due to " + t, t); } } - private static String loadFormattedKey(String[] args) { - String key = null; - List parsedArgs = parseArgs(args); - // Check if args contain protection key - if (parsedArgs.contains(KEY_FILE_FLAG)) { - key = getKeyFromKeyFileAndPrune(parsedArgs); - // Format the key (check hex validity and remove spaces) - key = formatHexKey(key); - } + private static NiFiRegistryProperties initializeProperties(CryptoKeyProvider masterKeyProvider) { - if (null == key) { - return ""; - } else if (!isHexKeyValid(key)) { - throw new IllegalArgumentException("The key was not provided in valid hex format and of the correct length"); - } else { - return key; + String key = CryptoKeyProvider.EMPTY_KEY; + try { + key = masterKeyProvider.getKey(); + } catch (MissingCryptoKeyException e) { + LOGGER.debug("CryptoKeyProvider provided to initializeProperties method was empty - did not contain a key."); + // Do nothing. The key can be empty when it is passed to the loader as the loader will only use it if any properties are protected. } - } - private static String getKeyFromKeyFileAndPrune(List parsedArgs) { - String key = null; - LOGGER.debug("The bootstrap process provided the " + KEY_FILE_FLAG + " flag"); - int i = parsedArgs.indexOf(KEY_FILE_FLAG); - if (parsedArgs.size() <= i + 1) { - LOGGER.error("The bootstrap process passed the {} flag without a filename", KEY_FILE_FLAG); - throw new IllegalArgumentException("The bootstrap process provided the " + KEY_FILE_FLAG + " flag but no key"); - } try { - String passwordfile_path = parsedArgs.get(i + 1); - // Slurp in the contents of the file: - byte[] encoded = Files.readAllBytes(Paths.get(passwordfile_path)); - key = new String(encoded, StandardCharsets.UTF_8); - if (0 == key.length()) - throw new IllegalArgumentException("Key in keyfile " + passwordfile_path + " yielded an empty key"); - - LOGGER.info("Now overwriting file in "+passwordfile_path); - - // Overwrite the contents of the file (to avoid littering file system - // unlinked with key material): - File password_file = new File(passwordfile_path); - FileWriter overwriter = new FileWriter(password_file,false); - - // Construe a random pad: - Random r = new Random(); - StringBuffer sb = new StringBuffer(); - // Note on correctness: this pad is longer, but equally sufficient. - while(sb.length() < encoded.length){ - sb.append(Integer.toHexString(r.nextInt())); + try { + // Load properties using key. If properties are protected and key missing, throw RuntimeException + NiFiRegistryProperties properties = NiFiRegistryPropertiesLoader.withKey(key).load(REGISTRY_PROPERTIES_FILE_LOCATION); + LOGGER.info("Loaded {} properties", properties.size()); + return properties; + } catch (SensitivePropertyProtectionException e) { + final String msg = "There was an issue decrypting protected properties"; + LOGGER.error(msg, e); + throw new IllegalArgumentException(msg); } - String pad = sb.toString(); - LOGGER.info("Overwriting key material with pad: "+pad); - overwriter.write(pad); - overwriter.close(); + } catch (IllegalArgumentException e) { + final String msg = "The bootstrap process did not provide a valid key and there are protected properties present in the properties file"; + LOGGER.error(msg, e); + throw new IllegalArgumentException(msg); + } + } - LOGGER.info("Removing/unlinking file: "+passwordfile_path); - password_file.delete(); + private static CryptoKeyProvider createMasterKeyProvider(String[] args) { + CryptoKeyProvider masterKeyProvider = null; + List parsedArgs = parseArgs(args); + try { + if (parsedArgs.contains(KEY_FILE_FLAG)) { + LOGGER.debug("The bootstrap process provided the {} flag", KEY_FILE_FLAG); + int i = parsedArgs.indexOf(KEY_FILE_FLAG); + if (parsedArgs.size() <= i + 1) { + LOGGER.error("The bootstrap process passed the {} flag without a filename", KEY_FILE_FLAG); + throw new IllegalArgumentException("The bootstrap process provided the " + KEY_FILE_FLAG + " flag but no key"); + } + String passwordfilePath = parsedArgs.get(i + 1); + String key = CryptoKeyLoader.extractKeyFromKeyFile(passwordfilePath, true); + if (!isHexKeyValid(key)) { + throw new IllegalArgumentException("The key was not provided in valid hex format and of a valid length"); + } + masterKeyProvider = new VolatileCryptoKeyProvider(key); + LOGGER.info("Read property protection key from key file provided by bootstrap process"); + } else { + // if the key was not passed from bootstrap via -K arg, assume it is in bootstrap.conf + masterKeyProvider = new BootstrapFileCryptoKeyProvider(REGISTRY_BOOTSTRAP_FILE_LOCATION); + LOGGER.info("Read property protection key from {}", REGISTRY_BOOTSTRAP_FILE_LOCATION); + } } catch (IOException e) { - LOGGER.error("Caught IOException while retrieving the "+KEY_FILE_FLAG+"-passed keyfile; aborting: "+e.toString()); + LOGGER.error("Caught IOException while attempting to load master decryption key; aborting: " + e.toString()); System.exit(1); } - LOGGER.info("Read property protection key from key file provided by bootstrap process"); - return key; + return masterKeyProvider; } private static List parseArgs(String[] args) { @@ -239,13 +239,6 @@ private static List parseArgs(String[] args) { return parsedArgs; } - private static String formatHexKey(String input) { - if (input == null || input.trim().isEmpty()) { - return ""; - } - return input.replaceAll("[^0-9a-fA-F]", "").toLowerCase(); - } - private static boolean isHexKeyValid(String key) { if (key == null || key.trim().isEmpty()) { return false; diff --git a/nifi-registry-web-api/pom.xml b/nifi-registry-web-api/pom.xml index 48c561559..f32b1ade6 100644 --- a/nifi-registry-web-api/pom.xml +++ b/nifi-registry-web-api/pom.xml @@ -167,8 +167,8 @@ org.apache.nifi.registry - nifi-registry-security-utils - 0.0.1-SNAPSHOT + nifi-registry-security-utils + 0.0.1-SNAPSHOT org.apache.commons diff --git a/nifi-registry-web-api/src/main/java/org/apache/nifi/registry/NiFiRegistryApiApplication.java b/nifi-registry-web-api/src/main/java/org/apache/nifi/registry/NiFiRegistryApiApplication.java index 4a0bcbc82..53698d408 100644 --- a/nifi-registry-web-api/src/main/java/org/apache/nifi/registry/NiFiRegistryApiApplication.java +++ b/nifi-registry-web-api/src/main/java/org/apache/nifi/registry/NiFiRegistryApiApplication.java @@ -32,6 +32,7 @@ public class NiFiRegistryApiApplication extends SpringBootServletInitializer { public static final String NIFI_REGISTRY_PROPERTIES_ATTRIBUTE = "nifi-registry.properties"; + public static final String NIFI_REGISTRY_MASTER_KEY_ATTRIBUTE = "nifi-registry.key"; public static void main(String[] args) { SpringApplication.run(NiFiRegistryApiApplication.class, args); diff --git a/nifi-registry-web-api/src/main/java/org/apache/nifi/registry/web/security/NiFiRegistryMasterKeyFactory.java b/nifi-registry-web-api/src/main/java/org/apache/nifi/registry/web/security/NiFiRegistryMasterKeyFactory.java new file mode 100644 index 000000000..3c261d3ce --- /dev/null +++ b/nifi-registry-web-api/src/main/java/org/apache/nifi/registry/web/security/NiFiRegistryMasterKeyFactory.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.registry.web.security; + +import org.apache.nifi.registry.NiFiRegistryApiApplication; +import org.apache.nifi.registry.security.crypto.BootstrapFileCryptoKeyProvider; +import org.apache.nifi.registry.security.crypto.CryptoKeyProvider; +import org.apache.nifi.registry.security.crypto.MissingCryptoKeyException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.context.ApplicationListener; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.event.ContextRefreshedEvent; + +import javax.servlet.ServletContextAttributeEvent; +import javax.servlet.ServletContextAttributeListener; + +@Configuration +public class NiFiRegistryMasterKeyFactory implements ServletContextAttributeListener, ApplicationListener { + + private static final Logger logger = LoggerFactory.getLogger(NiFiRegistryMasterKeyFactory.class); + + private CryptoKeyProvider masterKeyProvider = MISSING_KEY_PROVIDER; + + @Bean + public CryptoKeyProvider getNiFiRegistryMasterKeyProvider() { + return masterKeyProvider; + } + + // -- ServletContextAttributeListener methods + + @Override + public void attributeAdded(ServletContextAttributeEvent servletContextAttributeEvent) { + updateMasterKeyProvider(servletContextAttributeEvent); + } + + @Override + public void attributeReplaced(ServletContextAttributeEvent servletContextAttributeEvent) { + updateMasterKeyProvider(servletContextAttributeEvent); + } + + @Override + public void attributeRemoved(ServletContextAttributeEvent servletContextAttributeEvent) { + String attributeName = servletContextAttributeEvent.getName(); + if (attributeName != null && attributeName.equals(NiFiRegistryApiApplication.NIFI_REGISTRY_MASTER_KEY_ATTRIBUTE)) { + clearMasterKeyProvider(); + } + } + + // -- ApplicationListener methods + + @Override + public void onApplicationEvent(ContextRefreshedEvent contextRefreshedEvent) { + if (masterKeyProvider != null) { + if (!(masterKeyProvider instanceof BootstrapFileCryptoKeyProvider)) { + // If the Key Holder is not backed by the bootstrap.conf file, + // if might be holding the key material in memory, and in scope. + // If we receive this event, the ApplicationContext has finished initializing, + // so all sensitive props should be loaded and the master key is no longer needed. + // De-reference the holder so that GC can purge the key material from memory. + logger.info("Received {} indicating the ApplicationContext is done initializing. Now clearing the CryptoKeyProvider " + + "Bean holding the master key from ApplicationContext, as it should no longer be needed.", contextRefreshedEvent); + clearMasterKeyProvider(); + } else { + logger.debug("Received {} indicating the ApplicationContext is done initializing. CryptoKeyProvider instance is backed " + + "by bootstrap.conf file and will remain available in the ApplicationContext to load master key on demand if needed."); + } + } + + } + + private void updateMasterKeyProvider(ServletContextAttributeEvent servletContextAttributeEvent) { + String attributeName = servletContextAttributeEvent.getName(); + if (attributeName != null && attributeName.equals(NiFiRegistryApiApplication.NIFI_REGISTRY_MASTER_KEY_ATTRIBUTE)) { + Object attributeValue = servletContextAttributeEvent.getValue(); + if (attributeValue == null || !(attributeValue instanceof CryptoKeyProvider)) { + clearMasterKeyProvider(); + } else { + logger.debug("Received {}, updating CryptoKeyProvider Bean containing the master key.", servletContextAttributeEvent); + masterKeyProvider = (CryptoKeyProvider)attributeValue; + } + } + } + + private void clearMasterKeyProvider() { + logger.debug("Clearing CryptoKeyProvider Bean containing the master key. Master key will no longer be accessible in ApplicationContext."); + masterKeyProvider = MISSING_KEY_PROVIDER; + // actual master key holder is now out of scope and can be reaped from memory at next GC run + } + + private static final CryptoKeyProvider MISSING_KEY_PROVIDER = new CryptoKeyProvider() { + @Override + public String getKey() throws MissingCryptoKeyException { + throw new MissingCryptoKeyException("The actual CryptoKeyProvider used for ApplicationContext " + + "loading has been intentionally destroyed and is no longer accessible. Something is " + + "trying to access the master key after ApplicationContext has finished loading."); + } + }; + +} diff --git a/nifi-registry-web-api/src/test/java/org/apache/nifi/registry/web/api/SecureLdapIT.java b/nifi-registry-web-api/src/test/java/org/apache/nifi/registry/web/api/SecureLdapIT.java index 3ee4d83c0..8f3d64c82 100644 --- a/nifi-registry-web-api/src/test/java/org/apache/nifi/registry/web/api/SecureLdapIT.java +++ b/nifi-registry-web-api/src/test/java/org/apache/nifi/registry/web/api/SecureLdapIT.java @@ -92,7 +92,7 @@ public static class LdapTestConfiguration { @DependsOn({"directoryServer"}) // Can't load LdapUserGroupProvider until the embedded LDAP server, which creates the "directoryServer" bean, is running public static Authorizer getAuthorizer(@Autowired NiFiRegistryProperties properties, ExtensionManager extensionManager) { if (authorizerFactory == null) { - authorizerFactory = new AuthorizerFactory(properties, extensionManager); + authorizerFactory = new AuthorizerFactory(properties, extensionManager, null); } return authorizerFactory.getAuthorizer(); } From 8714059e1a7a5e0d57bfc9e9584a41443d0edd11 Mon Sep 17 00:00:00 2001 From: Kevin Doran Date: Tue, 12 Dec 2017 16:03:05 -0500 Subject: [PATCH 2/2] NIFIREG-61 Simplify Master Key Loading Simplifies the code associated with loading the master crypto key to standardize on using the bootstrap.conf file. --- .../registry/bootstrap/RunNiFiRegistry.java | 47 +--- .../IdentityProviderFactory.java | 2 +- .../nifi/registry/jetty/JettyServer.java | 2 + .../security/crypto/CryptoKeyLoader.java | 80 ++----- .../crypto/VolatileCryptoKeyProvider.java | 69 ------ .../apache/nifi/registry/NiFiRegistry.java | 67 +----- .../NiFiRegistryMasterKeyFactory.java | 115 --------- .../NiFiRegistryMasterKeyProviderFactory.java | 67 ++++++ .../nifi/registry/web/api/SecureLdapIT.java | 18 +- .../secure-ldap/authorizers.protected.xml | 221 ++++++++++++++++++ .../resources/conf/secure-ldap/bootstrap.conf | 60 +++++ .../identity-providers.protected.xml | 89 +++++++ .../conf/secure-ldap/nifi-registry.properties | 4 +- 13 files changed, 483 insertions(+), 358 deletions(-) delete mode 100644 nifi-registry-properties/src/main/java/org/apache/nifi/registry/security/crypto/VolatileCryptoKeyProvider.java delete mode 100644 nifi-registry-web-api/src/main/java/org/apache/nifi/registry/web/security/NiFiRegistryMasterKeyFactory.java create mode 100644 nifi-registry-web-api/src/main/java/org/apache/nifi/registry/web/security/NiFiRegistryMasterKeyProviderFactory.java create mode 100644 nifi-registry-web-api/src/test/resources/conf/secure-ldap/authorizers.protected.xml create mode 100644 nifi-registry-web-api/src/test/resources/conf/secure-ldap/bootstrap.conf create mode 100644 nifi-registry-web-api/src/test/resources/conf/secure-ldap/identity-providers.protected.xml diff --git a/nifi-registry-bootstrap/src/main/java/org/apache/nifi/registry/bootstrap/RunNiFiRegistry.java b/nifi-registry-bootstrap/src/main/java/org/apache/nifi/registry/bootstrap/RunNiFiRegistry.java index c7bec0428..c6d92ea25 100644 --- a/nifi-registry-bootstrap/src/main/java/org/apache/nifi/registry/bootstrap/RunNiFiRegistry.java +++ b/nifi-registry-bootstrap/src/main/java/org/apache/nifi/registry/bootstrap/RunNiFiRegistry.java @@ -16,8 +16,13 @@ */ package org.apache.nifi.registry.bootstrap; +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.registry.bootstrap.util.OSUtils; +import org.apache.nifi.registry.util.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.BufferedReader; -import java.io.BufferedWriter; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; @@ -34,12 +39,7 @@ import java.net.Socket; import java.nio.charset.StandardCharsets; import java.nio.file.Files; -import java.nio.file.Paths; -import java.nio.file.Path; -import java.nio.file.attribute.FileAttribute; import java.nio.file.attribute.PosixFilePermission; -import java.nio.file.attribute.PosixFilePermissions; -import java.nio.file.FileAlreadyExistsException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; @@ -58,11 +58,6 @@ import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; -import org.apache.commons.lang3.StringUtils; -import org.apache.nifi.registry.bootstrap.util.OSUtils; -import org.apache.nifi.registry.util.FileUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** *

@@ -951,36 +946,6 @@ public boolean accept(final File dir, final String filename) { cmd.add("-Dapp=NiFiRegistry"); cmd.add("-Dorg.apache.nifi.registry.bootstrap.config.log.dir=" + nifiRegistryLogDir); cmd.add("org.apache.nifi.registry.NiFiRegistry"); - if (props.containsKey(NIFI_REGISTRY_BOOTSTRAP_SENSITIVE_KEY) && !StringUtils.isBlank(props.get(NIFI_REGISTRY_BOOTSTRAP_SENSITIVE_KEY))) { - Path sensitiveKeyFile = Paths.get(confDir+"/sensitive.key"); - - try { - // Initially create file with the empty permission set (so nobody can get a file descriptor on it): - Set perms = new HashSet(); - FileAttribute> attr = PosixFilePermissions.asFileAttribute(perms); - sensitiveKeyFile = Files.createFile(sensitiveKeyFile, attr); - - // Then, once created, add owner-only rights: - perms.add(PosixFilePermission.OWNER_WRITE); - perms.add(PosixFilePermission.OWNER_READ); - attr = PosixFilePermissions.asFileAttribute(perms); - Files.setPosixFilePermissions(sensitiveKeyFile, perms); - - } catch (final FileAlreadyExistsException faee) { - cmdLogger.error("The sensitive.key file {} already exists. That shouldn't have been. Aborting.", sensitiveKeyFile); - System.exit(1); - } catch (final Exception e) { - cmdLogger.error("Other failure relating to setting permissions on {}. " - + "(so that only the owner can read it). " - + "This is fatal to the bootstrap process for security reasons. Exception was: {}", sensitiveKeyFile, e); - System.exit(1); - } - - BufferedWriter sensitiveKeyWriter = Files.newBufferedWriter(sensitiveKeyFile, StandardCharsets.UTF_8); - sensitiveKeyWriter.write(props.get(NIFI_REGISTRY_BOOTSTRAP_SENSITIVE_KEY)); - sensitiveKeyWriter.close(); - cmd.add("-K " + sensitiveKeyFile.toFile().getAbsolutePath()); - } builder.command(cmd); diff --git a/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authentication/IdentityProviderFactory.java b/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authentication/IdentityProviderFactory.java index 49bc1b5a0..3c2a3f4b0 100644 --- a/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authentication/IdentityProviderFactory.java +++ b/nifi-registry-framework/src/main/java/org/apache/nifi/registry/security/authentication/IdentityProviderFactory.java @@ -272,7 +272,7 @@ private void performFieldInjection(final IdentityProvider instance, final Class private String decryptValue(String cipherText, String encryptionScheme) throws SensitivePropertyProtectionException { if (sensitivePropertyProvider == null) { - throw new SensitivePropertyProtectionException("Sensitive Property Provider dependency was never wired, so protected" + + throw new SensitivePropertyProtectionException("Sensitive Property Provider dependency was never wired, so protected " + "properties cannot be decrypted. This usually indicates that a master key for this NiFi Registry was not " + "detected and configured during the bootstrap startup sequence. Contact the system administrator."); } diff --git a/nifi-registry-jetty/src/main/java/org/apache/nifi/registry/jetty/JettyServer.java b/nifi-registry-jetty/src/main/java/org/apache/nifi/registry/jetty/JettyServer.java index c63d59053..25c72f41b 100644 --- a/nifi-registry-jetty/src/main/java/org/apache/nifi/registry/jetty/JettyServer.java +++ b/nifi-registry-jetty/src/main/java/org/apache/nifi/registry/jetty/JettyServer.java @@ -238,7 +238,9 @@ private void loadWars() throws IOException { webUiContext = loadWar(webUiWar, "/nifi-registry"); webApiContext = loadWar(webApiWar, "/nifi-registry-api"); + logger.info("Adding {} object to ServletContext with key 'nifi-registry.properties'", properties.getClass().getSimpleName()); webApiContext.setAttribute("nifi-registry.properties", properties); + logger.info("Adding {} object to ServletContext with key 'nifi-registry.key'", masterKeyProvider.getClass().getSimpleName()); webApiContext.setAttribute("nifi-registry.key", masterKeyProvider); // there is an issue scanning the asm repackaged jar so narrow down what we are scanning diff --git a/nifi-registry-properties/src/main/java/org/apache/nifi/registry/security/crypto/CryptoKeyLoader.java b/nifi-registry-properties/src/main/java/org/apache/nifi/registry/security/crypto/CryptoKeyLoader.java index 483741815..d828773f3 100644 --- a/nifi-registry-properties/src/main/java/org/apache/nifi/registry/security/crypto/CryptoKeyLoader.java +++ b/nifi-registry-properties/src/main/java/org/apache/nifi/registry/security/crypto/CryptoKeyLoader.java @@ -21,13 +21,10 @@ import org.slf4j.LoggerFactory; import java.io.File; -import java.io.FileWriter; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; import java.util.Optional; -import java.util.Random; import java.util.stream.Stream; public class CryptoKeyLoader { @@ -45,95 +42,46 @@ public class CryptoKeyLoader { * @throws IOException if the file is not readable */ public static String extractKeyFromBootstrapFile(String bootstrapPath) throws IOException { - File expectedBootstrapFile; + File bootstrapFile; if (StringUtils.isBlank(bootstrapPath)) { logger.error("Cannot read from bootstrap.conf file to extract encryption key; location not specified"); throw new IOException("Cannot read from bootstrap.conf without file location"); } else { - expectedBootstrapFile = new File(bootstrapPath); + bootstrapFile = new File(bootstrapPath); } - if (expectedBootstrapFile.exists() && expectedBootstrapFile.canRead()) { - try (Stream stream = Files.lines(Paths.get(expectedBootstrapFile.getAbsolutePath()))) { + String keyValue; + if (bootstrapFile.exists() && bootstrapFile.canRead()) { + try (Stream stream = Files.lines(Paths.get(bootstrapFile.getAbsolutePath()))) { Optional keyLine = stream.filter(l -> l.startsWith(BOOTSTRAP_KEY_PREFIX)).findFirst(); if (keyLine.isPresent()) { - String keyValue = keyLine.get().split("=", 2)[1]; - return checkHexKey(keyValue); + keyValue = keyLine.get().split("=", 2)[1]; + keyValue = checkHexKey(keyValue); } else { - logger.warn("No encryption key present in the bootstrap.conf file at {}", expectedBootstrapFile.getAbsolutePath()); - return CryptoKeyProvider.EMPTY_KEY; + keyValue = CryptoKeyProvider.EMPTY_KEY; } } catch (IOException e) { - logger.error("Cannot read from bootstrap.conf file at {} to extract encryption key", expectedBootstrapFile.getAbsolutePath()); + logger.error("Cannot read from bootstrap.conf file at {} to extract encryption key", bootstrapFile.getAbsolutePath()); throw new IOException("Cannot read from bootstrap.conf", e); } } else { - logger.error("Cannot read from bootstrap.conf file at {} to extract encryption key -- file is missing or permissions are incorrect", expectedBootstrapFile.getAbsolutePath()); + logger.error("Cannot read from bootstrap.conf file at {} to extract encryption key -- file is missing or permissions are incorrect", bootstrapFile.getAbsolutePath()); throw new IOException("Cannot read from bootstrap.conf"); } - } - - /** - * Returns the key (if any) used to encrypt sensitive properties. - * The key extracted from the key file at the specified location. - * - * @param keyfilePath the path to the file containing the password/key - * @param securelyDeleteKeyfileOnSuccess If true, this method has the additional - * side-effect of overwriting the contents - * of the key file (with random bytes) and - * deleting it after successfully reading the key. - * If the key is not read from the file, secure - * deletion is not attempted even if this flag is set to true. - * @return the key in hexadecimal format, or {@link CryptoKeyProvider#EMPTY_KEY} if the key is null or empty - * @throws IOException if the file is not readable - */ - public static String extractKeyFromKeyFile(String keyfilePath, boolean securelyDeleteKeyfileOnSuccess) - throws IOException, IllegalArgumentException { - if (StringUtils.isBlank(keyfilePath)) { - logger.error("Cannot read from password file to extract encryption key; location not specified"); - throw new IOException("Cannot read from password file without file location"); + if (CryptoKeyProvider.EMPTY_KEY.equals(keyValue)) { + logger.info("No encryption key present in the bootstrap.conf file at {}", bootstrapFile.getAbsolutePath()); } - // Slurp in the contents of the file: - logger.info("Loading crypto key from file: {}", keyfilePath); - byte[] encoded = Files.readAllBytes(Paths.get(keyfilePath)); - String key = new String(encoded, StandardCharsets.UTF_8); - if (0 == key.length()) - throw new IllegalArgumentException("Key in keyfile " + keyfilePath + " yielded an empty key"); - - if (securelyDeleteKeyfileOnSuccess) { - // Overwrite the contents of the file (to avoid littering file system unlinked with key material): - logger.info("Now overwriting file '{}' with random bytes ", keyfilePath); - File password_file = new File(keyfilePath); - FileWriter overwriter = new FileWriter(password_file,false); - - // Construe a random pad: - Random r = new Random(); - StringBuffer sb = new StringBuffer(); - // Note on correctness: this pad is longer, but equally sufficient. - while(sb.length() < encoded.length){ - sb.append(Integer.toHexString(r.nextInt())); - } - String pad = sb.toString(); - logger.debug("Overwriting key material with pad: " + pad); - overwriter.write(pad); - overwriter.close(); - - logger.info("Removing/unlinking file: "+ keyfilePath); - password_file.delete(); - } - - return checkHexKey(key); + return keyValue; } private static String checkHexKey(String input) { if (input == null || input.trim().isEmpty()) { + logger.debug("Checking the hex key value that was loaded determined the key is empty."); return CryptoKeyProvider.EMPTY_KEY; } return input; } - - } diff --git a/nifi-registry-properties/src/main/java/org/apache/nifi/registry/security/crypto/VolatileCryptoKeyProvider.java b/nifi-registry-properties/src/main/java/org/apache/nifi/registry/security/crypto/VolatileCryptoKeyProvider.java deleted file mode 100644 index 1fd451ac4..000000000 --- a/nifi-registry-properties/src/main/java/org/apache/nifi/registry/security/crypto/VolatileCryptoKeyProvider.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.registry.security.crypto; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * An implementation of {@link CryptoKeyProvider} that loads the key from disk every time it is needed. - * - * The persistence-backing of the key is in the bootstrap.conf file, which must be provided to the - * constructor of this class. - * - * As key access for sensitive value decryption is only used a few times during server initialization, - * this implementation trades efficiency for security by only keeping the key in memory with an - * in-scope reference for a brief period of time (assuming callers do not maintain an in-scope reference). - * - * @see CryptoKeyProvider - */ -public final class VolatileCryptoKeyProvider implements CryptoKeyProvider { - - private static final Logger logger = LoggerFactory.getLogger(VolatileCryptoKeyProvider.class); - - private static final VolatileCryptoKeyProvider EMPTY = new VolatileCryptoKeyProvider(EMPTY_KEY); - - private final String key; - - /** - * Construct a new instance with an in-memory key. - * - * @param key The contents of the key in hexadecimal format. - * Must not be null. - */ - public VolatileCryptoKeyProvider(final String key) { - if (key == null) { - throw new IllegalArgumentException(VolatileCryptoKeyProvider.class.getSimpleName() + " cannot be initialized with a null key."); - } - this.key = key; - } - - @Override - public String getKey() throws MissingCryptoKeyException { - if (key == null) { - throw new MissingCryptoKeyException("Key is null."); - } - return key; - } - - @Override - public String toString() { - return "VolatileCryptoKeyProvider{" + - "key='[PROTECTED]'" + - '}'; - } -} diff --git a/nifi-registry-runtime/src/main/java/org/apache/nifi/registry/NiFiRegistry.java b/nifi-registry-runtime/src/main/java/org/apache/nifi/registry/NiFiRegistry.java index deb2a8daf..43f8ecfc4 100644 --- a/nifi-registry-runtime/src/main/java/org/apache/nifi/registry/NiFiRegistry.java +++ b/nifi-registry-runtime/src/main/java/org/apache/nifi/registry/NiFiRegistry.java @@ -17,14 +17,12 @@ package org.apache.nifi.registry; import org.apache.nifi.registry.jetty.JettyServer; -import org.apache.nifi.registry.security.crypto.BootstrapFileCryptoKeyProvider; -import org.apache.nifi.registry.security.crypto.CryptoKeyProvider; -import org.apache.nifi.registry.security.crypto.CryptoKeyLoader; -import org.apache.nifi.registry.security.crypto.MissingCryptoKeyException; import org.apache.nifi.registry.properties.NiFiRegistryProperties; import org.apache.nifi.registry.properties.NiFiRegistryPropertiesLoader; import org.apache.nifi.registry.properties.SensitivePropertyProtectionException; -import org.apache.nifi.registry.security.crypto.VolatileCryptoKeyProvider; +import org.apache.nifi.registry.security.crypto.BootstrapFileCryptoKeyProvider; +import org.apache.nifi.registry.security.crypto.CryptoKeyProvider; +import org.apache.nifi.registry.security.crypto.MissingCryptoKeyException; import org.apache.nifi.registry.util.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,9 +31,6 @@ import java.io.File; import java.io.IOException; import java.lang.reflect.InvocationTargetException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; import java.util.concurrent.TimeUnit; /** @@ -44,7 +39,6 @@ public class NiFiRegistry { private static final Logger LOGGER = LoggerFactory.getLogger(NiFiRegistry.class); - private static final String KEY_FILE_FLAG = "-K"; public static final String BOOTSTRAP_PORT_PROPERTY = "nifi.registry.bootstrap.listen.port"; @@ -153,7 +147,8 @@ public static void main(String[] args) { final CryptoKeyProvider masterKeyProvider; final NiFiRegistryProperties properties; try { - masterKeyProvider = createMasterKeyProvider(args); + masterKeyProvider = new BootstrapFileCryptoKeyProvider(REGISTRY_BOOTSTRAP_FILE_LOCATION); + LOGGER.info("Read property protection key from {}", REGISTRY_BOOTSTRAP_FILE_LOCATION); properties = initializeProperties(masterKeyProvider); } catch (final IllegalArgumentException iae) { throw new RuntimeException("Unable to load properties: " + iae, iae); @@ -194,56 +189,4 @@ private static NiFiRegistryProperties initializeProperties(CryptoKeyProvider mas } } - private static CryptoKeyProvider createMasterKeyProvider(String[] args) { - CryptoKeyProvider masterKeyProvider = null; - List parsedArgs = parseArgs(args); - - try { - if (parsedArgs.contains(KEY_FILE_FLAG)) { - LOGGER.debug("The bootstrap process provided the {} flag", KEY_FILE_FLAG); - int i = parsedArgs.indexOf(KEY_FILE_FLAG); - if (parsedArgs.size() <= i + 1) { - LOGGER.error("The bootstrap process passed the {} flag without a filename", KEY_FILE_FLAG); - throw new IllegalArgumentException("The bootstrap process provided the " + KEY_FILE_FLAG + " flag but no key"); - } - String passwordfilePath = parsedArgs.get(i + 1); - String key = CryptoKeyLoader.extractKeyFromKeyFile(passwordfilePath, true); - if (!isHexKeyValid(key)) { - throw new IllegalArgumentException("The key was not provided in valid hex format and of a valid length"); - } - masterKeyProvider = new VolatileCryptoKeyProvider(key); - LOGGER.info("Read property protection key from key file provided by bootstrap process"); - } else { - // if the key was not passed from bootstrap via -K arg, assume it is in bootstrap.conf - masterKeyProvider = new BootstrapFileCryptoKeyProvider(REGISTRY_BOOTSTRAP_FILE_LOCATION); - LOGGER.info("Read property protection key from {}", REGISTRY_BOOTSTRAP_FILE_LOCATION); - } - } catch (IOException e) { - LOGGER.error("Caught IOException while attempting to load master decryption key; aborting: " + e.toString()); - System.exit(1); - } - - return masterKeyProvider; - } - - private static List parseArgs(String[] args) { - List parsedArgs = new ArrayList<>(Arrays.asList(args)); - for (int i = 0; i < parsedArgs.size(); i++) { - if (parsedArgs.get(i).startsWith(KEY_FILE_FLAG + " ")) { - String[] split = parsedArgs.get(i).split(" ", 2); - parsedArgs.set(i, split[0]); - parsedArgs.add(i + 1, split[1]); - break; - } - } - return parsedArgs; - } - - private static boolean isHexKeyValid(String key) { - if (key == null || key.trim().isEmpty()) { - return false; - } - // Key length is in "nibbles" (i.e. one hex char = 4 bits) - return Arrays.asList(128, 196, 256).contains(key.length() * 4) && key.matches("^[0-9a-fA-F]*$"); - } } diff --git a/nifi-registry-web-api/src/main/java/org/apache/nifi/registry/web/security/NiFiRegistryMasterKeyFactory.java b/nifi-registry-web-api/src/main/java/org/apache/nifi/registry/web/security/NiFiRegistryMasterKeyFactory.java deleted file mode 100644 index 3c261d3ce..000000000 --- a/nifi-registry-web-api/src/main/java/org/apache/nifi/registry/web/security/NiFiRegistryMasterKeyFactory.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.registry.web.security; - -import org.apache.nifi.registry.NiFiRegistryApiApplication; -import org.apache.nifi.registry.security.crypto.BootstrapFileCryptoKeyProvider; -import org.apache.nifi.registry.security.crypto.CryptoKeyProvider; -import org.apache.nifi.registry.security.crypto.MissingCryptoKeyException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.context.ApplicationListener; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.event.ContextRefreshedEvent; - -import javax.servlet.ServletContextAttributeEvent; -import javax.servlet.ServletContextAttributeListener; - -@Configuration -public class NiFiRegistryMasterKeyFactory implements ServletContextAttributeListener, ApplicationListener { - - private static final Logger logger = LoggerFactory.getLogger(NiFiRegistryMasterKeyFactory.class); - - private CryptoKeyProvider masterKeyProvider = MISSING_KEY_PROVIDER; - - @Bean - public CryptoKeyProvider getNiFiRegistryMasterKeyProvider() { - return masterKeyProvider; - } - - // -- ServletContextAttributeListener methods - - @Override - public void attributeAdded(ServletContextAttributeEvent servletContextAttributeEvent) { - updateMasterKeyProvider(servletContextAttributeEvent); - } - - @Override - public void attributeReplaced(ServletContextAttributeEvent servletContextAttributeEvent) { - updateMasterKeyProvider(servletContextAttributeEvent); - } - - @Override - public void attributeRemoved(ServletContextAttributeEvent servletContextAttributeEvent) { - String attributeName = servletContextAttributeEvent.getName(); - if (attributeName != null && attributeName.equals(NiFiRegistryApiApplication.NIFI_REGISTRY_MASTER_KEY_ATTRIBUTE)) { - clearMasterKeyProvider(); - } - } - - // -- ApplicationListener methods - - @Override - public void onApplicationEvent(ContextRefreshedEvent contextRefreshedEvent) { - if (masterKeyProvider != null) { - if (!(masterKeyProvider instanceof BootstrapFileCryptoKeyProvider)) { - // If the Key Holder is not backed by the bootstrap.conf file, - // if might be holding the key material in memory, and in scope. - // If we receive this event, the ApplicationContext has finished initializing, - // so all sensitive props should be loaded and the master key is no longer needed. - // De-reference the holder so that GC can purge the key material from memory. - logger.info("Received {} indicating the ApplicationContext is done initializing. Now clearing the CryptoKeyProvider " + - "Bean holding the master key from ApplicationContext, as it should no longer be needed.", contextRefreshedEvent); - clearMasterKeyProvider(); - } else { - logger.debug("Received {} indicating the ApplicationContext is done initializing. CryptoKeyProvider instance is backed " + - "by bootstrap.conf file and will remain available in the ApplicationContext to load master key on demand if needed."); - } - } - - } - - private void updateMasterKeyProvider(ServletContextAttributeEvent servletContextAttributeEvent) { - String attributeName = servletContextAttributeEvent.getName(); - if (attributeName != null && attributeName.equals(NiFiRegistryApiApplication.NIFI_REGISTRY_MASTER_KEY_ATTRIBUTE)) { - Object attributeValue = servletContextAttributeEvent.getValue(); - if (attributeValue == null || !(attributeValue instanceof CryptoKeyProvider)) { - clearMasterKeyProvider(); - } else { - logger.debug("Received {}, updating CryptoKeyProvider Bean containing the master key.", servletContextAttributeEvent); - masterKeyProvider = (CryptoKeyProvider)attributeValue; - } - } - } - - private void clearMasterKeyProvider() { - logger.debug("Clearing CryptoKeyProvider Bean containing the master key. Master key will no longer be accessible in ApplicationContext."); - masterKeyProvider = MISSING_KEY_PROVIDER; - // actual master key holder is now out of scope and can be reaped from memory at next GC run - } - - private static final CryptoKeyProvider MISSING_KEY_PROVIDER = new CryptoKeyProvider() { - @Override - public String getKey() throws MissingCryptoKeyException { - throw new MissingCryptoKeyException("The actual CryptoKeyProvider used for ApplicationContext " + - "loading has been intentionally destroyed and is no longer accessible. Something is " + - "trying to access the master key after ApplicationContext has finished loading."); - } - }; - -} diff --git a/nifi-registry-web-api/src/main/java/org/apache/nifi/registry/web/security/NiFiRegistryMasterKeyProviderFactory.java b/nifi-registry-web-api/src/main/java/org/apache/nifi/registry/web/security/NiFiRegistryMasterKeyProviderFactory.java new file mode 100644 index 000000000..8026ccabb --- /dev/null +++ b/nifi-registry-web-api/src/main/java/org/apache/nifi/registry/web/security/NiFiRegistryMasterKeyProviderFactory.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.registry.web.security; + +import org.apache.nifi.registry.NiFiRegistryApiApplication; +import org.apache.nifi.registry.security.crypto.CryptoKeyProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.web.context.ServletContextAware; + +import javax.servlet.ServletContext; + +@Configuration +public class NiFiRegistryMasterKeyProviderFactory implements ServletContextAware { + + private static final Logger logger = LoggerFactory.getLogger(NiFiRegistryMasterKeyProviderFactory.class); + + private CryptoKeyProvider masterKeyProvider = null; + + @Bean + public CryptoKeyProvider getNiFiRegistryMasterKeyProvider() { + return masterKeyProvider; + } + + @Override + public void setServletContext(ServletContext servletContext) { + Object rawKeyProviderObject = servletContext.getAttribute(NiFiRegistryApiApplication.NIFI_REGISTRY_MASTER_KEY_ATTRIBUTE); + + if (rawKeyProviderObject == null) { + logger.warn("Value of {} was null. " + + "{} bean will not be available in Application Context, so any attempt to load protected property values may fail.", + NiFiRegistryApiApplication.NIFI_REGISTRY_MASTER_KEY_ATTRIBUTE, + CryptoKeyProvider.class.getSimpleName()); + return; + } + + if (!(rawKeyProviderObject instanceof CryptoKeyProvider)) { + logger.warn("Expected value of {} to be of type {}, but instead got {}. " + + "{} bean will NOT be available in Application Context, so any attempt to load protected property values may fail.", + NiFiRegistryApiApplication.NIFI_REGISTRY_MASTER_KEY_ATTRIBUTE, + CryptoKeyProvider.class.getName(), + rawKeyProviderObject.getClass().getName(), + CryptoKeyProvider.class.getSimpleName()); + return; + } + + logger.info("Updating Application Context with {} bean for obtaining NiFi Registry master key.", CryptoKeyProvider.class.getSimpleName()); + masterKeyProvider = (CryptoKeyProvider) rawKeyProviderObject; + } + +} diff --git a/nifi-registry-web-api/src/test/java/org/apache/nifi/registry/web/api/SecureLdapIT.java b/nifi-registry-web-api/src/test/java/org/apache/nifi/registry/web/api/SecureLdapIT.java index 8f3d64c82..e7af1d420 100644 --- a/nifi-registry-web-api/src/test/java/org/apache/nifi/registry/web/api/SecureLdapIT.java +++ b/nifi-registry-web-api/src/test/java/org/apache/nifi/registry/web/api/SecureLdapIT.java @@ -25,9 +25,13 @@ import org.apache.nifi.registry.authorization.CurrentUser; import org.apache.nifi.registry.authorization.Permissions; import org.apache.nifi.registry.authorization.Tenant; +import org.apache.nifi.registry.properties.AESSensitivePropertyProvider; import org.apache.nifi.registry.properties.NiFiRegistryProperties; +import org.apache.nifi.registry.properties.SensitivePropertyProvider; import org.apache.nifi.registry.security.authorization.Authorizer; import org.apache.nifi.registry.security.authorization.AuthorizerFactory; +import org.apache.nifi.registry.security.crypto.BootstrapFileCryptoKeyProvider; +import org.apache.nifi.registry.security.crypto.CryptoKeyProvider; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -90,13 +94,23 @@ public static class LdapTestConfiguration { @Primary @Bean @DependsOn({"directoryServer"}) // Can't load LdapUserGroupProvider until the embedded LDAP server, which creates the "directoryServer" bean, is running - public static Authorizer getAuthorizer(@Autowired NiFiRegistryProperties properties, ExtensionManager extensionManager) { + public static Authorizer getAuthorizer(@Autowired NiFiRegistryProperties properties, ExtensionManager extensionManager) throws Exception { if (authorizerFactory == null) { - authorizerFactory = new AuthorizerFactory(properties, extensionManager, null); + authorizerFactory = new AuthorizerFactory(properties, extensionManager, sensitivePropertyProvider()); } return authorizerFactory.getAuthorizer(); } + @Primary + @Bean + public static SensitivePropertyProvider sensitivePropertyProvider() throws Exception { + return new AESSensitivePropertyProvider(getNiFiRegistryMasterKeyProvider().getKey()); + } + + private static CryptoKeyProvider getNiFiRegistryMasterKeyProvider() { + return new BootstrapFileCryptoKeyProvider("src/test/resources/conf/secure-ldap/bootstrap.conf"); + } + } private String adminAuthToken; diff --git a/nifi-registry-web-api/src/test/resources/conf/secure-ldap/authorizers.protected.xml b/nifi-registry-web-api/src/test/resources/conf/secure-ldap/authorizers.protected.xml new file mode 100644 index 000000000..44007bdd5 --- /dev/null +++ b/nifi-registry-web-api/src/test/resources/conf/secure-ldap/authorizers.protected.xml @@ -0,0 +1,221 @@ + + + + + + + + ldap-user-group-provider + org.apache.nifi.registry.security.ldap.tenants.LdapUserGroupProvider + SIMPLE + + cn=read-only-admin,dc=example,dc=com + + oVU8w3uH7yZlKscG||Hu4ZtRgZWKISn3DyGuB50rKL1qGceWZp + + + + FOLLOW + 10 secs + 10 secs + + ldap://localhost:8389 + + 30 mins + + dc=example,dc=com + person + ONE_LEVEL + (uid=*) + uid + + + dc=example,dc=com + groupOfUniqueNames + ONE_LEVEL + (ou=*) + ou + uniqueMember + + + + + + + + + + + file-access-policy-provider + org.apache.nifi.registry.security.authorization.file.FileAccessPolicyProvider + ldap-user-group-provider + ./target/test-classes/conf/secure-ldap/authorizations.xml + nifiadmin + + + + + managed-authorizer + org.apache.nifi.registry.security.authorization.StandardManagedAuthorizer + file-access-policy-provider + + + \ No newline at end of file diff --git a/nifi-registry-web-api/src/test/resources/conf/secure-ldap/bootstrap.conf b/nifi-registry-web-api/src/test/resources/conf/secure-ldap/bootstrap.conf new file mode 100644 index 000000000..4bd28baf8 --- /dev/null +++ b/nifi-registry-web-api/src/test/resources/conf/secure-ldap/bootstrap.conf @@ -0,0 +1,60 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Java command to use when running nifi-registry +java=java + +# Username to use when running nifi-registry. This value will be ignored on Windows. +run.as= + +# Configure where nifi-registry's lib and conf directories live +lib.dir=./lib +conf.dir=./conf + +# How long to wait after telling nifi-registry to shutdown before explicitly killing the Process +graceful.shutdown.seconds=20 + +# Disable JSR 199 so that we can use JSP's without running a JDK +java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true + +# JVM memory settings +java.arg.2=-Xms512m +java.arg.3=-Xmx512m + +# Enable Remote Debugging +java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000 + +java.arg.4=-Djava.net.preferIPv4Stack=true + +# allowRestrictedHeaders is required for Cluster/Node communications to work properly +java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true +java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol + +# Java 7 and below have issues with Code Cache. The following lines allow us to run well even with +# many classes loaded in the JVM. +#java.arg.7=-XX:ReservedCodeCacheSize=256m +#java.arg.8=-XX:CodeCacheFlushingMinimumFreeSpace=10m +#java.arg.9=-XX:+UseCodeCacheFlushing +#java.arg.11=-XX:PermSize=128M +#java.arg.12=-XX:MaxPermSize=128M + +# The G1GC is still considered experimental but has proven to be very advantageous in providing great +# performance without significant "stop-the-world" delays. +#java.arg.10=-XX:+UseG1GC + +# Master key in hexadecimal format for encrypted sensitive configuration values +nifi.registry.bootstrap.sensitive.key=0123456789ABCDEFFEDCBA9876543210 diff --git a/nifi-registry-web-api/src/test/resources/conf/secure-ldap/identity-providers.protected.xml b/nifi-registry-web-api/src/test/resources/conf/secure-ldap/identity-providers.protected.xml new file mode 100644 index 000000000..51336e22c --- /dev/null +++ b/nifi-registry-web-api/src/test/resources/conf/secure-ldap/identity-providers.protected.xml @@ -0,0 +1,89 @@ + + + + + + + ldap-identity-provider + org.apache.nifi.registry.security.ldap.LdapIdentityProvider + SIMPLE + + cn=read-only-admin,dc=example,dc=com + + p43I3jRcK+wPhR3c||oaqMg3YGo2WblTxBJSgI8H9fLMBwQiaM + + FOLLOW + 10 secs + 10 secs + + ldap://localhost:8389 + dc=example,dc=com + (uid={0}) + + USE_USERNAME + 12 hours + + + \ No newline at end of file diff --git a/nifi-registry-web-api/src/test/resources/conf/secure-ldap/nifi-registry.properties b/nifi-registry-web-api/src/test/resources/conf/secure-ldap/nifi-registry.properties index 40b291684..1b46ac24d 100644 --- a/nifi-registry-web-api/src/test/resources/conf/secure-ldap/nifi-registry.properties +++ b/nifi-registry-web-api/src/test/resources/conf/secure-ldap/nifi-registry.properties @@ -23,9 +23,9 @@ nifi.registry.web.https.port=0 # # ** Server KeyStore and TrustStore configuration set in Spring profile properties for embedded Jetty ** # -nifi.registry.security.authorizers.configuration.file=./target/test-classes/conf/secure-ldap/authorizers.xml +nifi.registry.security.authorizers.configuration.file=./target/test-classes/conf/secure-ldap/authorizers.protected.xml nifi.registry.security.authorizer=managed-authorizer -nifi.registry.security.identity.providers.configuration.file=./target/test-classes/conf/secure-ldap/identity-providers.xml +nifi.registry.security.identity.providers.configuration.file=./target/test-classes/conf/secure-ldap/identity-providers.protected.xml nifi.registry.security.identity.provider=ldap-identity-provider # providers properties #