From ebd0a8ee06cfdec0330d1ee07bc248ae8d71f1fa Mon Sep 17 00:00:00 2001 From: Bryan Rosander Date: Tue, 18 Apr 2017 15:25:08 -0400 Subject: [PATCH 1/3] MINIFI-272 - Delegating C2 Provider, Caching ConfigService, Tests --- .../ingestors/PullHttpChangeIngestor.java | 33 ++- .../nifi/minifi/c2/api/Configuration.java | 2 + .../minifi/c2/api/ConfigurationProvider.java | 8 +- .../api/ConfigurationProviderException.java | 14 ++ .../c2/api/cache/ConfigurationCache.java | 2 +- .../c2/api/properties/C2Properties.java | 8 +- .../authorization/AuthorizationException.java | 4 +- minifi-c2/minifi-c2-assembly/pom.xml | 5 + .../main/resources/conf/authorizations.yaml | 11 +- .../main/resources/conf/minifi-c2-context.xml | 9 +- .../{config.yml.v1 => config.text.yml.v1} | 0 .../FileSystemConfigurationCache.java | 15 +- minifi-c2/minifi-c2-integration-tests/pom.xml | 11 + ...gatingConfigurationProviderSecureTest.java | 79 +++++++ ...tingConfigurationProviderUnsecureTest.java | 37 ++++ .../conf/minifi-c2-context.xml | 56 +++++ .../c2-secure-rest/conf/minifi-c2-context.xml | 3 + .../conf/minifi-c2-context.xml | 56 +++++ .../conf/minifi-c2-context.xml | 3 + .../c2-upstream-secure/conf/authorities.yaml | 17 ++ .../conf/authorizations.yaml | 32 +++ .../{config.yml.v1 => config.text.yml.v1} | 0 .../{config.yml.v1 => config.text.yml.v1} | 0 .../{config.yml.v2 => config.text.yml.v2} | 0 ...r-compose-DelegatingProviderSecureTest.yml | 55 +++++ ...compose-DelegatingProviderUnsecureTest.yml | 32 +++ .../nifi/minifi/c2/jetty/JettyServer.java | 8 +- .../cache/CacheConfigurationProvider.java | 14 +- .../cache/CacheConfigurationProviderTest.java | 18 +- .../minifi-c2-provider-delegating/pom.xml | 53 +++++ .../DelegatingConfigurationProvider.java | 161 +++++++++++++++ .../DelegatingConfigurationProviderTest.java | 195 ++++++++++++++++++ .../minifi-c2-provider-nifi-rest/pom.xml | 5 + .../rest/NiFiRestConfigurationProvider.java | 87 +++++--- .../provider/nifi/rest/NiFiRestConnector.java | 80 ------- .../provider/nifi/rest/TemplatesIterator.java | 5 +- .../NiFiRestConfigurationProviderTest.java | 12 +- .../nifi/rest/TemplatesIteratorTest.java | 15 +- .../minifi-c2-provider-util/pom.xml | 44 ++++ .../c2/provider/util/HttpConnector.java | 130 ++++++++++++ minifi-c2/minifi-c2-provider/pom.xml | 2 + minifi-c2/minifi-c2-service/pom.xml | 10 + .../nifi/minifi/c2/service/ConfigService.java | 160 +++++++++++--- .../c2/service/ConfigurationProviderInfo.java | 51 +++++ .../c2/service/ConfigurationProviderKey.java | 61 ++++++ .../service/ConfigurationProviderValue.java | 49 +++++ .../src/main/markdown/System_Admin_Guide.md | 4 + minifi-integration-tests/pom.xml | 29 +++ .../c2/HierarchicalC2IntegrationTest.java | 125 +++++++++++ .../standalone/test/StandaloneYamlTest.java | 41 +--- .../nifi/minifi/integration/util/LogUtil.java | 85 ++++++++ .../c2-authoritative/conf/authorities.yaml | 21 ++ .../c2-authoritative/conf/authorizations.yaml | 41 ++++ .../c2-authoritative/conf/c2.properties | 27 +++ .../conf/minifi-c2-context.xml | 63 ++++++ .../files/edge1/raspi3/config.text.yml.v1 | 63 ++++++ .../files/edge2/raspi2/config.text.yml.v1 | 63 ++++++ .../files/edge3/raspi3/config.text.yml.v1 | 63 ++++++ .../hierarchical/c2-edge2/conf/c2.properties | 27 +++ .../c2-edge2/conf/minifi-c2-context.xml | 62 ++++++ .../hierarchical/minifi-edge1/bootstrap.conf | 106 ++++++++++ .../hierarchical/minifi-edge1/expected.json | 8 + .../hierarchical/minifi-edge2/bootstrap.conf | 99 +++++++++ .../hierarchical/minifi-edge2/expected.json | 8 + .../hierarchical/minifi-edge3/bootstrap.conf | 109 ++++++++++ .../hierarchical/minifi-edge3/expected.json | 8 + .../docker-compose-c2-hierarchical.yml | 126 +++++++++++ .../src/test/resources/logback.xml | 2 + .../src/test/resources/squid/squid.conf | 19 ++ .../standalone/v1/CsvToJson/xml/expected.json | 10 +- .../standalone/v1/CsvToJson/yml/expected.json | 10 +- .../xml/expected.json | 10 +- .../yml/expected.json | 10 +- .../MiNiFiTailLogAttribute/xml/expected.json | 10 +- .../MiNiFiTailLogAttribute/yml/expected.json | 10 +- .../xml/expected.json | 10 +- .../yml/expected.json | 10 +- .../MultipleRelationships/xml/expected.json | 10 +- .../MultipleRelationships/yml/expected.json | 10 +- .../v2/ProcessGroups/xml/expected.json | 10 +- .../v2/ProcessGroups/yml/expected.json | 10 +- .../v2/StressTestFramework/xml/expected.json | 10 +- .../v2/StressTestFramework/yml/expected.json | 10 +- .../src/test/resources/tailFileServer.py | 12 +- 84 files changed, 2729 insertions(+), 274 deletions(-) rename minifi-c2/minifi-c2-assembly/src/main/resources/files/raspi3/{config.yml.v1 => config.text.yml.v1} (100%) create mode 100644 minifi-c2/minifi-c2-integration-tests/src/test/java/org/apache/nifi/minifi/c2/integration/test/DelegatingConfigurationProviderSecureTest.java create mode 100644 minifi-c2/minifi-c2-integration-tests/src/test/java/org/apache/nifi/minifi/c2/integration/test/DelegatingConfigurationProviderUnsecureTest.java create mode 100644 minifi-c2/minifi-c2-integration-tests/src/test/resources/c2-secure-delegating/conf/minifi-c2-context.xml create mode 100644 minifi-c2/minifi-c2-integration-tests/src/test/resources/c2-unsecure-delegating/conf/minifi-c2-context.xml create mode 100644 minifi-c2/minifi-c2-integration-tests/src/test/resources/c2-upstream-secure/conf/authorities.yaml create mode 100644 minifi-c2/minifi-c2-integration-tests/src/test/resources/c2-upstream-secure/conf/authorizations.yaml rename minifi-c2/minifi-c2-integration-tests/src/test/resources/c2/files/raspi2/{config.yml.v1 => config.text.yml.v1} (100%) rename minifi-c2/minifi-c2-integration-tests/src/test/resources/c2/files/raspi3/{config.yml.v1 => config.text.yml.v1} (100%) rename minifi-c2/minifi-c2-integration-tests/src/test/resources/c2/files/raspi3/{config.yml.v2 => config.text.yml.v2} (100%) create mode 100644 minifi-c2/minifi-c2-integration-tests/src/test/resources/docker-compose-DelegatingProviderSecureTest.yml create mode 100644 minifi-c2/minifi-c2-integration-tests/src/test/resources/docker-compose-DelegatingProviderUnsecureTest.yml create mode 100644 minifi-c2/minifi-c2-provider/minifi-c2-provider-delegating/pom.xml create mode 100644 minifi-c2/minifi-c2-provider/minifi-c2-provider-delegating/src/main/java/org/apache/nifi/minifi/c2/provider/delegating/DelegatingConfigurationProvider.java create mode 100644 minifi-c2/minifi-c2-provider/minifi-c2-provider-delegating/src/test/java/org/apache/nifi/minifi/c2/provider/delegating/DelegatingConfigurationProviderTest.java delete mode 100644 minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/main/java/org/apache/nifi/minifi/c2/provider/nifi/rest/NiFiRestConnector.java create mode 100644 minifi-c2/minifi-c2-provider/minifi-c2-provider-util/pom.xml create mode 100644 minifi-c2/minifi-c2-provider/minifi-c2-provider-util/src/main/java/org/apache/nifi/minifi/c2/provider/util/HttpConnector.java create mode 100644 minifi-c2/minifi-c2-service/src/main/java/org/apache/nifi/minifi/c2/service/ConfigurationProviderInfo.java create mode 100644 minifi-c2/minifi-c2-service/src/main/java/org/apache/nifi/minifi/c2/service/ConfigurationProviderKey.java create mode 100644 minifi-c2/minifi-c2-service/src/main/java/org/apache/nifi/minifi/c2/service/ConfigurationProviderValue.java create mode 100644 minifi-integration-tests/src/test/java/org/apache/nifi/minifi/integration/c2/HierarchicalC2IntegrationTest.java create mode 100644 minifi-integration-tests/src/test/java/org/apache/nifi/minifi/integration/util/LogUtil.java create mode 100644 minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/conf/authorities.yaml create mode 100644 minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/conf/authorizations.yaml create mode 100644 minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/conf/c2.properties create mode 100644 minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/conf/minifi-c2-context.xml create mode 100644 minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/files/edge1/raspi3/config.text.yml.v1 create mode 100644 minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/files/edge2/raspi2/config.text.yml.v1 create mode 100644 minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/files/edge3/raspi3/config.text.yml.v1 create mode 100644 minifi-integration-tests/src/test/resources/c2/hierarchical/c2-edge2/conf/c2.properties create mode 100644 minifi-integration-tests/src/test/resources/c2/hierarchical/c2-edge2/conf/minifi-c2-context.xml create mode 100644 minifi-integration-tests/src/test/resources/c2/hierarchical/minifi-edge1/bootstrap.conf create mode 100644 minifi-integration-tests/src/test/resources/c2/hierarchical/minifi-edge1/expected.json create mode 100644 minifi-integration-tests/src/test/resources/c2/hierarchical/minifi-edge2/bootstrap.conf create mode 100644 minifi-integration-tests/src/test/resources/c2/hierarchical/minifi-edge2/expected.json create mode 100644 minifi-integration-tests/src/test/resources/c2/hierarchical/minifi-edge3/bootstrap.conf create mode 100644 minifi-integration-tests/src/test/resources/c2/hierarchical/minifi-edge3/expected.json create mode 100644 minifi-integration-tests/src/test/resources/docker-compose-c2-hierarchical.yml create mode 100644 minifi-integration-tests/src/test/resources/squid/squid.conf diff --git a/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/PullHttpChangeIngestor.java b/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/PullHttpChangeIngestor.java index 3ec26d0f7..6c8adcc12 100644 --- a/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/PullHttpChangeIngestor.java +++ b/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/PullHttpChangeIngestor.java @@ -18,6 +18,7 @@ package org.apache.nifi.minifi.bootstrap.configuration.ingestors; import okhttp3.Call; +import okhttp3.Credentials; import okhttp3.HttpUrl; import okhttp3.OkHttpClient; import okhttp3.Request; @@ -37,6 +38,9 @@ import javax.net.ssl.TrustManagerFactory; import javax.net.ssl.X509TrustManager; import java.io.FileInputStream; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.Proxy; import java.nio.ByteBuffer; import java.security.KeyStore; import java.security.NoSuchAlgorithmException; @@ -73,6 +77,10 @@ public class PullHttpChangeIngestor extends AbstractPullChangeIngestor { public static final String HOST_KEY = PULL_HTTP_BASE_KEY + ".hostname"; public static final String PATH_KEY = PULL_HTTP_BASE_KEY + ".path"; public static final String QUERY_KEY = PULL_HTTP_BASE_KEY + ".query"; + public static final String PROXY_HOST_KEY = PULL_HTTP_BASE_KEY + ".proxy.hostname"; + public static final String PROXY_PORT_KEY = PULL_HTTP_BASE_KEY + ".proxy.port"; + public static final String PROXY_USERNAME = PULL_HTTP_BASE_KEY + ".proxy.username"; + public static final String PROXY_PASSWORD = PULL_HTTP_BASE_KEY + ".proxy.password"; public static final String TRUSTSTORE_LOCATION_KEY = PULL_HTTP_BASE_KEY + ".truststore.location"; public static final String TRUSTSTORE_PASSWORD_KEY = PULL_HTTP_BASE_KEY + ".truststore.password"; public static final String TRUSTSTORE_TYPE_KEY = PULL_HTTP_BASE_KEY + ".truststore.type"; @@ -147,6 +155,23 @@ public void initialize(Properties properties, ConfigurationFileHolder configurat // Set whether to follow redirects okHttpClientBuilder.followRedirects(true); + String proxyHost = properties.getProperty(PROXY_HOST_KEY, ""); + if (!proxyHost.isEmpty()) { + String proxyPort = properties.getProperty(PROXY_PORT_KEY); + if (proxyPort == null || proxyPort.isEmpty()) { + throw new IllegalArgumentException("Proxy port required if proxy specified."); + } + okHttpClientBuilder.proxy(new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, Integer.parseInt(proxyPort)))); + String proxyUsername = properties.getProperty(PROXY_USERNAME); + if (proxyUsername != null) { + String proxyPassword = properties.getProperty(PROXY_PASSWORD); + if (proxyPassword == null) { + throw new IllegalArgumentException("Must specify proxy password with proxy username."); + } + okHttpClientBuilder.proxyAuthenticator((route, response) -> response.request().newBuilder().addHeader("Proxy-Authorization", Credentials.basic(proxyUsername, proxyPassword)).build()); + } + } + // check if the ssl path is set and add the factory if so if (properties.containsKey(KEYSTORE_LOCATION_KEY)) { try { @@ -210,10 +235,16 @@ public void run() { logger.debug("Response received: {}", response.toString()); - if (response.code() == NOT_MODIFIED_STATUS_CODE) { + int code = response.code(); + + if (code == NOT_MODIFIED_STATUS_CODE) { return; } + if (code >= 400) { + throw new IOException("Got response code " + code + " while trying to pull configuration: " + response.body().string()); + } + ResponseBody body = response.body(); if (body == null) { logger.warn("No body returned when pulling a new configuration"); diff --git a/minifi-c2/minifi-c2-api/src/main/java/org/apache/nifi/minifi/c2/api/Configuration.java b/minifi-c2/minifi-c2-api/src/main/java/org/apache/nifi/minifi/c2/api/Configuration.java index a008f0acb..90fbdba34 100644 --- a/minifi-c2/minifi-c2-api/src/main/java/org/apache/nifi/minifi/c2/api/Configuration.java +++ b/minifi-c2/minifi-c2-api/src/main/java/org/apache/nifi/minifi/c2/api/Configuration.java @@ -21,6 +21,8 @@ /** * Represents a MiNiFi configuration of a given version, format matches the format of the ConfigurationProvider + * + * This object may be cached so it should attempt to minimize the amount of memory used to represent state (input stream should come from persistent storage if possible.) */ public interface Configuration { /** diff --git a/minifi-c2/minifi-c2-api/src/main/java/org/apache/nifi/minifi/c2/api/ConfigurationProvider.java b/minifi-c2/minifi-c2-api/src/main/java/org/apache/nifi/minifi/c2/api/ConfigurationProvider.java index a743be332..4892be57a 100644 --- a/minifi-c2/minifi-c2-api/src/main/java/org/apache/nifi/minifi/c2/api/ConfigurationProvider.java +++ b/minifi-c2/minifi-c2-api/src/main/java/org/apache/nifi/minifi/c2/api/ConfigurationProvider.java @@ -25,11 +25,11 @@ */ public interface ConfigurationProvider { /** - * Gets the content type that this provider returns + * Gets the content types that this provider returns * - * @return the content type that this provider returns + * @return the content types that this provider returns */ - String getContentType(); + List getContentTypes() throws ConfigurationProviderException; /** * Gets the configuration that corresponds to the passed in parameters @@ -39,5 +39,5 @@ public interface ConfigurationProvider { * @return an input stream of the configuration * @throws ConfigurationProviderException if there is an error in the configuration */ - Configuration getConfiguration(Integer version, Map> parameters) throws ConfigurationProviderException; + Configuration getConfiguration(String contentType, Integer version, Map> parameters) throws ConfigurationProviderException; } diff --git a/minifi-c2/minifi-c2-api/src/main/java/org/apache/nifi/minifi/c2/api/ConfigurationProviderException.java b/minifi-c2/minifi-c2-api/src/main/java/org/apache/nifi/minifi/c2/api/ConfigurationProviderException.java index 16b1ed28b..d51048270 100644 --- a/minifi-c2/minifi-c2-api/src/main/java/org/apache/nifi/minifi/c2/api/ConfigurationProviderException.java +++ b/minifi-c2/minifi-c2-api/src/main/java/org/apache/nifi/minifi/c2/api/ConfigurationProviderException.java @@ -25,4 +25,18 @@ public ConfigurationProviderException(String message) { public ConfigurationProviderException(String message, Throwable cause) { super(message, cause); } + + public ConfigurationProviderException.Wrapper wrap() { + return new Wrapper(this); + } + + public static class Wrapper extends RuntimeException { + public Wrapper(ConfigurationProviderException cause) { + super(cause); + } + + public ConfigurationProviderException unwrap() { + return (ConfigurationProviderException) getCause(); + } + } } diff --git a/minifi-c2/minifi-c2-api/src/main/java/org/apache/nifi/minifi/c2/api/cache/ConfigurationCache.java b/minifi-c2/minifi-c2-api/src/main/java/org/apache/nifi/minifi/c2/api/cache/ConfigurationCache.java index 00c271e69..43d4b93e5 100644 --- a/minifi-c2/minifi-c2-api/src/main/java/org/apache/nifi/minifi/c2/api/cache/ConfigurationCache.java +++ b/minifi-c2/minifi-c2-api/src/main/java/org/apache/nifi/minifi/c2/api/cache/ConfigurationCache.java @@ -33,5 +33,5 @@ public interface ConfigurationCache { * @return information on the entry * @throws InvalidParameterException if there are illegal/invalid parameters */ - ConfigurationCacheFileInfo getCacheFileInfo(Map> parameters) throws InvalidParameterException; + ConfigurationCacheFileInfo getCacheFileInfo(String contentType, Map> parameters) throws InvalidParameterException; } diff --git a/minifi-c2/minifi-c2-api/src/main/java/org/apache/nifi/minifi/c2/api/properties/C2Properties.java b/minifi-c2/minifi-c2-api/src/main/java/org/apache/nifi/minifi/c2/api/properties/C2Properties.java index 2a8df3a7d..f24958344 100644 --- a/minifi-c2/minifi-c2-api/src/main/java/org/apache/nifi/minifi/c2/api/properties/C2Properties.java +++ b/minifi-c2/minifi-c2-api/src/main/java/org/apache/nifi/minifi/c2/api/properties/C2Properties.java @@ -58,11 +58,11 @@ public static C2Properties getInstance() { return properties; } - public SslContextFactory getSslContextFactory() throws GeneralSecurityException, IOException { - if (!Boolean.valueOf(getProperty(MINIFI_C2_SERVER_SECURE, "false"))) { - return null; - } + public boolean isSecure() { + return Boolean.valueOf(getProperty(MINIFI_C2_SERVER_SECURE, "false")); + } + public SslContextFactory getSslContextFactory() throws GeneralSecurityException, IOException { SslContextFactory sslContextFactory = new SslContextFactory(); KeyStore keyStore = KeyStore.getInstance(properties.getProperty(MINIFI_C2_SERVER_KEYSTORE_TYPE)); Path keyStorePath = Paths.get(C2_SERVER_HOME).resolve(properties.getProperty(MINIFI_C2_SERVER_KEYSTORE)).toAbsolutePath(); diff --git a/minifi-c2/minifi-c2-api/src/main/java/org/apache/nifi/minifi/c2/api/security/authorization/AuthorizationException.java b/minifi-c2/minifi-c2-api/src/main/java/org/apache/nifi/minifi/c2/api/security/authorization/AuthorizationException.java index 2c459e0b8..454288a9e 100644 --- a/minifi-c2/minifi-c2-api/src/main/java/org/apache/nifi/minifi/c2/api/security/authorization/AuthorizationException.java +++ b/minifi-c2/minifi-c2-api/src/main/java/org/apache/nifi/minifi/c2/api/security/authorization/AuthorizationException.java @@ -17,7 +17,9 @@ package org.apache.nifi.minifi.c2.api.security.authorization; -public class AuthorizationException extends Exception { +import org.apache.nifi.minifi.c2.api.ConfigurationProviderException; + +public class AuthorizationException extends ConfigurationProviderException { public AuthorizationException(String message) { super(message); } diff --git a/minifi-c2/minifi-c2-assembly/pom.xml b/minifi-c2/minifi-c2-assembly/pom.xml index 957f8bd04..dfd43c6b7 100644 --- a/minifi-c2/minifi-c2-assembly/pom.xml +++ b/minifi-c2/minifi-c2-assembly/pom.xml @@ -87,6 +87,11 @@ limitations under the License. minifi-c2-provider-cache ${project.version} + + org.apache.nifi.minifi + minifi-c2-provider-delegating + ${project.version} + org.apache.nifi.minifi minifi-c2-provider-nifi-rest diff --git a/minifi-c2/minifi-c2-assembly/src/main/resources/conf/authorizations.yaml b/minifi-c2/minifi-c2-assembly/src/main/resources/conf/authorizations.yaml index 7016ff4e8..566945194 100644 --- a/minifi-c2/minifi-c2-assembly/src/main/resources/conf/authorizations.yaml +++ b/minifi-c2/minifi-c2-assembly/src/main/resources/conf/authorizations.yaml @@ -27,4 +27,13 @@ Paths: # Default authorization lets anonymous pull any config. Remove below to change that. - Authorization: ROLE_ANONYMOUS - Action: allow \ No newline at end of file + Action: allow + + /c2/config/contentTypes: + Default Action: deny + Actions: + - Authorization: CLASS_RASPI_3 + Action: allow + # Default authorization lets anonymous pull any config. Remove below to change that. + - Authorization: ROLE_ANONYMOUS + Action: allow diff --git a/minifi-c2/minifi-c2-assembly/src/main/resources/conf/minifi-c2-context.xml b/minifi-c2/minifi-c2-assembly/src/main/resources/conf/minifi-c2-context.xml index 82a3dfcc3..bc23c9773 100644 --- a/minifi-c2/minifi-c2-assembly/src/main/resources/conf/minifi-c2-context.xml +++ b/minifi-c2/minifi-c2-assembly/src/main/resources/conf/minifi-c2-context.xml @@ -31,7 +31,9 @@ - text/yml + + text/yml + @@ -39,7 +41,7 @@ ./files - \${class}/config.yml + \${class}/config @@ -58,6 +60,9 @@ ${minifi.c2.server.provider.nifi.rest.api.url} + + \${class}.v\${version} + --> diff --git a/minifi-c2/minifi-c2-assembly/src/main/resources/files/raspi3/config.yml.v1 b/minifi-c2/minifi-c2-assembly/src/main/resources/files/raspi3/config.text.yml.v1 similarity index 100% rename from minifi-c2/minifi-c2-assembly/src/main/resources/files/raspi3/config.yml.v1 rename to minifi-c2/minifi-c2-assembly/src/main/resources/files/raspi3/config.text.yml.v1 diff --git a/minifi-c2/minifi-c2-cache/minifi-c2-cache-filesystem/src/main/java/org/apache/nifi/minifi/c2/cache/filesystem/FileSystemConfigurationCache.java b/minifi-c2/minifi-c2-cache/minifi-c2-cache-filesystem/src/main/java/org/apache/nifi/minifi/c2/cache/filesystem/FileSystemConfigurationCache.java index ced36a956..1e2600975 100644 --- a/minifi-c2/minifi-c2-cache/minifi-c2-cache-filesystem/src/main/java/org/apache/nifi/minifi/c2/cache/filesystem/FileSystemConfigurationCache.java +++ b/minifi-c2/minifi-c2-cache/minifi-c2-cache-filesystem/src/main/java/org/apache/nifi/minifi/c2/cache/filesystem/FileSystemConfigurationCache.java @@ -21,6 +21,8 @@ import org.apache.nifi.minifi.c2.api.cache.ConfigurationCache; import org.apache.nifi.minifi.c2.api.cache.ConfigurationCacheFileInfo; import org.apache.nifi.minifi.c2.api.util.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.nio.file.Files; @@ -29,8 +31,11 @@ import java.util.List; import java.util.Map; import java.util.regex.Pattern; +import java.util.stream.Collectors; public class FileSystemConfigurationCache implements ConfigurationCache { + private static final Logger logger = LoggerFactory.getLogger(FileSystemConfigurationCache.class); + private final Path pathRoot; private final String pathPattern; @@ -50,7 +55,7 @@ protected Path resolveChildAndVerifyParent(Path parent, String s) throws Invalid } @Override - public ConfigurationCacheFileInfo getCacheFileInfo(Map> parameters) throws InvalidParameterException { + public ConfigurationCacheFileInfo getCacheFileInfo(String contentType, Map> parameters) throws InvalidParameterException { String pathString = pathPattern; for (Map.Entry> entry : parameters.entrySet()) { if (entry.getValue().size() != 1) { @@ -58,6 +63,7 @@ public ConfigurationCacheFileInfo getCacheFileInfo(Map> par } pathString = pathString.replaceAll(Pattern.quote("${" + entry.getKey() + "}"), entry.getValue().get(0)); } + pathString = pathString + "." + contentType.replace('/', '.'); String[] split = pathString.split("/"); for (String s1 : split) { int openBrace = s1.indexOf("${"); @@ -75,6 +81,13 @@ public ConfigurationCacheFileInfo getCacheFileInfo(Map> par path = resolveChildAndVerifyParent(path, s); } Pair dirPathAndFilename = new Pair<>(path, splitPath[splitPath.length - 1]); + if (logger.isDebugEnabled()) { + StringBuilder message = new StringBuilder("Parameters {"); + message.append(parameters.entrySet().stream().map(e -> e.getKey() + ": [" + String.join(", ", e.getValue()) + "]").collect(Collectors.joining(", "))); + message.append("} -> "); + message.append(dirPathAndFilename.getFirst().resolve(dirPathAndFilename.getSecond()).toAbsolutePath()); + logger.debug(message.toString()); + } return new FileSystemCacheFileInfoImpl(this, dirPathAndFilename.getFirst(), dirPathAndFilename.getSecond() + ".v"); } } diff --git a/minifi-c2/minifi-c2-integration-tests/pom.xml b/minifi-c2/minifi-c2-integration-tests/pom.xml index 60af910bd..4662cba9f 100644 --- a/minifi-c2/minifi-c2-integration-tests/pom.xml +++ b/minifi-c2/minifi-c2-integration-tests/pom.xml @@ -96,6 +96,17 @@ limitations under the License. + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + diff --git a/minifi-c2/minifi-c2-integration-tests/src/test/java/org/apache/nifi/minifi/c2/integration/test/DelegatingConfigurationProviderSecureTest.java b/minifi-c2/minifi-c2-integration-tests/src/test/java/org/apache/nifi/minifi/c2/integration/test/DelegatingConfigurationProviderSecureTest.java new file mode 100644 index 000000000..68cf9f90f --- /dev/null +++ b/minifi-c2/minifi-c2-integration-tests/src/test/java/org/apache/nifi/minifi/c2/integration/test/DelegatingConfigurationProviderSecureTest.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.c2.integration.test; + +import com.palantir.docker.compose.DockerComposeRule; +import org.apache.nifi.minifi.c2.integration.test.health.HttpsStatusCodeHealthCheck; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSocketFactory; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; + +public class DelegatingConfigurationProviderSecureTest extends AbstractTestSecure { + public static final String C2_UPSTREAM_URL = "https://c2:10443/c2/config"; + private static SSLSocketFactory healthCheckSocketFactory; + private static Path certificatesDirectory; + private static SSLContext trustSslContext; + + // Not annotated as rule because we need to generate certificatesDirectory first + public static DockerComposeRule docker = DockerComposeRule.builder() + .file("target/test-classes/docker-compose-DelegatingProviderSecureTest.yml") + .waitingForServices(Arrays.asList("squid", "c2-upstream"), + new HttpsStatusCodeHealthCheck(container -> C2_UPSTREAM_URL, containers -> containers.get(0), containers -> containers.get(1), () -> healthCheckSocketFactory, 403)) + .waitingForServices(Arrays.asList("squid", "c2"), + new HttpsStatusCodeHealthCheck(container -> C2_URL, containers -> containers.get(0), containers -> containers.get(1), () -> healthCheckSocketFactory, 403)) + .build(); + + public DelegatingConfigurationProviderSecureTest() { + super(docker, certificatesDirectory, trustSslContext); + } + + /** + * Generates certificates with the tls-toolkit and then starts up the docker compose file + */ + @BeforeClass + public static void initCertificates() throws Exception { + certificatesDirectory = Paths.get(DelegatingConfigurationProviderSecureTest.class.getClassLoader() + .getResource("docker-compose-DelegatingProviderSecureTest.yml").getFile()).getParent().toAbsolutePath().resolve("certificates-DelegatingConfigurationProviderSecureTest"); + trustSslContext = initCertificates(certificatesDirectory, Arrays.asList("c2", "c2-upstream")); + healthCheckSocketFactory = trustSslContext.getSocketFactory(); + + docker.before(); + } + + @AfterClass + public static void cleanup() { + docker.after(); + } + + @Before + public void setup() { + super.setup(docker); + } + + @Test + public void testUpstreamPermissionDenied() throws Exception { + assertReturnCode("?class=raspi4", loadSslContext("user3"), 403); + } +} diff --git a/minifi-c2/minifi-c2-integration-tests/src/test/java/org/apache/nifi/minifi/c2/integration/test/DelegatingConfigurationProviderUnsecureTest.java b/minifi-c2/minifi-c2-integration-tests/src/test/java/org/apache/nifi/minifi/c2/integration/test/DelegatingConfigurationProviderUnsecureTest.java new file mode 100644 index 000000000..8394427dc --- /dev/null +++ b/minifi-c2/minifi-c2-integration-tests/src/test/java/org/apache/nifi/minifi/c2/integration/test/DelegatingConfigurationProviderUnsecureTest.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.c2.integration.test; + +import com.palantir.docker.compose.DockerComposeRule; +import org.apache.nifi.minifi.c2.integration.test.health.HttpStatusCodeHealthCheck; +import org.junit.Before; +import org.junit.ClassRule; + +public class DelegatingConfigurationProviderUnsecureTest extends AbstractTestUnsecure { + @ClassRule + public static DockerComposeRule docker = DockerComposeRule.builder() + .file("target/test-classes/docker-compose-DelegatingProviderUnsecureTest.yml") + .waitingForService("c2-upstream", new HttpStatusCodeHealthCheck(DelegatingConfigurationProviderUnsecureTest::getUnsecureConfigUrl, 400)) + .waitingForService("c2", new HttpStatusCodeHealthCheck(DelegatingConfigurationProviderUnsecureTest::getUnsecureConfigUrl, 400)) + .build(); + + @Before + public void setup() { + super.setup(docker); + } +} diff --git a/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2-secure-delegating/conf/minifi-c2-context.xml b/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2-secure-delegating/conf/minifi-c2-context.xml new file mode 100644 index 000000000..e3e4976db --- /dev/null +++ b/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2-secure-delegating/conf/minifi-c2-context.xml @@ -0,0 +1,56 @@ + + + + + + + + + + + + + ./cache + + + ${class}/${class} + + + + + https://c2-upstream:10443 + + + + + + + + + + + diff --git a/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2-secure-rest/conf/minifi-c2-context.xml b/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2-secure-rest/conf/minifi-c2-context.xml index d648d8899..42c98ea68 100644 --- a/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2-secure-rest/conf/minifi-c2-context.xml +++ b/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2-secure-rest/conf/minifi-c2-context.xml @@ -45,6 +45,9 @@ https://mocknifi:8443/nifi-api + + ${class}.v${version} + diff --git a/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2-unsecure-delegating/conf/minifi-c2-context.xml b/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2-unsecure-delegating/conf/minifi-c2-context.xml new file mode 100644 index 000000000..a24621977 --- /dev/null +++ b/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2-unsecure-delegating/conf/minifi-c2-context.xml @@ -0,0 +1,56 @@ + + + + + + + + + + + + + ./cache + + + ${class}/${class} + + + + + http://c2-upstream:10080 + + + + + + + + + + + diff --git a/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2-unsecure-rest/conf/minifi-c2-context.xml b/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2-unsecure-rest/conf/minifi-c2-context.xml index f2e4eee23..3834cc8a6 100644 --- a/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2-unsecure-rest/conf/minifi-c2-context.xml +++ b/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2-unsecure-rest/conf/minifi-c2-context.xml @@ -44,6 +44,9 @@ http://mocknifi:8080/nifi-api + + ${class}.v${version} + diff --git a/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2-upstream-secure/conf/authorities.yaml b/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2-upstream-secure/conf/authorities.yaml new file mode 100644 index 000000000..22be1347e --- /dev/null +++ b/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2-upstream-secure/conf/authorities.yaml @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the \"License\"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +CN=c2, OU=NIFI: + - C2 diff --git a/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2-upstream-secure/conf/authorizations.yaml b/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2-upstream-secure/conf/authorizations.yaml new file mode 100644 index 000000000..6d626f720 --- /dev/null +++ b/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2-upstream-secure/conf/authorizations.yaml @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the \"License\"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Default Action: deny +Paths: + /c2/config: + Default Action: deny + Actions: + - Authorization: C2 + Query Parameters: + class: raspi4 + Action: deny + - Authorization: C2 + Action: allow + + /c2/config/contentTypes: + Default Action: deny + Actions: + - Authorization: C2 + Action: allow \ No newline at end of file diff --git a/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2/files/raspi2/config.yml.v1 b/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2/files/raspi2/config.text.yml.v1 similarity index 100% rename from minifi-c2/minifi-c2-integration-tests/src/test/resources/c2/files/raspi2/config.yml.v1 rename to minifi-c2/minifi-c2-integration-tests/src/test/resources/c2/files/raspi2/config.text.yml.v1 diff --git a/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2/files/raspi3/config.yml.v1 b/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2/files/raspi3/config.text.yml.v1 similarity index 100% rename from minifi-c2/minifi-c2-integration-tests/src/test/resources/c2/files/raspi3/config.yml.v1 rename to minifi-c2/minifi-c2-integration-tests/src/test/resources/c2/files/raspi3/config.text.yml.v1 diff --git a/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2/files/raspi3/config.yml.v2 b/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2/files/raspi3/config.text.yml.v2 similarity index 100% rename from minifi-c2/minifi-c2-integration-tests/src/test/resources/c2/files/raspi3/config.yml.v2 rename to minifi-c2/minifi-c2-integration-tests/src/test/resources/c2/files/raspi3/config.text.yml.v2 diff --git a/minifi-c2/minifi-c2-integration-tests/src/test/resources/docker-compose-DelegatingProviderSecureTest.yml b/minifi-c2/minifi-c2-integration-tests/src/test/resources/docker-compose-DelegatingProviderSecureTest.yml new file mode 100644 index 000000000..2c193085f --- /dev/null +++ b/minifi-c2/minifi-c2-integration-tests/src/test/resources/docker-compose-DelegatingProviderSecureTest.yml @@ -0,0 +1,55 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the \"License\"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "2" + +services: + c2-upstream: + image: apacheminific2:${minifi.c2.version} + ports: + - "10443" + hostname: c2-upstream + volumes: + - ./c2/files:/opt/minifi-c2/minifi-c2-${minifi.c2.version}/files + + - ./c2-secure/conf/c2.properties:/opt/minifi-c2/minifi-c2-${minifi.c2.version}/conf/c2.properties + - ./c2-upstream-secure/conf/authorities.yaml:/opt/minifi-c2/minifi-c2-${minifi.c2.version}/conf/authorities.yaml + - ./c2-upstream-secure/conf/authorizations.yaml:/opt/minifi-c2/minifi-c2-${minifi.c2.version}/conf/authorizations.yaml + + - ./certificates-DelegatingConfigurationProviderSecureTest/c2-upstream/keystore.jks:/opt/minifi-c2/minifi-c2-${minifi.c2.version}/conf/keystore.jks + - ./certificates-DelegatingConfigurationProviderSecureTest/c2-upstream/truststore.jks:/opt/minifi-c2/minifi-c2-${minifi.c2.version}/conf/truststore.jks + + c2: + image: apacheminific2:${minifi.c2.version} + ports: + - "10443" + hostname: c2 + volumes: + - ./c2-secure-delegating/conf/minifi-c2-context.xml:/opt/minifi-c2/minifi-c2-${minifi.c2.version}/conf/minifi-c2-context.xml + + - ./c2-secure/conf/c2.properties:/opt/minifi-c2/minifi-c2-${minifi.c2.version}/conf/c2.properties + - ./c2-secure/conf/authorities.yaml:/opt/minifi-c2/minifi-c2-${minifi.c2.version}/conf/authorities.yaml + - ./c2-secure/conf/authorizations.yaml:/opt/minifi-c2/minifi-c2-${minifi.c2.version}/conf/authorizations.yaml + + - ./certificates-DelegatingConfigurationProviderSecureTest/c2/keystore.jks:/opt/minifi-c2/minifi-c2-${minifi.c2.version}/conf/keystore.jks + - ./certificates-DelegatingConfigurationProviderSecureTest/c2/truststore.jks:/opt/minifi-c2/minifi-c2-${minifi.c2.version}/conf/truststore.jks + + squid: + image: chrisdaish/squid + ports: + - "3128" + hostname: squid + volumes: + - ./squid/squid.conf:/etc/squid/squid.conf \ No newline at end of file diff --git a/minifi-c2/minifi-c2-integration-tests/src/test/resources/docker-compose-DelegatingProviderUnsecureTest.yml b/minifi-c2/minifi-c2-integration-tests/src/test/resources/docker-compose-DelegatingProviderUnsecureTest.yml new file mode 100644 index 000000000..a5d0da4f5 --- /dev/null +++ b/minifi-c2/minifi-c2-integration-tests/src/test/resources/docker-compose-DelegatingProviderUnsecureTest.yml @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the \"License\"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "2" + +services: + c2-upstream: + image: apacheminific2:${minifi.c2.version} + ports: + - "10080" + hostname: c2-upstream + volumes: + - ./c2/files:/opt/minifi-c2/minifi-c2-${minifi.c2.version}/files + c2: + image: apacheminific2:${minifi.c2.version} + ports: + - "10080" + hostname: c2 + volumes: + - ./c2-unsecure-delegating/conf/minifi-c2-context.xml:/opt/minifi-c2/minifi-c2-${minifi.c2.version}/conf/minifi-c2-context.xml diff --git a/minifi-c2/minifi-c2-jetty/src/main/java/org/apache/nifi/minifi/c2/jetty/JettyServer.java b/minifi-c2/minifi-c2-jetty/src/main/java/org/apache/nifi/minifi/c2/jetty/JettyServer.java index 5c8b1f8df..abace06eb 100644 --- a/minifi-c2/minifi-c2-jetty/src/main/java/org/apache/nifi/minifi/c2/jetty/JettyServer.java +++ b/minifi-c2/minifi-c2-jetty/src/main/java/org/apache/nifi/minifi/c2/jetty/JettyServer.java @@ -57,10 +57,8 @@ public static void main(String[] args) throws Exception { Server server; int port = Integer.parseInt(properties.getProperty("minifi.c2.server.port", "10080")); - SslContextFactory sslContextFactory = properties.getSslContextFactory(); - if (sslContextFactory == null) { - server = new Server(port); - } else { + if (properties.isSecure()) { + SslContextFactory sslContextFactory = properties.getSslContextFactory(); HttpConfiguration config = new HttpConfiguration(); config.setSecureScheme("https"); config.setSecurePort(port); @@ -72,6 +70,8 @@ public static void main(String[] args) throws Exception { serverConnector.setPort(port); server.addConnector(serverConnector); + } else { + server = new Server(port); } server.setHandler(handlers); diff --git a/minifi-c2/minifi-c2-provider/minifi-c2-provider-cache/src/main/java/org/apache/nifi/minifi/c2/provider/cache/CacheConfigurationProvider.java b/minifi-c2/minifi-c2-provider/minifi-c2-provider-cache/src/main/java/org/apache/nifi/minifi/c2/provider/cache/CacheConfigurationProvider.java index b0864ac9d..f65a86bc9 100644 --- a/minifi-c2/minifi-c2-provider/minifi-c2-provider-cache/src/main/java/org/apache/nifi/minifi/c2/provider/cache/CacheConfigurationProvider.java +++ b/minifi-c2/minifi-c2-provider/minifi-c2-provider-cache/src/main/java/org/apache/nifi/minifi/c2/provider/cache/CacheConfigurationProvider.java @@ -26,21 +26,21 @@ import java.util.Map; public class CacheConfigurationProvider implements ConfigurationProvider { - private final String contentType; + private final List contentTypes; private final ConfigurationCache configurationCache; - public CacheConfigurationProvider(String contentType, ConfigurationCache configurationCache) { - this.contentType = contentType; + public CacheConfigurationProvider(List contentTypes, ConfigurationCache configurationCache) { + this.contentTypes = contentTypes; this.configurationCache = configurationCache; } @Override - public String getContentType() { - return contentType; + public List getContentTypes() { + return contentTypes; } @Override - public Configuration getConfiguration(Integer version, Map> parameters) throws ConfigurationProviderException { - return configurationCache.getCacheFileInfo(parameters).getConfiguration(version); + public Configuration getConfiguration(String contentType, Integer version, Map> parameters) throws ConfigurationProviderException { + return configurationCache.getCacheFileInfo(contentType, parameters).getConfiguration(version); } } diff --git a/minifi-c2/minifi-c2-provider/minifi-c2-provider-cache/src/test/java/org/apache/nifi/minifi/c2/provider/cache/CacheConfigurationProviderTest.java b/minifi-c2/minifi-c2-provider/minifi-c2-provider-cache/src/test/java/org/apache/nifi/minifi/c2/provider/cache/CacheConfigurationProviderTest.java index d038194c2..bcc37e90f 100644 --- a/minifi-c2/minifi-c2-provider/minifi-c2-provider-cache/src/test/java/org/apache/nifi/minifi/c2/provider/cache/CacheConfigurationProviderTest.java +++ b/minifi-c2/minifi-c2-provider/minifi-c2-provider-cache/src/test/java/org/apache/nifi/minifi/c2/provider/cache/CacheConfigurationProviderTest.java @@ -21,10 +21,10 @@ import org.apache.nifi.minifi.c2.api.cache.ConfigurationCache; import org.apache.nifi.minifi.c2.api.cache.ConfigurationCacheFileInfo; import org.apache.nifi.minifi.c2.api.cache.WriteableConfiguration; -import org.apache.nifi.minifi.c2.provider.cache.CacheConfigurationProvider; import org.junit.Before; import org.junit.Test; +import java.util.Arrays; import java.util.List; import java.util.Map; @@ -34,6 +34,7 @@ public class CacheConfigurationProviderTest { public static final String TEST_CONTENT_TYPE = "test/contenttype"; + public static final String TEST_CONTENT_TYPE_2 = "test/contenttype2"; private CacheConfigurationProvider cacheConfigurationProvider; private ConfigurationCache configConfigurationCache; @@ -41,25 +42,30 @@ public class CacheConfigurationProviderTest { @Before public void setup() { configConfigurationCache = mock(ConfigurationCache.class); - cacheConfigurationProvider = new CacheConfigurationProvider(TEST_CONTENT_TYPE, configConfigurationCache); + cacheConfigurationProvider = new CacheConfigurationProvider(Arrays.asList(TEST_CONTENT_TYPE, TEST_CONTENT_TYPE_2), configConfigurationCache); } @Test public void testContentType() { - assertEquals(TEST_CONTENT_TYPE, cacheConfigurationProvider.getContentType()); + assertEquals(Arrays.asList(TEST_CONTENT_TYPE, TEST_CONTENT_TYPE_2), cacheConfigurationProvider.getContentTypes()); } @Test public void testGetConfiguration() throws ConfigurationProviderException { int version = 99; - Map> parameters = mock(Map.class); ConfigurationCacheFileInfo configurationCacheFileInfo = mock(ConfigurationCacheFileInfo.class); + ConfigurationCacheFileInfo configurationCacheFileInfo2 = mock(ConfigurationCacheFileInfo.class); WriteableConfiguration configuration = mock(WriteableConfiguration.class); + WriteableConfiguration configuration2 = mock(WriteableConfiguration.class); - when(configConfigurationCache.getCacheFileInfo(parameters)).thenReturn(configurationCacheFileInfo); + Map> parameters = mock(Map.class); + when(configConfigurationCache.getCacheFileInfo(TEST_CONTENT_TYPE, parameters)).thenReturn(configurationCacheFileInfo); + when(configConfigurationCache.getCacheFileInfo(TEST_CONTENT_TYPE_2, parameters)).thenReturn(configurationCacheFileInfo2); when(configurationCacheFileInfo.getConfiguration(version)).thenReturn(configuration); + when(configurationCacheFileInfo2.getConfiguration(version)).thenReturn(configuration2); - assertEquals(configuration, cacheConfigurationProvider.getConfiguration(version, parameters)); + assertEquals(configuration, cacheConfigurationProvider.getConfiguration(TEST_CONTENT_TYPE, version, parameters)); + assertEquals(configuration2, cacheConfigurationProvider.getConfiguration(TEST_CONTENT_TYPE_2, version, parameters)); } } diff --git a/minifi-c2/minifi-c2-provider/minifi-c2-provider-delegating/pom.xml b/minifi-c2/minifi-c2-provider/minifi-c2-provider-delegating/pom.xml new file mode 100644 index 000000000..4b7e7ab62 --- /dev/null +++ b/minifi-c2/minifi-c2-provider/minifi-c2-provider-delegating/pom.xml @@ -0,0 +1,53 @@ + + + + 4.0.0 + + minifi-c2-provider + org.apache.nifi.minifi + 0.2.0-SNAPSHOT + + minifi-c2-provider-delegating + jar + + + + org.apache.nifi.minifi + minifi-c2-api + ${project.version} + + + org.apache.nifi.minifi + minifi-c2-provider-util + ${project.version} + + + com.fasterxml.jackson.core + jackson-databind + + + commons-io + commons-io + + + org.mockito + mockito-all + test + + + diff --git a/minifi-c2/minifi-c2-provider/minifi-c2-provider-delegating/src/main/java/org/apache/nifi/minifi/c2/provider/delegating/DelegatingConfigurationProvider.java b/minifi-c2/minifi-c2-provider/minifi-c2-provider-delegating/src/main/java/org/apache/nifi/minifi/c2/provider/delegating/DelegatingConfigurationProvider.java new file mode 100644 index 000000000..6e0fec386 --- /dev/null +++ b/minifi-c2/minifi-c2-provider/minifi-c2-provider-delegating/src/main/java/org/apache/nifi/minifi/c2/provider/delegating/DelegatingConfigurationProvider.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.c2.provider.delegating; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.commons.io.IOUtils; +import org.apache.nifi.minifi.c2.api.Configuration; +import org.apache.nifi.minifi.c2.api.ConfigurationProvider; +import org.apache.nifi.minifi.c2.api.ConfigurationProviderException; +import org.apache.nifi.minifi.c2.api.InvalidParameterException; +import org.apache.nifi.minifi.c2.api.cache.ConfigurationCache; +import org.apache.nifi.minifi.c2.api.cache.ConfigurationCacheFileInfo; +import org.apache.nifi.minifi.c2.api.cache.WriteableConfiguration; +import org.apache.nifi.minifi.c2.api.security.authorization.AuthorizationException; +import org.apache.nifi.minifi.c2.provider.util.HttpConnector; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.UnsupportedEncodingException; +import java.net.HttpURLConnection; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.security.GeneralSecurityException; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class DelegatingConfigurationProvider implements ConfigurationProvider { + public static final Pattern errorPattern = Pattern.compile("^Server returned HTTP response code: ([0-9]+) for URL:.*"); + private final ConfigurationCache configurationCache; + private final HttpConnector httpConnector; + private final ObjectMapper objectMapper; + + public DelegatingConfigurationProvider(ConfigurationCache configurationCache, String delegateUrl) throws InvalidParameterException, GeneralSecurityException, IOException { + this(configurationCache, new HttpConnector(delegateUrl)); + } + + public DelegatingConfigurationProvider(ConfigurationCache configurationCache, HttpConnector httpConnector) { + this.configurationCache = configurationCache; + this.httpConnector = httpConnector; + this.objectMapper = new ObjectMapper(); + } + + @Override + public List getContentTypes() throws ConfigurationProviderException { + try { + HttpURLConnection httpURLConnection = httpConnector.get("/c2/config/contentTypes"); + try { + return objectMapper.readValue(httpURLConnection.getInputStream(), List.class); + } finally { + httpURLConnection.disconnect(); + } + } catch (IOException e) { + throw new ConfigurationProviderException("Unable to get content types from delegate.", e); + } + } + + @Override + public Configuration getConfiguration(String contentType, Integer version, Map> parameters) throws ConfigurationProviderException { + HttpURLConnection remoteC2ServerConnection = null; + try { + if (version == null) { + remoteC2ServerConnection = getDelegateConnection(contentType, parameters); + version = Integer.parseInt(remoteC2ServerConnection.getHeaderField("X-Content-Version")); + } + ConfigurationCacheFileInfo cacheFileInfo = configurationCache.getCacheFileInfo(contentType, parameters); + WriteableConfiguration configuration = cacheFileInfo.getConfiguration(version); + if (!configuration.exists()) { + if (remoteC2ServerConnection == null) { + remoteC2ServerConnection = getDelegateConnection(contentType, parameters); + } + try (InputStream inputStream = remoteC2ServerConnection.getInputStream(); + OutputStream outputStream = configuration.getOutputStream()) { + IOUtils.copy(inputStream, outputStream); + } catch (IOException e) { + throw new ConfigurationProviderException("Unable to copy remote configuration to cache.", e); + } + } + return configuration; + } finally { + if (remoteC2ServerConnection != null) { + remoteC2ServerConnection.disconnect(); + } + } + } + + protected HttpURLConnection getDelegateConnection(String contentType, Map> parameters) throws ConfigurationProviderException { + StringBuilder queryStringBuilder = new StringBuilder(); + try { + parameters.entrySet().stream().sorted(Comparator.comparing(Map.Entry::getKey)).forEachOrdered(e -> e.getValue().stream().sorted().forEachOrdered(v -> { + try { + queryStringBuilder.append(URLEncoder.encode(e.getKey(), "UTF-8")).append("=").append(URLEncoder.encode(v, "UTF-8")); + } catch (UnsupportedEncodingException ex) { + throw new ConfigurationProviderException("Unsupported encoding.", ex).wrap(); + } + queryStringBuilder.append("&"); + })); + } catch (ConfigurationProviderException.Wrapper e) { + throw e.unwrap(); + } + String url = "/c2/config"; + if (queryStringBuilder.length() > 0) { + queryStringBuilder.setLength(queryStringBuilder.length() - 1); + url = url + "?" + queryStringBuilder.toString(); + } + HttpURLConnection httpURLConnection = httpConnector.get(url); + httpURLConnection.setRequestProperty("Accepts", contentType); + try { + int responseCode; + try { + responseCode = httpURLConnection.getResponseCode(); + } catch (IOException e) { + Matcher matcher = errorPattern.matcher(e.getMessage()); + if (matcher.matches()) { + responseCode = Integer.parseInt(matcher.group(1)); + } else { + throw e; + } + } + if (responseCode >= 400) { + String message = ""; + InputStream inputStream = httpURLConnection.getErrorStream(); + if (inputStream != null) { + try { + message = IOUtils.toString(inputStream, StandardCharsets.UTF_8); + } finally { + inputStream.close(); + } + } + if (responseCode == 400) { + throw new InvalidParameterException(message); + } else if (responseCode == 403) { + throw new AuthorizationException("Got authorization exception from upstream server " + message); + } else { + throw new ConfigurationProviderException(message); + } + } + } catch (IOException e) { + throw new ConfigurationProviderException("Unable to get response code from upstream server.", e); + } + return httpURLConnection; + } +} diff --git a/minifi-c2/minifi-c2-provider/minifi-c2-provider-delegating/src/test/java/org/apache/nifi/minifi/c2/provider/delegating/DelegatingConfigurationProviderTest.java b/minifi-c2/minifi-c2-provider/minifi-c2-provider-delegating/src/test/java/org/apache/nifi/minifi/c2/provider/delegating/DelegatingConfigurationProviderTest.java new file mode 100644 index 000000000..3127ca74b --- /dev/null +++ b/minifi-c2/minifi-c2-provider/minifi-c2-provider-delegating/src/test/java/org/apache/nifi/minifi/c2/provider/delegating/DelegatingConfigurationProviderTest.java @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.c2.provider.delegating; + +import org.apache.nifi.minifi.c2.api.ConfigurationProviderException; +import org.apache.nifi.minifi.c2.api.InvalidParameterException; +import org.apache.nifi.minifi.c2.api.cache.ConfigurationCache; +import org.apache.nifi.minifi.c2.api.cache.ConfigurationCacheFileInfo; +import org.apache.nifi.minifi.c2.api.cache.WriteableConfiguration; +import org.apache.nifi.minifi.c2.api.security.authorization.AuthorizationException; +import org.apache.nifi.minifi.c2.provider.util.HttpConnector; +import org.junit.Before; +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class DelegatingConfigurationProviderTest { + private ConfigurationCache configurationCache; + private HttpConnector httpConnector; + private HttpURLConnection httpURLConnection; + private DelegatingConfigurationProvider delegatingConfigurationProvider; + private Map> parameters; + private Integer version; + private String endpointPath; + private String contentType; + + @Before + public void setup() throws ConfigurationProviderException { + contentType = "text/yml"; + version = 2; + parameters = new HashMap<>(); + parameters.put("net", Collections.singletonList("edge")); + parameters.put("class", Collections.singletonList("raspi3")); + parameters.put("version", Collections.singletonList(Integer.toString(version))); + endpointPath = "/c2/config?class=raspi3&net=edge&version=2"; + initMocks(); + } + + public void initMocks() throws ConfigurationProviderException { + configurationCache = mock(ConfigurationCache.class); + httpConnector = mock(HttpConnector.class); + httpURLConnection = mock(HttpURLConnection.class); + delegatingConfigurationProvider = new DelegatingConfigurationProvider(configurationCache, httpConnector); + when(httpConnector.get(endpointPath)).thenReturn(httpURLConnection); + } + + @Test + public void testGetDelegateConnectionNoParameters() throws ConfigurationProviderException { + endpointPath = "/c2/config"; + initMocks(); + assertEquals(httpURLConnection, delegatingConfigurationProvider.getDelegateConnection(contentType, Collections.emptyMap())); + verify(httpURLConnection).setRequestProperty("Accepts", contentType); + } + + @Test + public void testGetDelegateConnectionParameters() throws ConfigurationProviderException { + assertEquals(httpURLConnection, delegatingConfigurationProvider.getDelegateConnection(contentType, parameters)); + verify(httpURLConnection).setRequestProperty("Accepts", contentType); + } + + @Test(expected = AuthorizationException.class) + public void testGetDelegateConnection403() throws ConfigurationProviderException, IOException { + when(httpURLConnection.getResponseCode()).thenReturn(403); + delegatingConfigurationProvider.getDelegateConnection(contentType, parameters); + } + + @Test(expected = InvalidParameterException.class) + public void testGetDelegateConnection400() throws IOException, ConfigurationProviderException { + when(httpURLConnection.getResponseCode()).thenThrow(new IOException("Server returned HTTP response code: 400 for URL: " + endpointPath)); + delegatingConfigurationProvider.getDelegateConnection(contentType, parameters); + } + + @Test(expected = ConfigurationProviderException.class) + public void testGetDelegateConnection401() throws IOException, ConfigurationProviderException { + when(httpURLConnection.getResponseCode()).thenReturn(401); + delegatingConfigurationProvider.getDelegateConnection(contentType, parameters); + } + + @Test(expected = ConfigurationProviderException.class) + public void testGetContentTypesMalformed() throws ConfigurationProviderException, IOException { + endpointPath = "/c2/config/contentTypes"; + initMocks(); + when(httpURLConnection.getInputStream()).thenReturn(new ByteArrayInputStream("[malformed".getBytes(StandardCharsets.UTF_8))); + delegatingConfigurationProvider.getContentTypes(); + } + + @Test(expected = ConfigurationProviderException.class) + public void testGetContentTypesIOE() throws ConfigurationProviderException, IOException { + endpointPath = "/c2/config/contentTypes"; + initMocks(); + when(httpURLConnection.getInputStream()).thenThrow(new IOException()); + delegatingConfigurationProvider.getContentTypes(); + } + + @Test + public void testGetContentTypes() throws ConfigurationProviderException, IOException { + endpointPath = "/c2/config/contentTypes"; + initMocks(); + List contentTypes = Arrays.asList(contentType, "application/json"); + when(httpURLConnection.getInputStream()).thenReturn(new ByteArrayInputStream(("[\"" + String.join("\", \"", contentTypes) + "\"]").getBytes(StandardCharsets.UTF_8))); + assertEquals(contentTypes, delegatingConfigurationProvider.getContentTypes()); + } + + @Test + public void testGetConfigurationExistsWithVersion() throws ConfigurationProviderException { + ConfigurationCacheFileInfo configurationCacheFileInfo = mock(ConfigurationCacheFileInfo.class); + WriteableConfiguration configuration = mock(WriteableConfiguration.class); + when(configurationCache.getCacheFileInfo(contentType, parameters)).thenReturn(configurationCacheFileInfo); + when(configurationCacheFileInfo.getConfiguration(version)).thenReturn(configuration); + when(configuration.exists()).thenReturn(true); + + assertEquals(configuration, delegatingConfigurationProvider.getConfiguration(contentType, version, parameters)); + } + + @Test + public void testGetConfigurationDoesntExistWithVersion() throws ConfigurationProviderException, IOException { + ConfigurationCacheFileInfo configurationCacheFileInfo = mock(ConfigurationCacheFileInfo.class); + WriteableConfiguration configuration = mock(WriteableConfiguration.class); + byte[] payload = "payload".getBytes(StandardCharsets.UTF_8); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + when(httpURLConnection.getInputStream()).thenReturn(new ByteArrayInputStream(payload)); + when(configuration.getOutputStream()).thenReturn(output); + when(configurationCache.getCacheFileInfo(contentType, parameters)).thenReturn(configurationCacheFileInfo); + when(configurationCacheFileInfo.getConfiguration(version)).thenReturn(configuration); + when(configuration.exists()).thenReturn(false); + + assertEquals(configuration, delegatingConfigurationProvider.getConfiguration(contentType, version, parameters)); + assertArrayEquals(payload, output.toByteArray()); + } + + @Test + public void testGetConfigurationExistsWithNoVersion() throws ConfigurationProviderException { + parameters.remove("version"); + endpointPath = "/c2/config?class=raspi3&net=edge"; + initMocks(); + ConfigurationCacheFileInfo configurationCacheFileInfo = mock(ConfigurationCacheFileInfo.class); + WriteableConfiguration configuration = mock(WriteableConfiguration.class); + when(httpURLConnection.getHeaderField("X-Content-Version")).thenReturn("2"); + when(configurationCache.getCacheFileInfo(contentType, parameters)).thenReturn(configurationCacheFileInfo); + when(configurationCacheFileInfo.getConfiguration(version)).thenReturn(configuration); + when(configuration.exists()).thenReturn(true); + + assertEquals(configuration, delegatingConfigurationProvider.getConfiguration(contentType, null, parameters)); + } + + @Test + public void testGetConfigurationDoesntExistWithNoVersion() throws ConfigurationProviderException, IOException { + parameters.remove("version"); + endpointPath = "/c2/config?class=raspi3&net=edge"; + initMocks(); + ConfigurationCacheFileInfo configurationCacheFileInfo = mock(ConfigurationCacheFileInfo.class); + WriteableConfiguration configuration = mock(WriteableConfiguration.class); + byte[] payload = "payload".getBytes(StandardCharsets.UTF_8); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + when(httpURLConnection.getInputStream()).thenReturn(new ByteArrayInputStream(payload)); + when(configuration.getOutputStream()).thenReturn(output); + when(httpURLConnection.getHeaderField("X-Content-Version")).thenReturn("2"); + when(configurationCache.getCacheFileInfo(contentType, parameters)).thenReturn(configurationCacheFileInfo); + when(configurationCacheFileInfo.getConfiguration(version)).thenReturn(configuration); + when(configuration.exists()).thenReturn(false); + + assertEquals(configuration, delegatingConfigurationProvider.getConfiguration(contentType, null, parameters)); + assertArrayEquals(payload, output.toByteArray()); + } +} diff --git a/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/pom.xml b/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/pom.xml index 859524692..43c5225e7 100644 --- a/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/pom.xml +++ b/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/pom.xml @@ -31,6 +31,11 @@ limitations under the License. minifi-c2-api ${project.version} + + org.apache.nifi.minifi + minifi-c2-provider-util + ${project.version} + com.fasterxml.jackson.core jackson-core diff --git a/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/main/java/org/apache/nifi/minifi/c2/provider/nifi/rest/NiFiRestConfigurationProvider.java b/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/main/java/org/apache/nifi/minifi/c2/provider/nifi/rest/NiFiRestConfigurationProvider.java index 77175d79a..8f0bf2c55 100644 --- a/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/main/java/org/apache/nifi/minifi/c2/provider/nifi/rest/NiFiRestConfigurationProvider.java +++ b/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/main/java/org/apache/nifi/minifi/c2/provider/nifi/rest/NiFiRestConfigurationProvider.java @@ -23,9 +23,9 @@ import org.apache.nifi.minifi.c2.api.ConfigurationProviderException; import org.apache.nifi.minifi.c2.api.InvalidParameterException; import org.apache.nifi.minifi.c2.api.cache.ConfigurationCache; -import org.apache.nifi.minifi.c2.api.cache.ConfigurationCacheFileInfo; import org.apache.nifi.minifi.c2.api.cache.WriteableConfiguration; import org.apache.nifi.minifi.c2.api.util.Pair; +import org.apache.nifi.minifi.c2.provider.util.HttpConnector; import org.apache.nifi.minifi.commons.schema.ConfigSchema; import org.apache.nifi.minifi.commons.schema.serialization.SchemaSaver; import org.apache.nifi.minifi.toolkit.configuration.ConfigMain; @@ -38,12 +38,17 @@ import java.io.InputStream; import java.net.HttpURLConnection; import java.security.GeneralSecurityException; +import java.util.Arrays; +import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Spliterator; import java.util.Spliterators; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.stream.StreamSupport; @@ -52,32 +57,58 @@ public class NiFiRestConfigurationProvider implements ConfigurationProvider { private static final Logger logger = LoggerFactory.getLogger(NiFiRestConfigurationProvider.class); private final JsonFactory jsonFactory = new JsonFactory(); private final ConfigurationCache configurationCache; - private final NiFiRestConnector niFiRestConnector; + private final HttpConnector httpConnector; + private final String templateNamePattern; - public NiFiRestConfigurationProvider(ConfigurationCache configurationCache, String nifiUrl) throws InvalidParameterException, GeneralSecurityException, IOException { - this(configurationCache, new NiFiRestConnector(nifiUrl)); + public NiFiRestConfigurationProvider(ConfigurationCache configurationCache, String nifiUrl, String templateNamePattern) throws InvalidParameterException, GeneralSecurityException, IOException { + this(configurationCache, new HttpConnector(nifiUrl), templateNamePattern); } - public NiFiRestConfigurationProvider(ConfigurationCache configurationCache, NiFiRestConnector niFiRestConnector) { + public NiFiRestConfigurationProvider(ConfigurationCache configurationCache, HttpConnector httpConnector, String templateNamePattern) { this.configurationCache = configurationCache; - this.niFiRestConnector = niFiRestConnector; + this.httpConnector = httpConnector; + this.templateNamePattern = templateNamePattern; } @Override - public String getContentType() { - return CONTENT_TYPE; + public List getContentTypes() { + return Collections.singletonList(CONTENT_TYPE); } @Override - public Configuration getConfiguration(Integer version, Map> parameters) throws ConfigurationProviderException { - ConfigurationCacheFileInfo configurationCacheFileInfo = configurationCache.getCacheFileInfo(parameters); + public Configuration getConfiguration(String contentType, Integer version, Map> parameters) throws ConfigurationProviderException { + if (!CONTENT_TYPE.equals(contentType)) { + throw new ConfigurationProviderException("Unsupported content type: " + contentType + " supported value is " + CONTENT_TYPE); + } + String filename = templateNamePattern; + for (Map.Entry> entry : parameters.entrySet()) { + if (entry.getValue().size() != 1) { + throw new InvalidParameterException("Multiple values for same parameter not supported in this provider."); + } + filename = filename.replaceAll(Pattern.quote("${" + entry.getKey() + "}"), entry.getValue().get(0)); + } + int index = filename.indexOf("${"); + while (index != -1) { + int endIndex = filename.indexOf("}", index); + if (endIndex == -1) { + break; + } + String variable = filename.substring(index + 2, endIndex); + if (!"version".equals(variable)) { + throw new InvalidParameterException("Found unsubstituted parameter " + variable); + } + index = endIndex + 1; + } + String id = null; if (version == null) { - Pair maxIdAndVersion = getMaxIdAndVersion(configurationCacheFileInfo); + String filenamePattern = Arrays.stream(filename.split(Pattern.quote("${version}"), -1)).map(Pattern::quote).collect(Collectors.joining("([0-9+])")); + Pair maxIdAndVersion = getMaxIdAndVersion(filenamePattern); id = maxIdAndVersion.getFirst(); version = maxIdAndVersion.getSecond(); } - WriteableConfiguration configuration = configurationCacheFileInfo.getConfiguration(version); + filename = filename.replaceAll(Pattern.quote("${version}"), Integer.toString(version)); + WriteableConfiguration configuration = configurationCache.getCacheFileInfo(contentType, parameters).getConfiguration(version); if (configuration.exists()) { if (logger.isDebugEnabled()) { logger.debug("Configuration " + configuration + " exists and can be served from configurationCache."); @@ -88,11 +119,18 @@ public Configuration getConfiguration(Integer version, Map> } if (id == null) { try { - String filename = configuration.getName(); + String tmpFilename = templateNamePattern; + for (Map.Entry> entry : parameters.entrySet()) { + if (entry.getValue().size() != 1) { + throw new InvalidParameterException("Multiple values for same parameter not supported in this provider."); + } + tmpFilename = tmpFilename.replaceAll(Pattern.quote("${" + entry.getKey() + "}"), entry.getValue().get(0)); + } Pair>, Closeable> streamCloseablePair = getIdAndFilenameStream(); try { - id = streamCloseablePair.getFirst().filter(p -> filename.equals(p.getSecond())).map(Pair::getFirst).findFirst() - .orElseThrow(() -> new InvalidParameterException("Unable to find template named " + filename)); + String finalFilename = filename; + id = streamCloseablePair.getFirst().filter(p -> finalFilename.equals(p.getSecond())).map(Pair::getFirst).findFirst() + .orElseThrow(() -> new InvalidParameterException("Unable to find template named " + finalFilename)); } finally { streamCloseablePair.getSecond().close(); } @@ -101,7 +139,7 @@ public Configuration getConfiguration(Integer version, Map> } } - HttpURLConnection urlConnection = niFiRestConnector.get("/templates/" + id + "/download"); + HttpURLConnection urlConnection = httpConnector.get("/templates/" + id + "/download"); try (InputStream inputStream = urlConnection.getInputStream()){ ConfigSchema configSchema = ConfigMain.transformTemplateToSchema(inputStream); @@ -118,27 +156,28 @@ public Configuration getConfiguration(Integer version, Map> } private Pair>, Closeable> getIdAndFilenameStream() throws ConfigurationProviderException, IOException { - TemplatesIterator templatesIterator = new TemplatesIterator(niFiRestConnector, jsonFactory); + TemplatesIterator templatesIterator = new TemplatesIterator(httpConnector, jsonFactory); return new Pair<>(StreamSupport.stream(Spliterators.spliteratorUnknownSize(templatesIterator, Spliterator.ORDERED), false), templatesIterator); } - private Pair>, Closeable> getIdAndVersionStream(ConfigurationCacheFileInfo configurationCacheFileInfo) throws ConfigurationProviderException, IOException { + private Pair>, Closeable> getIdAndVersionStream(String filenamePattern) throws ConfigurationProviderException, IOException { + Pattern filename = Pattern.compile(filenamePattern); Pair>, Closeable> streamCloseablePair = getIdAndFilenameStream(); return new Pair<>(streamCloseablePair.getFirst().map(p -> { - Integer version = configurationCacheFileInfo.getVersionIfMatch(p.getSecond()); - if (version == null) { + Matcher matcher = filename.matcher(p.getSecond()); + if (!matcher.matches()) { return null; } - return new Pair<>(p.getFirst(), version); + return new Pair<>(p.getFirst(), Integer.parseInt(matcher.group(1))); }).filter(Objects::nonNull), streamCloseablePair.getSecond()); } - private Pair getMaxIdAndVersion(ConfigurationCacheFileInfo configurationCacheFileInfo) throws ConfigurationProviderException { + private Pair getMaxIdAndVersion(String filenamePattern) throws ConfigurationProviderException { try { - Pair>, Closeable> streamCloseablePair = getIdAndVersionStream(configurationCacheFileInfo); + Pair>, Closeable> streamCloseablePair = getIdAndVersionStream(filenamePattern); try { return streamCloseablePair.getFirst().sorted(Comparator.comparing(p -> ((Pair) p).getSecond()).reversed()).findFirst() - .orElseThrow(() -> new ConfigurationProviderException("Didn't find any templates that matched " + configurationCacheFileInfo + ".v[0-9]+")); + .orElseThrow(() -> new ConfigurationProviderException("Didn't find any templates that matched " + filenamePattern)); } finally { streamCloseablePair.getSecond().close(); } diff --git a/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/main/java/org/apache/nifi/minifi/c2/provider/nifi/rest/NiFiRestConnector.java b/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/main/java/org/apache/nifi/minifi/c2/provider/nifi/rest/NiFiRestConnector.java deleted file mode 100644 index 9a0befc1d..000000000 --- a/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/main/java/org/apache/nifi/minifi/c2/provider/nifi/rest/NiFiRestConnector.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.nifi.minifi.c2.provider.nifi.rest; - -import org.apache.nifi.minifi.c2.api.ConfigurationProviderException; -import org.apache.nifi.minifi.c2.api.InvalidParameterException; -import org.apache.nifi.minifi.c2.api.properties.C2Properties; -import org.eclipse.jetty.util.ssl.SslContextFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.net.ssl.HttpsURLConnection; -import javax.net.ssl.SSLContext; -import javax.net.ssl.SSLSocketFactory; -import java.io.IOException; -import java.net.HttpURLConnection; -import java.net.MalformedURLException; -import java.net.URL; -import java.security.GeneralSecurityException; - -public class NiFiRestConnector { - private static final Logger logger = LoggerFactory.getLogger(NiFiRestConnector.class); - - private final String nifiApiUrl; - private final SslContextFactory sslContextFactory; - - public NiFiRestConnector(String nifiApiUrl) throws InvalidParameterException, GeneralSecurityException, IOException { - if (nifiApiUrl.startsWith("https:")) { - sslContextFactory = C2Properties.getInstance().getSslContextFactory(); - if (sslContextFactory == null) { - throw new InvalidParameterException("Need sslContextFactory to connect to https NiFi endpoint (" + nifiApiUrl + ")"); - } - } else { - sslContextFactory = null; - } - this.nifiApiUrl = nifiApiUrl; - } - - protected HttpURLConnection get(String endpointPath) throws ConfigurationProviderException { - String endpointUrl = nifiApiUrl + endpointPath; - if (logger.isDebugEnabled()) { - logger.debug("Connecting to NiFi endpoint: " + endpointUrl); - } - URL url; - try { - url = new URL(endpointUrl); - } catch (MalformedURLException e) { - throw new ConfigurationProviderException("Malformed url " + endpointUrl, e); - } - - try { - if (sslContextFactory == null) { - return (HttpURLConnection) url.openConnection(); - } else { - HttpsURLConnection httpsURLConnection = (HttpsURLConnection) url.openConnection(); - SSLContext sslContext = sslContextFactory.getSslContext(); - SSLSocketFactory socketFactory = sslContext.getSocketFactory(); - httpsURLConnection.setSSLSocketFactory(socketFactory); - return httpsURLConnection; - } - } catch (IOException e) { - throw new ConfigurationProviderException("Unable to connect to " + url, e); - } - } -} diff --git a/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/main/java/org/apache/nifi/minifi/c2/provider/nifi/rest/TemplatesIterator.java b/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/main/java/org/apache/nifi/minifi/c2/provider/nifi/rest/TemplatesIterator.java index afe6c6eef..947a8538b 100644 --- a/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/main/java/org/apache/nifi/minifi/c2/provider/nifi/rest/TemplatesIterator.java +++ b/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/main/java/org/apache/nifi/minifi/c2/provider/nifi/rest/TemplatesIterator.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.core.JsonToken; import org.apache.nifi.minifi.c2.api.ConfigurationProviderException; import org.apache.nifi.minifi.c2.api.util.Pair; +import org.apache.nifi.minifi.c2.provider.util.HttpConnector; import java.io.Closeable; import java.io.IOException; @@ -38,8 +39,8 @@ public class TemplatesIterator implements Iterator>, Closea private final JsonParser parser; private Pair next; - public TemplatesIterator(NiFiRestConnector niFiRestConnector, JsonFactory jsonFactory) throws ConfigurationProviderException, IOException { - urlConnection = niFiRestConnector.get(FLOW_TEMPLATES); + public TemplatesIterator(HttpConnector httpConnector, JsonFactory jsonFactory) throws ConfigurationProviderException, IOException { + urlConnection = httpConnector.get(FLOW_TEMPLATES); inputStream = urlConnection.getInputStream(); parser = jsonFactory.createParser(inputStream); while (parser.nextToken() != JsonToken.END_OBJECT) { diff --git a/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/test/java/org/apache/nifi/minifi/c2/provider/nifi/rest/NiFiRestConfigurationProviderTest.java b/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/test/java/org/apache/nifi/minifi/c2/provider/nifi/rest/NiFiRestConfigurationProviderTest.java index 2cb3a8d0d..efc7f6ca3 100644 --- a/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/test/java/org/apache/nifi/minifi/c2/provider/nifi/rest/NiFiRestConfigurationProviderTest.java +++ b/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/test/java/org/apache/nifi/minifi/c2/provider/nifi/rest/NiFiRestConfigurationProviderTest.java @@ -18,6 +18,7 @@ package org.apache.nifi.minifi.c2.provider.nifi.rest; import org.apache.nifi.minifi.c2.api.cache.ConfigurationCache; +import org.apache.nifi.minifi.c2.provider.util.HttpConnector; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -25,6 +26,7 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Collections; import java.util.Comparator; import static org.junit.Assert.assertEquals; @@ -32,15 +34,15 @@ public class NiFiRestConfigurationProviderTest { private ConfigurationCache configConfigurationCache; - private NiFiRestConnector niFiRestConnector; + private HttpConnector httpConnector; private NiFiRestConfigurationProvider niFiRestConfigurationProvider; private Path cachePath; @Before public void setup() throws IOException { configConfigurationCache = mock(ConfigurationCache.class); - niFiRestConnector = mock(NiFiRestConnector.class); - niFiRestConfigurationProvider = new NiFiRestConfigurationProvider(configConfigurationCache, niFiRestConnector); + httpConnector = mock(HttpConnector.class); + niFiRestConfigurationProvider = new NiFiRestConfigurationProvider(configConfigurationCache, httpConnector, "${class}.v${version}"); cachePath = Files.createTempDirectory(NiFiRestConfigurationProviderTest.class.getCanonicalName()); } @@ -59,8 +61,6 @@ public void teardown() throws IOException { @Test public void testContentType() { - assertEquals(NiFiRestConfigurationProvider.CONTENT_TYPE, niFiRestConfigurationProvider.getContentType()); + assertEquals(Collections.singletonList(NiFiRestConfigurationProvider.CONTENT_TYPE), niFiRestConfigurationProvider.getContentTypes()); } - - } diff --git a/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/test/java/org/apache/nifi/minifi/c2/provider/nifi/rest/TemplatesIteratorTest.java b/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/test/java/org/apache/nifi/minifi/c2/provider/nifi/rest/TemplatesIteratorTest.java index ca20e4ac2..c8e19b2ae 100644 --- a/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/test/java/org/apache/nifi/minifi/c2/provider/nifi/rest/TemplatesIteratorTest.java +++ b/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/test/java/org/apache/nifi/minifi/c2/provider/nifi/rest/TemplatesIteratorTest.java @@ -21,6 +21,7 @@ import com.google.common.collect.Lists; import org.apache.nifi.minifi.c2.api.ConfigurationProviderException; import org.apache.nifi.minifi.c2.api.util.Pair; +import org.apache.nifi.minifi.c2.provider.util.HttpConnector; import org.junit.Before; import org.junit.Test; @@ -38,21 +39,21 @@ public class TemplatesIteratorTest { private JsonFactory jsonFactory; private HttpURLConnection httpURLConnection; - private NiFiRestConnector niFiRestConnector; + private HttpConnector httpConnector; @Before public void setup() throws ConfigurationProviderException { jsonFactory = new JsonFactory(); httpURLConnection = mock(HttpURLConnection.class); - niFiRestConnector = mock(NiFiRestConnector.class); - when(niFiRestConnector.get(TemplatesIterator.FLOW_TEMPLATES)).thenReturn(httpURLConnection); + httpConnector = mock(HttpConnector.class); + when(httpConnector.get(TemplatesIterator.FLOW_TEMPLATES)).thenReturn(httpURLConnection); } @Test(expected = NoSuchElementException.class) public void testIteratorNoSuchElementException() throws ConfigurationProviderException, IOException { when(httpURLConnection.getInputStream()).thenReturn(TemplatesIteratorTest.class.getClassLoader().getResourceAsStream("noTemplates.json")); - try (TemplatesIterator templatesIterator = new TemplatesIterator(niFiRestConnector, jsonFactory)) { + try (TemplatesIterator templatesIterator = new TemplatesIterator(httpConnector, jsonFactory)) { assertFalse(templatesIterator.hasNext()); templatesIterator.next(); } finally { @@ -64,7 +65,7 @@ public void testIteratorNoSuchElementException() throws ConfigurationProviderExc public void testIteratorNoTemplates() throws ConfigurationProviderException, IOException { when(httpURLConnection.getInputStream()).thenReturn(TemplatesIteratorTest.class.getClassLoader().getResourceAsStream("noTemplates.json")); List> idToNameList; - try (TemplatesIterator templatesIterator = new TemplatesIterator(niFiRestConnector, jsonFactory)) { + try (TemplatesIterator templatesIterator = new TemplatesIterator(httpConnector, jsonFactory)) { idToNameList = Lists.newArrayList(templatesIterator); } assertEquals(0, idToNameList.size()); @@ -76,7 +77,7 @@ public void testIteratorNoTemplates() throws ConfigurationProviderException, IOE public void testIteratorSingleTemplate() throws ConfigurationProviderException, IOException { when(httpURLConnection.getInputStream()).thenReturn(TemplatesIteratorTest.class.getClassLoader().getResourceAsStream("oneTemplate.json")); List> idToNameList; - try (TemplatesIterator templatesIterator = new TemplatesIterator(niFiRestConnector, jsonFactory)) { + try (TemplatesIterator templatesIterator = new TemplatesIterator(httpConnector, jsonFactory)) { idToNameList = Lists.newArrayList(templatesIterator); } assertEquals(1, idToNameList.size()); @@ -91,7 +92,7 @@ public void testIteratorSingleTemplate() throws ConfigurationProviderException, public void testIteratorTwoTemplates() throws ConfigurationProviderException, IOException { when(httpURLConnection.getInputStream()).thenReturn(TemplatesIteratorTest.class.getClassLoader().getResourceAsStream("twoTemplates.json")); List> idToNameList; - try (TemplatesIterator templatesIterator = new TemplatesIterator(niFiRestConnector, jsonFactory)) { + try (TemplatesIterator templatesIterator = new TemplatesIterator(httpConnector, jsonFactory)) { idToNameList = Lists.newArrayList(templatesIterator); } assertEquals(2, idToNameList.size()); diff --git a/minifi-c2/minifi-c2-provider/minifi-c2-provider-util/pom.xml b/minifi-c2/minifi-c2-provider/minifi-c2-provider-util/pom.xml new file mode 100644 index 000000000..cbb390518 --- /dev/null +++ b/minifi-c2/minifi-c2-provider/minifi-c2-provider-util/pom.xml @@ -0,0 +1,44 @@ + + + + 4.0.0 + + minifi-c2-provider + org.apache.nifi.minifi + 0.2.0-SNAPSHOT + + minifi-c2-provider-util + jar + + + + org.apache.nifi.minifi + minifi-c2-api + ${project.version} + + + org.eclipse.jetty + jetty-util + + + org.mockito + mockito-all + test + + + diff --git a/minifi-c2/minifi-c2-provider/minifi-c2-provider-util/src/main/java/org/apache/nifi/minifi/c2/provider/util/HttpConnector.java b/minifi-c2/minifi-c2-provider/minifi-c2-provider-util/src/main/java/org/apache/nifi/minifi/c2/provider/util/HttpConnector.java new file mode 100644 index 000000000..a681af942 --- /dev/null +++ b/minifi-c2/minifi-c2-provider/minifi-c2-provider-util/src/main/java/org/apache/nifi/minifi/c2/provider/util/HttpConnector.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.c2.provider.util; + +import org.apache.nifi.minifi.c2.api.ConfigurationProviderException; +import org.apache.nifi.minifi.c2.api.InvalidParameterException; +import org.apache.nifi.minifi.c2.api.properties.C2Properties; +import org.eclipse.jetty.util.ssl.SslContextFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSocketFactory; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.InetSocketAddress; +import java.net.MalformedURLException; +import java.net.Proxy; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.security.GeneralSecurityException; +import java.util.Base64; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class HttpConnector { + private static final Logger logger = LoggerFactory.getLogger(HttpConnector.class); + + private final String baseUrl; + private final SslContextFactory sslContextFactory; + private final Proxy proxy; + private final String proxyAuthorization; + + public HttpConnector(String baseUrl) throws InvalidParameterException, GeneralSecurityException, IOException { + this(baseUrl, null, 0); + } + + public HttpConnector(String baseUrl, String proxyHost, int proxyPort) throws InvalidParameterException, GeneralSecurityException, IOException { + this(baseUrl, proxyHost, proxyPort, null, null); + } + + public HttpConnector(String baseUrl, String proxyHost, int proxyPort, String proxyUsername, String proxyPassword) throws InvalidParameterException, GeneralSecurityException, IOException { + if (baseUrl.startsWith("https:")) { + sslContextFactory = C2Properties.getInstance().getSslContextFactory(); + if (sslContextFactory == null) { + throw new InvalidParameterException("Need sslContextFactory to connect to https endpoint (" + baseUrl + ")"); + } + } else { + sslContextFactory = null; + } + this.baseUrl = baseUrl; + if (proxyHost != null && !proxyHost.isEmpty()) { + if (proxyPort == 0) { + throw new InvalidParameterException("Must specify proxy port with proxy host"); + } + proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort)); + } else { + proxy = null; + } + + if (proxyUsername != null && !proxyUsername.isEmpty()) { + if (proxy == null) { + throw new InvalidParameterException("Cannot specify proxy username without proxy host."); + } + if (proxyPassword == null) { + throw new InvalidParameterException("Must specify proxy password with proxy username."); + } + proxyAuthorization = Base64.getEncoder().encodeToString((proxyHost + ":" + proxyPassword).getBytes(StandardCharsets.UTF_8)); + } else { + proxyAuthorization = null; + } + } + + public HttpURLConnection get(String endpointPath) throws ConfigurationProviderException { + return get(endpointPath, Collections.emptyMap()); + } + + public HttpURLConnection get(String endpointPath, Map> headers) throws ConfigurationProviderException { + String endpointUrl = baseUrl + endpointPath; + if (logger.isDebugEnabled()) { + logger.debug("Connecting to endpoint: " + endpointUrl); + } + URL url; + try { + url = new URL(endpointUrl); + } catch (MalformedURLException e) { + throw new ConfigurationProviderException("Malformed url " + endpointUrl, e); + } + + HttpURLConnection httpURLConnection; + try { + if (proxy == null) { + httpURLConnection = (HttpURLConnection) url.openConnection(); + } else { + httpURLConnection = (HttpURLConnection) url.openConnection(proxy); + } + if (sslContextFactory != null) { + HttpsURLConnection httpsURLConnection = (HttpsURLConnection) httpURLConnection; + SSLContext sslContext = sslContextFactory.getSslContext(); + SSLSocketFactory socketFactory = sslContext.getSocketFactory(); + httpsURLConnection.setSSLSocketFactory(socketFactory); + } + } catch (IOException e) { + throw new ConfigurationProviderException("Unable to connect to " + url, e); + } + if (proxyAuthorization != null) { + httpURLConnection.setRequestProperty("Proxy-Authorization", proxyAuthorization); + } + headers.forEach((s, strings) -> httpURLConnection.setRequestProperty(s, strings.stream().collect(Collectors.joining(",")))); + return httpURLConnection; + } +} diff --git a/minifi-c2/minifi-c2-provider/pom.xml b/minifi-c2/minifi-c2-provider/pom.xml index fc0fbae86..4dff46641 100644 --- a/minifi-c2/minifi-c2-provider/pom.xml +++ b/minifi-c2/minifi-c2-provider/pom.xml @@ -26,7 +26,9 @@ limitations under the License. pom + minifi-c2-provider-util minifi-c2-provider-cache + minifi-c2-provider-delegating minifi-c2-provider-nifi-rest diff --git a/minifi-c2/minifi-c2-service/pom.xml b/minifi-c2/minifi-c2-service/pom.xml index eb52d7def..e40366744 100644 --- a/minifi-c2/minifi-c2-service/pom.xml +++ b/minifi-c2/minifi-c2-service/pom.xml @@ -57,6 +57,16 @@ limitations under the License. spring-security-config provided + + com.google.guava + guava + provided + + + com.fasterxml.jackson.core + jackson-databind + provided + org.yaml snakeyaml diff --git a/minifi-c2/minifi-c2-service/src/main/java/org/apache/nifi/minifi/c2/service/ConfigService.java b/minifi-c2/minifi-c2-service/src/main/java/org/apache/nifi/minifi/c2/service/ConfigService.java index e57089ee4..993608718 100644 --- a/minifi-c2/minifi-c2-service/src/main/java/org/apache/nifi/minifi/c2/service/ConfigService.java +++ b/minifi-c2/minifi-c2-service/src/main/java/org/apache/nifi/minifi/c2/service/ConfigService.java @@ -17,9 +17,17 @@ package org.apache.nifi.minifi.c2.service; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.util.concurrent.UncheckedExecutionException; import com.wordnik.swagger.annotations.Api; import org.apache.nifi.minifi.c2.api.Configuration; import org.apache.nifi.minifi.c2.api.ConfigurationProvider; +import org.apache.nifi.minifi.c2.api.ConfigurationProviderException; import org.apache.nifi.minifi.c2.api.InvalidParameterException; import org.apache.nifi.minifi.c2.api.security.authorization.AuthorizationException; import org.apache.nifi.minifi.c2.api.security.authorization.Authorizer; @@ -32,6 +40,7 @@ import javax.servlet.http.HttpServletRequest; import javax.ws.rs.GET; import javax.ws.rs.Path; +import javax.ws.rs.Produces; import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.Context; import javax.ws.rs.core.HttpHeaders; @@ -43,10 +52,15 @@ import java.io.InputStream; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @Path("/config") @@ -56,15 +70,105 @@ ) public class ConfigService { private static final Logger logger = LoggerFactory.getLogger(ConfigService.class); - private final List> configurationProviders; private final Authorizer authorizer; + private final ObjectMapper objectMapper; + private final Supplier configurationProviderInfo; + private final LoadingCache configurationCache; public ConfigService(List configurationProviders, Authorizer authorizer) { + this(configurationProviders, authorizer, 1000, 300_000); + } + public ConfigService(List configurationProviders, Authorizer authorizer, long maximumCacheSize, long cacheTtlMillis) { this.authorizer = authorizer; + this.objectMapper = new ObjectMapper(); if (configurationProviders == null || configurationProviders.size() == 0) { throw new IllegalArgumentException("Expected at least one configuration provider"); } - this.configurationProviders = configurationProviders.stream().map(c -> new Pair<>(MediaType.valueOf(c.getContentType()), c)).collect(Collectors.toList()); + this.configurationProviderInfo = Suppliers.memoizeWithExpiration(() -> initContentTypeInfo(configurationProviders), cacheTtlMillis, TimeUnit.MILLISECONDS); + CacheBuilder cacheBuilder = CacheBuilder.newBuilder(); + if (maximumCacheSize >= 0) { + cacheBuilder = cacheBuilder.maximumSize(maximumCacheSize); + } + if (cacheTtlMillis >= 0) { + cacheBuilder = cacheBuilder.expireAfterAccess(cacheTtlMillis, TimeUnit.MILLISECONDS); + } + this.configurationCache = cacheBuilder + .build(new CacheLoader() { + @Override + public ConfigurationProviderValue load(ConfigurationProviderKey key) throws Exception { + return initConfigurationProviderValue(key); + } + }); + } + + public ConfigurationProviderValue initConfigurationProviderValue(ConfigurationProviderKey key) { + try { + List acceptValues = key.getAcceptValues(); + Pair providerPair = getProvider(acceptValues); + + Map> parameters = key.getParameters(); + + Integer version = null; + List versionList = parameters.get("version"); + if (versionList != null && versionList.size() > 0) { + try { + version = Integer.parseInt(versionList.get(0)); + } catch (NumberFormatException e) { + throw new InvalidParameterException("Unable to parse " + version + " as integer.", e); + } + } + return new ConfigurationProviderValue(providerPair.getSecond().getConfiguration(providerPair.getFirst().toString(), version, parameters), providerPair.getFirst(), null); + } catch (ConfigurationProviderException e) { + return new ConfigurationProviderValue(null, null, e); + } + } + + protected ConfigurationProviderInfo initContentTypeInfo(List configurationProviders) { + List> mediaTypeList = new ArrayList<>(); + List contentTypes = new ArrayList<>(); + Set seenMediaTypes = new LinkedHashSet<>(); + + for (ConfigurationProvider configurationProvider : configurationProviders) { + try { + for (String contentTypeString : configurationProvider.getContentTypes()) { + MediaType mediaType = MediaType.valueOf(contentTypeString); + if (seenMediaTypes.add(mediaType)) { + contentTypes.add(contentTypeString); + mediaTypeList.add(new Pair<>(mediaType, configurationProvider)); + } + } + } catch (ConfigurationProviderException e) { + return new ConfigurationProviderInfo(null, null, e); + } + } + return new ConfigurationProviderInfo(mediaTypeList, contentTypes, null); + } + + @GET + @Path("/contentTypes") + @Produces(MediaType.APPLICATION_JSON) + public Response getContentTypes(@Context HttpServletRequest request, @Context UriInfo uriInfo) { + try { + authorizer.authorize(SecurityContextHolder.getContext().getAuthentication(), uriInfo); + } catch (AuthorizationException e) { + logger.warn(HttpRequestUtil.getClientString(request) + " not authorized to access " + uriInfo, e); + return Response.status(403).build(); + } + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + List contentTypes; + try { + contentTypes = configurationProviderInfo.get().getContentTypes(); + } catch (ConfigurationProviderException e) { + logger.warn("Unable to initialize content type information.", e); + return Response.status(500).build(); + } + try { + objectMapper.writerWithDefaultPrettyPrinter().writeValue(byteArrayOutputStream, contentTypes); + } catch (IOException e) { + logger.warn("Unable to write configuration providers to output stream.", e); + return Response.status(500).build(); + } + return Response.ok().type(MediaType.APPLICATION_JSON_TYPE).entity(byteArrayOutputStream.toByteArray()).build(); } @GET @@ -98,29 +202,20 @@ public Response getConfig(@Context HttpServletRequest request, @Context HttpHead .append(acceptValues.stream().map(Object::toString).collect(Collectors.joining(", "))); logger.debug(builder.toString()); } - Pair providerPair = getProvider(acceptValues); try { - Integer version = null; - List versionList = parameters.get("version"); - if (versionList != null && versionList.size() > 0) { - try { - version = Integer.parseInt(versionList.get(0)); - } catch (NumberFormatException e) { - throw new InvalidParameterException("Unable to parse " + version + " as integer.", e); - } - } + ConfigurationProviderValue configurationProviderValue = configurationCache.get(new ConfigurationProviderKey(acceptValues, parameters)); + Configuration configuration = configurationProviderValue.getConfiguration(); Response.ResponseBuilder ok = Response.ok(); - Configuration configuration = providerPair.getSecond().getConfiguration(version, parameters); ok = ok.header("X-Content-Version", configuration.getVersion()); - ok = ok.type(providerPair.getFirst()); + ok = ok.type(configurationProviderValue.getMediaType()); byte[] buffer = new byte[1024]; int read; try (InputStream inputStream = configuration.getInputStream(); ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) { MessageDigest md5 = MessageDigest.getInstance("MD5"); MessageDigest sha256 = MessageDigest.getInstance("SHA-256"); - while((read = inputStream.read(buffer)) >= 0) { + while ((read = inputStream.read(buffer)) >= 0) { outputStream.write(buffer, 0, read); md5.update(buffer, 0, read); sha256.update(buffer, 0, read); @@ -128,16 +223,26 @@ public Response getConfig(@Context HttpServletRequest request, @Context HttpHead ok = ok.header("Content-MD5", bytesToHex(md5.digest())); ok = ok.header("X-Content-SHA-256", bytesToHex(sha256.digest())); ok = ok.entity(outputStream.toByteArray()); - } catch (IOException|NoSuchAlgorithmException e) { + } catch (ConfigurationProviderException | IOException | NoSuchAlgorithmException e) { logger.error("Error reading or checksumming configuration file", e); throw new WebApplicationException(500); } return ok.build(); + } catch (AuthorizationException e) { + logger.warn(HttpRequestUtil.getClientString(request) + " not authorized to access " + uriInfo, e); + return Response.status(403).build(); } catch (InvalidParameterException e) { logger.info(HttpRequestUtil.getClientString(request) + " made invalid request with " + HttpRequestUtil.getQueryString(request), e); return Response.status(400).entity("Invalid request.").build(); - } catch (Throwable t) { - logger.error(HttpRequestUtil.getClientString(request) + " made request with " + HttpRequestUtil.getQueryString(request) + " that caused error in " + providerPair.getSecond(), t); + } catch (ConfigurationProviderException e) { + logger.warn("Unable to get configuration.", e); + return Response.status(500).build(); + } catch (ExecutionException|UncheckedExecutionException e) { + Throwable cause = e.getCause(); + if (cause instanceof WebApplicationException) { + throw (WebApplicationException) cause; + } + logger.error(HttpRequestUtil.getClientString(request) + " made request with " + HttpRequestUtil.getQueryString(request) + " that caused error.", cause); return Response.status(500).entity("Internal error").build(); } } @@ -145,23 +250,30 @@ public Response getConfig(@Context HttpServletRequest request, @Context HttpHead // see: http://stackoverflow.com/questions/15429257/how-to-convert-byte-array-to-hexstring-in-java#answer-15429408 protected static String bytesToHex(byte[] in) { final StringBuilder builder = new StringBuilder(); - for(byte b : in) { + for (byte b : in) { builder.append(String.format("%02x", b)); } return builder.toString(); } - private Pair getProvider(List acceptValues) { + private Pair getProvider(List acceptValues) throws ConfigurationProviderException { + List> mediaTypeList; + try { + mediaTypeList = this.configurationProviderInfo.get().getMediaTypeList(); + } catch (ConfigurationProviderException.Wrapper e) { + throw e.unwrap(); + } for (MediaType accept : acceptValues) { - for (Pair configurationProviderPair : configurationProviders) { - if (accept.isCompatible(configurationProviderPair.getFirst())) { - return configurationProviderPair; + for (Pair pair : mediaTypeList) { + MediaType mediaType = pair.getFirst(); + if (accept.isCompatible(mediaType)) { + return new Pair<>(mediaType, pair.getSecond()); } } } throw new WebApplicationException(Response.status(406).entity("Unable to find configuration provider for " + "\"Accept: " + acceptValues.stream().map(Object::toString).collect(Collectors.joining(", ")) + "\" supported media types are " + - configurationProviders.stream().map(Pair::getFirst).map(Object::toString).collect(Collectors.joining(", "))).build()); + mediaTypeList.stream().map(Pair::getFirst).map(Object::toString).collect(Collectors.joining(", "))).build()); } } diff --git a/minifi-c2/minifi-c2-service/src/main/java/org/apache/nifi/minifi/c2/service/ConfigurationProviderInfo.java b/minifi-c2/minifi-c2-service/src/main/java/org/apache/nifi/minifi/c2/service/ConfigurationProviderInfo.java new file mode 100644 index 000000000..755a4e230 --- /dev/null +++ b/minifi-c2/minifi-c2-service/src/main/java/org/apache/nifi/minifi/c2/service/ConfigurationProviderInfo.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.c2.service; + +import org.apache.nifi.minifi.c2.api.ConfigurationProvider; +import org.apache.nifi.minifi.c2.api.ConfigurationProviderException; +import org.apache.nifi.minifi.c2.api.util.Pair; + +import javax.ws.rs.core.MediaType; +import java.util.List; + +public class ConfigurationProviderInfo { + private final List> mediaTypeList; + private final List contentTypes; + private final ConfigurationProviderException configurationProviderException; + + public ConfigurationProviderInfo(List> mediaTypeList, List contentTypes, ConfigurationProviderException configurationProviderException) { + this.mediaTypeList = mediaTypeList; + this.contentTypes = contentTypes; + this.configurationProviderException = configurationProviderException; + } + + public List> getMediaTypeList() throws ConfigurationProviderException { + if (configurationProviderException != null) { + throw configurationProviderException; + } + return mediaTypeList; + } + + public List getContentTypes() throws ConfigurationProviderException { + if (configurationProviderException != null) { + throw configurationProviderException; + } + return contentTypes; + } +} diff --git a/minifi-c2/minifi-c2-service/src/main/java/org/apache/nifi/minifi/c2/service/ConfigurationProviderKey.java b/minifi-c2/minifi-c2-service/src/main/java/org/apache/nifi/minifi/c2/service/ConfigurationProviderKey.java new file mode 100644 index 000000000..ca75d0012 --- /dev/null +++ b/minifi-c2/minifi-c2-service/src/main/java/org/apache/nifi/minifi/c2/service/ConfigurationProviderKey.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.c2.service; + +import javax.ws.rs.core.MediaType; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class ConfigurationProviderKey { + private final List acceptValues; + private final Map> parameters; + + public ConfigurationProviderKey(List acceptValues, Map> parameters) { + this.acceptValues = Collections.unmodifiableList(new ArrayList<>(acceptValues)); + this.parameters = Collections.unmodifiableMap(parameters.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> Collections.unmodifiableList(new ArrayList<>(e.getValue()))))); + } + + public List getAcceptValues() { + return acceptValues; + } + + public Map> getParameters() { + return parameters; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + ConfigurationProviderKey that = (ConfigurationProviderKey) o; + + if (!acceptValues.equals(that.acceptValues)) return false; + return parameters.equals(that.parameters); + } + + @Override + public int hashCode() { + int result = acceptValues.hashCode(); + result = 31 * result + parameters.hashCode(); + return result; + } +} diff --git a/minifi-c2/minifi-c2-service/src/main/java/org/apache/nifi/minifi/c2/service/ConfigurationProviderValue.java b/minifi-c2/minifi-c2-service/src/main/java/org/apache/nifi/minifi/c2/service/ConfigurationProviderValue.java new file mode 100644 index 000000000..875f4cb94 --- /dev/null +++ b/minifi-c2/minifi-c2-service/src/main/java/org/apache/nifi/minifi/c2/service/ConfigurationProviderValue.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.c2.service; + +import org.apache.nifi.minifi.c2.api.Configuration; +import org.apache.nifi.minifi.c2.api.ConfigurationProviderException; + +import javax.ws.rs.core.MediaType; + +public class ConfigurationProviderValue { + private final Configuration configuration; + private final MediaType mediaType; + private final ConfigurationProviderException configurationProviderException; + + public ConfigurationProviderValue(Configuration configuration, MediaType mediaType, ConfigurationProviderException configurationProviderException) { + this.configuration = configuration; + this.mediaType = mediaType; + this.configurationProviderException = configurationProviderException; + } + + public Configuration getConfiguration() throws ConfigurationProviderException { + if (configurationProviderException != null) { + throw configurationProviderException; + } + return configuration; + } + + public MediaType getMediaType() throws ConfigurationProviderException { + if (configurationProviderException != null) { + throw configurationProviderException; + } + return mediaType; + } +} diff --git a/minifi-docs/src/main/markdown/System_Admin_Guide.md b/minifi-docs/src/main/markdown/System_Admin_Guide.md index f1d85c3f4..9c7115aac 100644 --- a/minifi-docs/src/main/markdown/System_Admin_Guide.md +++ b/minifi-docs/src/main/markdown/System_Admin_Guide.md @@ -104,6 +104,10 @@ Option | Description ------ | ----------- nifi.minifi.notifier.ingestors.pull.http.hostname | Hostname on which to pull configurations from nifi.minifi.notifier.ingestors.pull.http.port | Port on which to pull configurations from +nifi.minifi.notifier.ingestors.pull.http.proxy.hostname | Proxy server hostname +nifi.minifi.notifier.ingestors.pull.http.proxy.port | Proxy server port +nifi.minifi.notifier.ingestors.pull.http.proxy.username | Proxy username +nifi.minifi.notifier.ingestors.pull.http.proxy.password | Proxy password nifi.minifi.notifier.ingestors.pull.http.path | Path on which to pull configurations from nifi.minifi.notifier.ingestors.pull.http.period.ms | Period on which to pull configurations from, defaults to 5 minutes if not set. nifi.minifi.notifier.ingestors.pull.http.use.etag | If the destination server is set up with cache control ability and utilizes an "ETag" header, then this should be set to true to utilize it. Very simply, the Ingestor remembers the "ETag" of the last successful pull (returned 200) then uses that "ETag" in a "If-None-Match" header on the next request. diff --git a/minifi-integration-tests/pom.xml b/minifi-integration-tests/pom.xml index 6356434f4..224858cc6 100644 --- a/minifi-integration-tests/pom.xml +++ b/minifi-integration-tests/pom.xml @@ -28,6 +28,7 @@ limitations under the License. ${project.version} + ${project.version} @@ -43,6 +44,24 @@ limitations under the License. 0.31.1 test + + org.apache.nifi.minifi + minifi-c2-integration-tests + ${project.version} + test-jar + test + + + org.apache.nifi + nifi-toolkit-tls + test + + + org.slf4j + slf4j-log4j12 + + + org.apache.nifi.minifi minifi-toolkit-configuration @@ -71,6 +90,16 @@ limitations under the License. src/test/resources true + + docker-compose-*.yml + + + + src/test/resources + false + + docker-compose-*.yml + diff --git a/minifi-integration-tests/src/test/java/org/apache/nifi/minifi/integration/c2/HierarchicalC2IntegrationTest.java b/minifi-integration-tests/src/test/java/org/apache/nifi/minifi/integration/c2/HierarchicalC2IntegrationTest.java new file mode 100644 index 000000000..58c936f53 --- /dev/null +++ b/minifi-integration-tests/src/test/java/org/apache/nifi/minifi/integration/c2/HierarchicalC2IntegrationTest.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.integration.c2; + +import com.palantir.docker.compose.DockerComposeRule; +import org.apache.nifi.minifi.c2.integration.test.health.HttpsStatusCodeHealthCheck; +import org.apache.nifi.minifi.integration.util.LogUtil; +import org.apache.nifi.security.util.SslContextFactory; +import org.apache.nifi.toolkit.tls.standalone.TlsToolkitStandalone; +import org.apache.nifi.toolkit.tls.standalone.TlsToolkitStandaloneCommandLine; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSocketFactory; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class HierarchicalC2IntegrationTest { + private static Path certificatesDirectory; + private static SSLContext trustSslContext; + private static SSLSocketFactory healthCheckSocketFactory; + + // Not annotated as rule because we need to generate certificatesDirectory first + public static DockerComposeRule docker = DockerComposeRule.builder() + .file("target/test-classes/docker-compose-c2-hierarchical.yml") + .waitingForServices(Arrays.asList("squid-edge3", "c2"), + new HttpsStatusCodeHealthCheck(container -> "https://c2-authoritative:10443/c2/config", + containers -> containers.get(0), containers -> containers.get(1), () -> healthCheckSocketFactory, 403)) + .build(); + private static Path resourceDirectory; + private static Path authoritativeFiles; + private static Path minifiEdge1Version2; + private static Path minifiEdge2Version2; + private static Path minifiEdge3Version2; + + /** + * Generates certificates with the tls-toolkit and then starts up the docker compose file + */ + @BeforeClass + public static void initCertificates() throws Exception { + resourceDirectory = Paths.get(HierarchicalC2IntegrationTest.class.getClassLoader() + .getResource("docker-compose-c2-hierarchical.yml").getFile()).getParent(); + certificatesDirectory = resourceDirectory.toAbsolutePath().resolve("certificates-c2-hierarchical"); + authoritativeFiles = resourceDirectory.resolve("c2").resolve("hierarchical").resolve("c2-authoritative").resolve("files"); + minifiEdge1Version2 = authoritativeFiles.resolve("edge1").resolve("raspi3").resolve("config.text.yml.v2"); + minifiEdge2Version2 = authoritativeFiles.resolve("edge2").resolve("raspi2").resolve("config.text.yml.v2"); + minifiEdge3Version2 = authoritativeFiles.resolve("edge3").resolve("raspi3").resolve("config.text.yml.v2"); + + if (Files.exists(minifiEdge1Version2)) { + Files.delete(minifiEdge1Version2); + } + if (Files.exists(minifiEdge2Version2)) { + Files.delete(minifiEdge2Version2); + } + if (Files.exists(minifiEdge3Version2)) { + Files.delete(minifiEdge3Version2); + } + + List toolkitCommandLine = new ArrayList<>(Arrays.asList("-O", "-o", certificatesDirectory.toFile().getAbsolutePath(), "-S", "badKeystorePass", "-P", "badTrustPass")); + for (String serverHostname : Arrays.asList("c2-authoritative", "minifi-edge1", "c2-edge2", "minifi-edge3")) { + toolkitCommandLine.add("-n"); + toolkitCommandLine.add(serverHostname); + } + Files.createDirectories(certificatesDirectory); + TlsToolkitStandaloneCommandLine tlsToolkitStandaloneCommandLine = new TlsToolkitStandaloneCommandLine(); + tlsToolkitStandaloneCommandLine.parse(toolkitCommandLine.toArray(new String[toolkitCommandLine.size()])); + new TlsToolkitStandalone().createNifiKeystoresAndTrustStores(tlsToolkitStandaloneCommandLine.createConfig()); + + trustSslContext = SslContextFactory.createTrustSslContext(certificatesDirectory.resolve("c2-authoritative") + .resolve("truststore.jks").toFile().getAbsolutePath(), "badTrustPass".toCharArray(), "jks", "TLS"); + healthCheckSocketFactory = trustSslContext.getSocketFactory(); + + docker.before(); + } + + @AfterClass + public static void afterClass() { + docker.after(); + } + + @Test(timeout = 120_000) + public void testMiNiFiEdge1() throws Exception { + LogUtil.verifyLogEntries("c2/hierarchical/minifi-edge1/expected.json", docker.containers().container("minifi-edge1")); + Path csvToJsonDir = resourceDirectory.resolve("standalone").resolve("v1").resolve("CsvToJson").resolve("yml"); + Files.copy(csvToJsonDir.resolve("CsvToJson.yml"), minifiEdge1Version2); + LogUtil.verifyLogEntries("standalone/v1/CsvToJson/yml/expected.json", docker.containers().container("minifi-edge1")); + } + + @Test(timeout = 120_000) + public void testMiNiFiEdge2() throws Exception { + LogUtil.verifyLogEntries("c2/hierarchical/minifi-edge2/expected.json", docker.containers().container("minifi-edge2")); + Path csvToJsonDir = resourceDirectory.resolve("standalone").resolve("v1").resolve("CsvToJson").resolve("yml"); + Files.copy(csvToJsonDir.resolve("CsvToJson.yml"), minifiEdge2Version2); + LogUtil.verifyLogEntries("standalone/v1/CsvToJson/yml/expected.json", docker.containers().container("minifi-edge2")); + } + + @Test(timeout = 120_000) + public void testMiNiFiEdge3() throws Exception { + LogUtil.verifyLogEntries("c2/hierarchical/minifi-edge3/expected.json", docker.containers().container("minifi-edge3")); + Path csvToJsonDir = resourceDirectory.resolve("standalone").resolve("v1").resolve("CsvToJson").resolve("yml"); + Files.copy(csvToJsonDir.resolve("CsvToJson.yml"), minifiEdge3Version2); + LogUtil.verifyLogEntries("standalone/v1/CsvToJson/yml/expected.json", docker.containers().container("minifi-edge3")); + } +} diff --git a/minifi-integration-tests/src/test/java/org/apache/nifi/minifi/integration/standalone/test/StandaloneYamlTest.java b/minifi-integration-tests/src/test/java/org/apache/nifi/minifi/integration/standalone/test/StandaloneYamlTest.java index bf9cd9690..3d3dee895 100644 --- a/minifi-integration-tests/src/test/java/org/apache/nifi/minifi/integration/standalone/test/StandaloneYamlTest.java +++ b/minifi-integration-tests/src/test/java/org/apache/nifi/minifi/integration/standalone/test/StandaloneYamlTest.java @@ -18,10 +18,9 @@ package org.apache.nifi.minifi.integration.standalone.test; -import com.fasterxml.jackson.databind.ObjectMapper; import com.palantir.docker.compose.DockerComposeRule; -import com.palantir.docker.compose.connection.DockerPort; import com.palantir.docker.compose.connection.waiting.HealthChecks; +import org.apache.nifi.minifi.integration.util.LogUtil; import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; @@ -35,15 +34,8 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; -import java.net.HttpURLConnection; -import java.net.URL; import java.util.Arrays; import java.util.Collection; -import java.util.Map; -import java.util.concurrent.ExecutionException; -import java.util.regex.Pattern; - -import static org.junit.Assert.fail; @RunWith(Parameterized.class) public class StandaloneYamlTest { @@ -98,34 +90,7 @@ protected String getExpectedJson() { } @Test(timeout = 60_000) - public void verifyLogEntries() throws IOException, InterruptedException, ExecutionException { - Pattern expectedLine; - int expectedOccurences; - try (InputStream inputStream = StandaloneYamlTest.class.getClassLoader().getResourceAsStream(getExpectedJson())) { - Map map = new ObjectMapper().readValue(inputStream, Map.class); - expectedLine = Pattern.compile((String) map.get("pattern")); - expectedOccurences = (int) map.getOrDefault("occurrences", 1); - } - DockerPort dockerPort = dockerComposeRule.containers().container("minifi").port(8000); - URL url = new URL("http://" + dockerPort.getIp() + ":" + dockerPort.getExternalPort()); - HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection(); - try (InputStream inputStream = urlConnection.getInputStream(); - InputStreamReader inputStreamReader = new InputStreamReader(inputStream); - BufferedReader bufferedReader = new BufferedReader(inputStreamReader)) { - String line; - int occurrences = 0; - while ((line = bufferedReader.readLine()) != null) { - if (expectedLine.matcher(line).find()) { - logger.info("Found expected: " + line); - if (++occurrences >= expectedOccurences) { - logger.info("Found target " + occurrences + " times"); - return; - } - } - } - fail("End of log reached without " + expectedOccurences + " match(es)"); - } finally { - urlConnection.disconnect(); - } + public void verifyLogEntries() throws Exception { + LogUtil.verifyLogEntries(getExpectedJson(), dockerComposeRule.containers().container("minifi")); } } \ No newline at end of file diff --git a/minifi-integration-tests/src/test/java/org/apache/nifi/minifi/integration/util/LogUtil.java b/minifi-integration-tests/src/test/java/org/apache/nifi/minifi/integration/util/LogUtil.java new file mode 100644 index 000000000..3c60ecd83 --- /dev/null +++ b/minifi-integration-tests/src/test/java/org/apache/nifi/minifi/integration/util/LogUtil.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.integration.util; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.palantir.docker.compose.connection.Container; +import com.palantir.docker.compose.connection.DockerPort; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import static org.junit.Assert.fail; + +public class LogUtil { + private static final Logger logger = LoggerFactory.getLogger(LogUtil.class); + + public static void verifyLogEntries(String expectedJsonFilename, Container container) throws Exception { + List expectedLogEntries; + try (InputStream inputStream = LogUtil.class.getClassLoader().getResourceAsStream(expectedJsonFilename)) { + List> expected = new ObjectMapper().readValue(inputStream, List.class); + expectedLogEntries = expected.stream().map(map -> new ExpectedLogEntry(Pattern.compile((String)map.get("pattern")), (int) map.getOrDefault("occurrences", 1))).collect(Collectors.toList()); + } + DockerPort dockerPort = container.port(8000); + URL url = new URL("http://" + dockerPort.getIp() + ":" + dockerPort.getExternalPort()); + HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection(); + try (InputStream inputStream = urlConnection.getInputStream(); + InputStreamReader inputStreamReader = new InputStreamReader(inputStream); + BufferedReader bufferedReader = new BufferedReader(inputStreamReader)) { + String line; + for (ExpectedLogEntry expectedLogEntry : expectedLogEntries) { + boolean satisfied = false; + int occurrences = 0; + while ((line = bufferedReader.readLine()) != null) { + if (expectedLogEntry.pattern.matcher(line).find()) { + logger.info("Found expected: " + line); + if (++occurrences >= expectedLogEntry.numOccurrences) { + logger.info("Found target " + occurrences + " times"); + satisfied = true; + break; + } + } + } + if (!satisfied) { + fail("End of log reached without " + expectedLogEntry.numOccurrences + " match(es) of " + expectedLogEntry.pattern); + } + } + } finally { + urlConnection.disconnect(); + } + } + + private static class ExpectedLogEntry { + private final Pattern pattern; + private final int numOccurrences; + + private ExpectedLogEntry(Pattern pattern, int numOccurrences) { + this.pattern = pattern; + this.numOccurrences = numOccurrences; + } + } +} diff --git a/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/conf/authorities.yaml b/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/conf/authorities.yaml new file mode 100644 index 000000000..426f1b81e --- /dev/null +++ b/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/conf/authorities.yaml @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the \"License\"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +CN=minifi-edge1, OU=NIFI: + - EDGE_1 +CN=c2-edge2, OU=NIFI: + - EDGE_2 +CN=minifi-edge3, OU=NIFI: + - EDGE_3 diff --git a/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/conf/authorizations.yaml b/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/conf/authorizations.yaml new file mode 100644 index 000000000..24097b8b1 --- /dev/null +++ b/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/conf/authorizations.yaml @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the \"License\"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Default Action: deny +Paths: + /c2/config: + Default Action: deny + Actions: + - Authorization: EDGE_1 + Query Parameters: + net: edge1 + class: raspi3 + Action: allow + - Authorization: EDGE_2 + Query Parameters: + net: edge2 + class: raspi2 + Action: allow + - Authorization: EDGE_3 + Query Parameters: + net: edge3 + class: raspi3 + Action: allow + + /c2/config/contentTypes: + Default Action: deny + Actions: + - Authorization: EDGE_2 + Action: allow \ No newline at end of file diff --git a/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/conf/c2.properties b/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/conf/c2.properties new file mode 100644 index 000000000..d1b01ee7b --- /dev/null +++ b/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/conf/c2.properties @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +minifi.c2.server.port=10443 + +minifi.c2.server.secure=true +minifi.c2.server.keystore=./conf/keystore.jks +minifi.c2.server.keystoreType=JKS +minifi.c2.server.keystorePasswd=badKeystorePass +minifi.c2.server.keyPasswd=badKeystorePass +minifi.c2.server.truststore=./conf/truststore.jks +minifi.c2.server.truststoreType=JKS +minifi.c2.server.truststorePasswd=badTrustPass \ No newline at end of file diff --git a/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/conf/minifi-c2-context.xml b/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/conf/minifi-c2-context.xml new file mode 100644 index 000000000..95db87102 --- /dev/null +++ b/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/conf/minifi-c2-context.xml @@ -0,0 +1,63 @@ + + + + + + + + + + + text/yml + + + + + + ./files + + + ${net}/${class}/config + + + + + + + + + + + + + 1000 + + + 1000 + + + diff --git a/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/files/edge1/raspi3/config.text.yml.v1 b/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/files/edge1/raspi3/config.text.yml.v1 new file mode 100644 index 000000000..d778600f2 --- /dev/null +++ b/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/files/edge1/raspi3/config.text.yml.v1 @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the \"License\"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +MiNiFi Config Version: 3 +Flow Controller: + name: Edge 1 raspi3.v1 + comment: '' +Core Properties: + flow controller graceful shutdown period: 10 sec + flow service write delay interval: 500 ms + administrative yield duration: 30 sec + bored yield duration: 10 millis + max concurrent threads: 1 +FlowFile Repository: + partitions: 256 + checkpoint interval: 2 mins + always sync: false + Swap: + threshold: 20000 + in period: 5 sec + in threads: 1 + out period: 5 sec + out threads: 4 +Content Repository: + content claim max appendable size: 10 MB + content claim max flow files: 100 + always sync: false +Provenance Repository: + provenance rollover time: 1 min +Component Status Repository: + buffer size: 1440 + snapshot frequency: 1 min +Security Properties: + keystore: '' + keystore type: '' + keystore password: '' + key password: '' + truststore: '' + truststore type: '' + truststore password: '' + ssl protocol: '' + Sensitive Props: + key: '' + algorithm: PBEWITHMD5AND256BITAES-CBC-OPENSSL + provider: BC +Processors: [] +Process Groups: [] +Funnels: [] +Connections: [] +Remote Process Groups: [] +NiFi Properties Overrides: {} diff --git a/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/files/edge2/raspi2/config.text.yml.v1 b/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/files/edge2/raspi2/config.text.yml.v1 new file mode 100644 index 000000000..d762ad691 --- /dev/null +++ b/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/files/edge2/raspi2/config.text.yml.v1 @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the \"License\"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +MiNiFi Config Version: 3 +Flow Controller: + name: Edge 2 raspi2.v1 + comment: '' +Core Properties: + flow controller graceful shutdown period: 10 sec + flow service write delay interval: 500 ms + administrative yield duration: 30 sec + bored yield duration: 10 millis + max concurrent threads: 1 +FlowFile Repository: + partitions: 256 + checkpoint interval: 2 mins + always sync: false + Swap: + threshold: 20000 + in period: 5 sec + in threads: 1 + out period: 5 sec + out threads: 4 +Content Repository: + content claim max appendable size: 10 MB + content claim max flow files: 100 + always sync: false +Provenance Repository: + provenance rollover time: 1 min +Component Status Repository: + buffer size: 1440 + snapshot frequency: 1 min +Security Properties: + keystore: '' + keystore type: '' + keystore password: '' + key password: '' + truststore: '' + truststore type: '' + truststore password: '' + ssl protocol: '' + Sensitive Props: + key: '' + algorithm: PBEWITHMD5AND256BITAES-CBC-OPENSSL + provider: BC +Processors: [] +Process Groups: [] +Funnels: [] +Connections: [] +Remote Process Groups: [] +NiFi Properties Overrides: {} diff --git a/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/files/edge3/raspi3/config.text.yml.v1 b/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/files/edge3/raspi3/config.text.yml.v1 new file mode 100644 index 000000000..da3079040 --- /dev/null +++ b/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/files/edge3/raspi3/config.text.yml.v1 @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the \"License\"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +MiNiFi Config Version: 3 +Flow Controller: + name: Edge 3 raspi3.v1 + comment: '' +Core Properties: + flow controller graceful shutdown period: 10 sec + flow service write delay interval: 500 ms + administrative yield duration: 30 sec + bored yield duration: 10 millis + max concurrent threads: 1 +FlowFile Repository: + partitions: 256 + checkpoint interval: 2 mins + always sync: false + Swap: + threshold: 20000 + in period: 5 sec + in threads: 1 + out period: 5 sec + out threads: 4 +Content Repository: + content claim max appendable size: 10 MB + content claim max flow files: 100 + always sync: false +Provenance Repository: + provenance rollover time: 1 min +Component Status Repository: + buffer size: 1440 + snapshot frequency: 1 min +Security Properties: + keystore: '' + keystore type: '' + keystore password: '' + key password: '' + truststore: '' + truststore type: '' + truststore password: '' + ssl protocol: '' + Sensitive Props: + key: '' + algorithm: PBEWITHMD5AND256BITAES-CBC-OPENSSL + provider: BC +Processors: [] +Process Groups: [] +Funnels: [] +Connections: [] +Remote Process Groups: [] +NiFi Properties Overrides: {} diff --git a/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-edge2/conf/c2.properties b/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-edge2/conf/c2.properties new file mode 100644 index 000000000..58522fc7c --- /dev/null +++ b/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-edge2/conf/c2.properties @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +minifi.c2.server.port=10080 + +minifi.c2.server.secure=false +minifi.c2.server.keystore=./conf/keystore.jks +minifi.c2.server.keystoreType=JKS +minifi.c2.server.keystorePasswd=badKeystorePass +minifi.c2.server.keyPasswd=badKeystorePass +minifi.c2.server.truststore=./conf/truststore.jks +minifi.c2.server.truststoreType=JKS +minifi.c2.server.truststorePasswd=badTrustPass \ No newline at end of file diff --git a/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-edge2/conf/minifi-c2-context.xml b/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-edge2/conf/minifi-c2-context.xml new file mode 100644 index 000000000..3c11dd7e8 --- /dev/null +++ b/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-edge2/conf/minifi-c2-context.xml @@ -0,0 +1,62 @@ + + + + + + + + + + + + + ./cache + + + ${class}/${class} + + + + + https://c2-authoritative:10443 + + + + + + + + + + + 1000 + + + 1000 + + + diff --git a/minifi-integration-tests/src/test/resources/c2/hierarchical/minifi-edge1/bootstrap.conf b/minifi-integration-tests/src/test/resources/c2/hierarchical/minifi-edge1/bootstrap.conf new file mode 100644 index 000000000..80476423f --- /dev/null +++ b/minifi-integration-tests/src/test/resources/c2/hierarchical/minifi-edge1/bootstrap.conf @@ -0,0 +1,106 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Java command to use when running MiNiFi +java=java + +# Username to use when running MiNiFi. This value will be ignored on Windows. +run.as= + +# Configure where MiNiFi's lib and conf directories live +lib.dir=./lib +conf.dir=./conf + +# How long to wait after telling MiNiFi to shutdown before explicitly killing the Process +graceful.shutdown.seconds=20 + +# The location for the configuration file +nifi.minifi.config=./conf/config.yml + +# Notifiers to use for the associated agent, comma separated list of class names +#nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.FileChangeIngestor +#nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.RestChangeIngestor +nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.PullHttpChangeIngestor + +# File change notifier configuration + +# Path of the file to monitor for changes. When these occur, the FileChangeNotifier, if configured, will begin the configuration reloading process +#nifi.minifi.notifier.ingestors.file.config.path= +# How frequently the file specified by 'nifi.minifi.notifier.file.config.path' should be evaluated for changes. +#nifi.minifi.notifier.ingestors.file.polling.period.seconds=5 + +# Rest change notifier configuration + +# Port on which the Jetty server will bind to, keep commented for a random open port +#nifi.minifi.notifier.ingestors.receive.http.port=8338 + +#Pull HTTP change notifier configuration + +# Hostname on which to pull configurations from +nifi.minifi.notifier.ingestors.pull.http.hostname=c2-authoritative +# Port on which to pull configurations from +nifi.minifi.notifier.ingestors.pull.http.port=10443 +# Path to pull configurations from +nifi.minifi.notifier.ingestors.pull.http.path=/c2/config +# Query string to pull configurations with +nifi.minifi.notifier.ingestors.pull.http.query=net=edge1&class=raspi3 +# Period on which to pull configurations from, defaults to 5 minutes if commented out +nifi.minifi.notifier.ingestors.pull.http.period.ms=3000 + +nifi.minifi.notifier.ingestors.pull.http.keystore.location=./conf/keystore.jks +nifi.minifi.notifier.ingestors.pull.http.keystore.type=JKS +nifi.minifi.notifier.ingestors.pull.http.keystore.password=badKeystorePass +nifi.minifi.notifier.ingestors.pull.http.truststore.location=./conf/truststore.jks +nifi.minifi.notifier.ingestors.pull.http.truststore.type=JKS +nifi.minifi.notifier.ingestors.pull.http.truststore.password=badTrustPass + +# Periodic Status Reporters to use for the associated agent, comma separated list of class names +#nifi.minifi.status.reporter.components=org.apache.nifi.minifi.bootstrap.status.reporters.StatusLogger + +# Periodic Status Logger configuration + +# The FlowStatus query to submit to the MiNiFi instance +#nifi.minifi.status.reporter.log.query=instance:health,bulletins +# The log level at which the status will be logged +#nifi.minifi.status.reporter.log.level=INFO +# The period (in milliseconds) at which to log the status +#nifi.minifi.status.reporter.log.period=60000 + +# Disable JSR 199 so that we can use JSP's without running a JDK +java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true + +# JVM memory settings +java.arg.2=-Xms256m +java.arg.3=-Xmx256m + +# Enable Remote Debugging +#java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000 + +java.arg.4=-Djava.net.preferIPv4Stack=true + +# allowRestrictedHeaders is required for Cluster/Node communications to work properly +java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true +java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol + +# The G1GC is still considered experimental but has proven to be very advantageous in providing great +# performance without significant "stop-the-world" delays. +#java.arg.13=-XX:+UseG1GC + +#Set headless mode by default +java.arg.14=-Djava.awt.headless=true + +java.arg.15=-Djava.security.egd=file:/dev/./urandom \ No newline at end of file diff --git a/minifi-integration-tests/src/test/resources/c2/hierarchical/minifi-edge1/expected.json b/minifi-integration-tests/src/test/resources/c2/hierarchical/minifi-edge1/expected.json new file mode 100644 index 000000000..ef85cefb9 --- /dev/null +++ b/minifi-integration-tests/src/test/resources/c2/hierarchical/minifi-edge1/expected.json @@ -0,0 +1,8 @@ +[ + { + "pattern": "ConfigurationChangeCoordinator Notifying Listeners of a change" + }, + { + "pattern": "MiNiFi has finished reloading successfully and swap file exists. Deleting old configuration" + } +] \ No newline at end of file diff --git a/minifi-integration-tests/src/test/resources/c2/hierarchical/minifi-edge2/bootstrap.conf b/minifi-integration-tests/src/test/resources/c2/hierarchical/minifi-edge2/bootstrap.conf new file mode 100644 index 000000000..927891f8e --- /dev/null +++ b/minifi-integration-tests/src/test/resources/c2/hierarchical/minifi-edge2/bootstrap.conf @@ -0,0 +1,99 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Java command to use when running MiNiFi +java=java + +# Username to use when running MiNiFi. This value will be ignored on Windows. +run.as= + +# Configure where MiNiFi's lib and conf directories live +lib.dir=./lib +conf.dir=./conf + +# How long to wait after telling MiNiFi to shutdown before explicitly killing the Process +graceful.shutdown.seconds=20 + +# The location for the configuration file +nifi.minifi.config=./conf/config.yml + +# Notifiers to use for the associated agent, comma separated list of class names +#nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.FileChangeIngestor +#nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.RestChangeIngestor +nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.PullHttpChangeIngestor + +# File change notifier configuration + +# Path of the file to monitor for changes. When these occur, the FileChangeNotifier, if configured, will begin the configuration reloading process +#nifi.minifi.notifier.ingestors.file.config.path= +# How frequently the file specified by 'nifi.minifi.notifier.file.config.path' should be evaluated for changes. +#nifi.minifi.notifier.ingestors.file.polling.period.seconds=5 + +# Rest change notifier configuration + +# Port on which the Jetty server will bind to, keep commented for a random open port +#nifi.minifi.notifier.ingestors.receive.http.port=8338 + +#Pull HTTP change notifier configuration + +# Hostname on which to pull configurations from +nifi.minifi.notifier.ingestors.pull.http.hostname=c2-edge2 +# Port on which to pull configurations from +nifi.minifi.notifier.ingestors.pull.http.port=10080 +# Path to pull configurations from +nifi.minifi.notifier.ingestors.pull.http.path=/c2/config +# Query string to pull configurations with +nifi.minifi.notifier.ingestors.pull.http.query=net=edge2&class=raspi2 +# Period on which to pull configurations from, defaults to 5 minutes if commented out +nifi.minifi.notifier.ingestors.pull.http.period.ms=3000 + +# Periodic Status Reporters to use for the associated agent, comma separated list of class names +#nifi.minifi.status.reporter.components=org.apache.nifi.minifi.bootstrap.status.reporters.StatusLogger + +# Periodic Status Logger configuration + +# The FlowStatus query to submit to the MiNiFi instance +#nifi.minifi.status.reporter.log.query=instance:health,bulletins +# The log level at which the status will be logged +#nifi.minifi.status.reporter.log.level=INFO +# The period (in milliseconds) at which to log the status +#nifi.minifi.status.reporter.log.period=60000 + +# Disable JSR 199 so that we can use JSP's without running a JDK +java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true + +# JVM memory settings +java.arg.2=-Xms256m +java.arg.3=-Xmx256m + +# Enable Remote Debugging +#java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000 + +java.arg.4=-Djava.net.preferIPv4Stack=true + +# allowRestrictedHeaders is required for Cluster/Node communications to work properly +java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true +java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol + +# The G1GC is still considered experimental but has proven to be very advantageous in providing great +# performance without significant "stop-the-world" delays. +#java.arg.13=-XX:+UseG1GC + +#Set headless mode by default +java.arg.14=-Djava.awt.headless=true + +java.arg.15=-Djava.security.egd=file:/dev/./urandom \ No newline at end of file diff --git a/minifi-integration-tests/src/test/resources/c2/hierarchical/minifi-edge2/expected.json b/minifi-integration-tests/src/test/resources/c2/hierarchical/minifi-edge2/expected.json new file mode 100644 index 000000000..ef85cefb9 --- /dev/null +++ b/minifi-integration-tests/src/test/resources/c2/hierarchical/minifi-edge2/expected.json @@ -0,0 +1,8 @@ +[ + { + "pattern": "ConfigurationChangeCoordinator Notifying Listeners of a change" + }, + { + "pattern": "MiNiFi has finished reloading successfully and swap file exists. Deleting old configuration" + } +] \ No newline at end of file diff --git a/minifi-integration-tests/src/test/resources/c2/hierarchical/minifi-edge3/bootstrap.conf b/minifi-integration-tests/src/test/resources/c2/hierarchical/minifi-edge3/bootstrap.conf new file mode 100644 index 000000000..9ffe7d35a --- /dev/null +++ b/minifi-integration-tests/src/test/resources/c2/hierarchical/minifi-edge3/bootstrap.conf @@ -0,0 +1,109 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Java command to use when running MiNiFi +java=java + +# Username to use when running MiNiFi. This value will be ignored on Windows. +run.as= + +# Configure where MiNiFi's lib and conf directories live +lib.dir=./lib +conf.dir=./conf + +# How long to wait after telling MiNiFi to shutdown before explicitly killing the Process +graceful.shutdown.seconds=20 + +# The location for the configuration file +nifi.minifi.config=./conf/config.yml + +# Notifiers to use for the associated agent, comma separated list of class names +#nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.FileChangeIngestor +#nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.RestChangeIngestor +nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.PullHttpChangeIngestor + +# File change notifier configuration + +# Path of the file to monitor for changes. When these occur, the FileChangeNotifier, if configured, will begin the configuration reloading process +#nifi.minifi.notifier.ingestors.file.config.path= +# How frequently the file specified by 'nifi.minifi.notifier.file.config.path' should be evaluated for changes. +#nifi.minifi.notifier.ingestors.file.polling.period.seconds=5 + +# Rest change notifier configuration + +# Port on which the Jetty server will bind to, keep commented for a random open port +#nifi.minifi.notifier.ingestors.receive.http.port=8338 + +#Pull HTTP change notifier configuration + +# Hostname on which to pull configurations from +nifi.minifi.notifier.ingestors.pull.http.hostname=c2-authoritative +# Port on which to pull configurations from +nifi.minifi.notifier.ingestors.pull.http.port=10443 +# Path to pull configurations from +nifi.minifi.notifier.ingestors.pull.http.path=/c2/config +# Query string to pull configurations with +nifi.minifi.notifier.ingestors.pull.http.query=net=edge3&class=raspi3 +# Period on which to pull configurations from, defaults to 5 minutes if commented out +nifi.minifi.notifier.ingestors.pull.http.period.ms=3000 + +nifi.minifi.notifier.ingestors.pull.http.proxy.hostname=squid-edge3 +nifi.minifi.notifier.ingestors.pull.http.proxy.port=3128 + +nifi.minifi.notifier.ingestors.pull.http.keystore.location=./conf/keystore.jks +nifi.minifi.notifier.ingestors.pull.http.keystore.type=JKS +nifi.minifi.notifier.ingestors.pull.http.keystore.password=badKeystorePass +nifi.minifi.notifier.ingestors.pull.http.truststore.location=./conf/truststore.jks +nifi.minifi.notifier.ingestors.pull.http.truststore.type=JKS +nifi.minifi.notifier.ingestors.pull.http.truststore.password=badTrustPass + +# Periodic Status Reporters to use for the associated agent, comma separated list of class names +#nifi.minifi.status.reporter.components=org.apache.nifi.minifi.bootstrap.status.reporters.StatusLogger + +# Periodic Status Logger configuration + +# The FlowStatus query to submit to the MiNiFi instance +#nifi.minifi.status.reporter.log.query=instance:health,bulletins +# The log level at which the status will be logged +#nifi.minifi.status.reporter.log.level=INFO +# The period (in milliseconds) at which to log the status +#nifi.minifi.status.reporter.log.period=60000 + +# Disable JSR 199 so that we can use JSP's without running a JDK +java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true + +# JVM memory settings +java.arg.2=-Xms256m +java.arg.3=-Xmx256m + +# Enable Remote Debugging +#java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000 + +java.arg.4=-Djava.net.preferIPv4Stack=true + +# allowRestrictedHeaders is required for Cluster/Node communications to work properly +java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true +java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol + +# The G1GC is still considered experimental but has proven to be very advantageous in providing great +# performance without significant "stop-the-world" delays. +#java.arg.13=-XX:+UseG1GC + +#Set headless mode by default +java.arg.14=-Djava.awt.headless=true + +java.arg.15=-Djava.security.egd=file:/dev/./urandom \ No newline at end of file diff --git a/minifi-integration-tests/src/test/resources/c2/hierarchical/minifi-edge3/expected.json b/minifi-integration-tests/src/test/resources/c2/hierarchical/minifi-edge3/expected.json new file mode 100644 index 000000000..ef85cefb9 --- /dev/null +++ b/minifi-integration-tests/src/test/resources/c2/hierarchical/minifi-edge3/expected.json @@ -0,0 +1,8 @@ +[ + { + "pattern": "ConfigurationChangeCoordinator Notifying Listeners of a change" + }, + { + "pattern": "MiNiFi has finished reloading successfully and swap file exists. Deleting old configuration" + } +] \ No newline at end of file diff --git a/minifi-integration-tests/src/test/resources/docker-compose-c2-hierarchical.yml b/minifi-integration-tests/src/test/resources/docker-compose-c2-hierarchical.yml new file mode 100644 index 000000000..a07aa5d97 --- /dev/null +++ b/minifi-integration-tests/src/test/resources/docker-compose-c2-hierarchical.yml @@ -0,0 +1,126 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the \"License\"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "2" + +services: + c2-authoritative: + image: apacheminific2:${minifi.c2.version} + ports: + - "10443" + hostname: c2-authoritative + volumes: + - ./c2/hierarchical/c2-authoritative/files:/opt/minifi-c2/minifi-c2-${minifi.c2.version}/files + - ./c2/hierarchical/c2-authoritative/conf/minifi-c2-context.xml:/opt/minifi-c2/minifi-c2-${minifi.c2.version}/conf/minifi-c2-context.xml + + - ./c2/hierarchical/c2-authoritative/conf/c2.properties:/opt/minifi-c2/minifi-c2-${minifi.c2.version}/conf/c2.properties + - ./c2/hierarchical/c2-authoritative/conf/authorities.yaml:/opt/minifi-c2/minifi-c2-${minifi.c2.version}/conf/authorities.yaml + - ./c2/hierarchical/c2-authoritative/conf/authorizations.yaml:/opt/minifi-c2/minifi-c2-${minifi.c2.version}/conf/authorizations.yaml + + - ./certificates-c2-hierarchical/c2-authoritative/keystore.jks:/opt/minifi-c2/minifi-c2-${minifi.c2.version}/conf/keystore.jks + - ./certificates-c2-hierarchical/c2-authoritative/truststore.jks:/opt/minifi-c2/minifi-c2-${minifi.c2.version}/conf/truststore.jks + networks: + - cluster + - edge1 + + minifi-edge1: + image: apacheminifi:${minifi.version} + ports: + - "8000" + volumes: + - ./tailFileServer.py:/home/minifi/tailFileServer.py + + - ./c2/hierarchical/minifi-edge1/bootstrap.conf:/opt/minifi/minifi-${minifi.version}/conf/bootstrap.conf + - ./logback.xml:/opt/minifi/minifi-${minifi.version}/conf/logback.xml + + - ./certificates-c2-hierarchical/minifi-edge1/keystore.jks:/opt/minifi/minifi-${minifi.version}/conf/keystore.jks + - ./certificates-c2-hierarchical/minifi-edge1/truststore.jks:/opt/minifi/minifi-${minifi.version}/conf/truststore.jks + entrypoint: + - bash + - -c + - /opt/minifi/minifi-${minifi.version}/bin/minifi.sh start && python /home/minifi/tailFileServer.py --file /opt/minifi/minifi-${minifi.version}/logs/minifi-app.log --file /opt/minifi/minifi-${minifi.version}/logs/minifi-bootstrap.log + networks: + - edge1 + + c2-edge2: + image: apacheminific2:${minifi.c2.version} + ports: + - "10080" + hostname: c2-edge2 + volumes: + - ./c2/hierarchical/c2-edge2/conf/c2.properties:/opt/minifi-c2/minifi-c2-${minifi.c2.version}/conf/c2.properties + - ./c2/hierarchical/c2-edge2/conf/minifi-c2-context.xml:/opt/minifi-c2/minifi-c2-${minifi.c2.version}/conf/minifi-c2-context.xml + + - ./certificates-c2-hierarchical/c2-edge2/keystore.jks:/opt/minifi-c2/minifi-c2-${minifi.c2.version}/conf/keystore.jks + - ./certificates-c2-hierarchical/c2-edge2/truststore.jks:/opt/minifi-c2/minifi-c2-${minifi.c2.version}/conf/truststore.jks + networks: + - cluster + - edge2 + + minifi-edge2: + image: apacheminifi:${minifi.version} + ports: + - "8000" + volumes: + - ./tailFileServer.py:/home/minifi/tailFileServer.py + + - ./c2/hierarchical/minifi-edge2/bootstrap.conf:/opt/minifi/minifi-${minifi.version}/conf/bootstrap.conf + - ./logback.xml:/opt/minifi/minifi-${minifi.version}/conf/logback.xml + entrypoint: + - bash + - -c + - /opt/minifi/minifi-${minifi.version}/bin/minifi.sh start && python /home/minifi/tailFileServer.py --file /opt/minifi/minifi-${minifi.version}/logs/minifi-app.log --file /opt/minifi/minifi-${minifi.version}/logs/minifi-bootstrap.log + networks: + - edge2 + + squid-edge3: + image: chrisdaish/squid + ports: + - "3128" + hostname: squid-edge3 + volumes: + - ./squid/squid.conf:/etc/squid/squid.conf + networks: + - cluster + - edge3 + + minifi-edge3: + image: apacheminifi:${minifi.version} + ports: + - "8000" + volumes: + - ./tailFileServer.py:/home/minifi/tailFileServer.py + + - ./c2/hierarchical/minifi-edge3/bootstrap.conf:/opt/minifi/minifi-${minifi.version}/conf/bootstrap.conf + - ./logback.xml:/opt/minifi/minifi-${minifi.version}/conf/logback.xml + + - ./certificates-c2-hierarchical/minifi-edge3/keystore.jks:/opt/minifi/minifi-${minifi.version}/conf/keystore.jks + - ./certificates-c2-hierarchical/minifi-edge3/truststore.jks:/opt/minifi/minifi-${minifi.version}/conf/truststore.jks + entrypoint: + - bash + - -c + - /opt/minifi/minifi-${minifi.version}/bin/minifi.sh start && python /home/minifi/tailFileServer.py --file /opt/minifi/minifi-${minifi.version}/logs/minifi-app.log --file /opt/minifi/minifi-${minifi.version}/logs/minifi-bootstrap.log + networks: + - edge3 + +networks: + cluster: + driver: bridge + edge1: + driver: bridge + edge2: + driver: bridge + edge3: + driver: bridge diff --git a/minifi-integration-tests/src/test/resources/logback.xml b/minifi-integration-tests/src/test/resources/logback.xml index 168ff9e87..7226fe394 100644 --- a/minifi-integration-tests/src/test/resources/logback.xml +++ b/minifi-integration-tests/src/test/resources/logback.xml @@ -59,6 +59,8 @@ + + diff --git a/minifi-integration-tests/src/test/resources/squid/squid.conf b/minifi-integration-tests/src/test/resources/squid/squid.conf new file mode 100644 index 000000000..90a524815 --- /dev/null +++ b/minifi-integration-tests/src/test/resources/squid/squid.conf @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the \"License\"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +http_access allow all + +# Choose the port you want. Below we set it to default 3128. +http_port 3128 \ No newline at end of file diff --git a/minifi-integration-tests/src/test/resources/standalone/v1/CsvToJson/xml/expected.json b/minifi-integration-tests/src/test/resources/standalone/v1/CsvToJson/xml/expected.json index ebc43fc15..b259ef780 100644 --- a/minifi-integration-tests/src/test/resources/standalone/v1/CsvToJson/xml/expected.json +++ b/minifi-integration-tests/src/test/resources/standalone/v1/CsvToJson/xml/expected.json @@ -1,4 +1,6 @@ -{ - "pattern": "o.a.n.c.r.StandardProcessSession StandardProcessSession\\[id=.*\\] for UpdateAttribute\\[id=1d00089c-78cd-467f-9aa6-31e3bdf90cb0\\], committed the following events: Transferred FlowFiles \\[.*\\] to 'success'", - "occurrences": 2 -} +[ + { + "pattern": "o.a.n.c.r.StandardProcessSession StandardProcessSession\\[id=.*\\] for UpdateAttribute\\[id=1d00089c-78cd-467f-9aa6-31e3bdf90cb0\\], committed the following events: Transferred FlowFiles \\[.*\\] to 'success'", + "occurrences": 2 + } +] diff --git a/minifi-integration-tests/src/test/resources/standalone/v1/CsvToJson/yml/expected.json b/minifi-integration-tests/src/test/resources/standalone/v1/CsvToJson/yml/expected.json index 1f8945763..afd1bde2b 100644 --- a/minifi-integration-tests/src/test/resources/standalone/v1/CsvToJson/yml/expected.json +++ b/minifi-integration-tests/src/test/resources/standalone/v1/CsvToJson/yml/expected.json @@ -1,4 +1,6 @@ -{ - "pattern": "o.a.n.c.r.StandardProcessSession StandardProcessSession\\[id=.*\\] for UpdateAttribute\\[id=2aac227f-e8e9-370f-87c8-4f970e0b260e\\], committed the following events: Transferred FlowFiles \\[.*\\] to 'success'", - "occurrences": 2 -} +[ + { + "pattern": "o.a.n.c.r.StandardProcessSession StandardProcessSession\\[id=.*\\] for UpdateAttribute\\[id=2aac227f-e8e9-370f-87c8-4f970e0b260e\\], committed the following events: Transferred FlowFiles \\[.*\\] to 'success'", + "occurrences": 2 + } +] diff --git a/minifi-integration-tests/src/test/resources/standalone/v1/DecompressionCircularFlow/xml/expected.json b/minifi-integration-tests/src/test/resources/standalone/v1/DecompressionCircularFlow/xml/expected.json index 781255913..b12e21ebc 100644 --- a/minifi-integration-tests/src/test/resources/standalone/v1/DecompressionCircularFlow/xml/expected.json +++ b/minifi-integration-tests/src/test/resources/standalone/v1/DecompressionCircularFlow/xml/expected.json @@ -1,4 +1,6 @@ -{ - "pattern": "o.a.n.c.r.StandardProcessSession StandardProcessSession\\[id=.*\\] for LogAttribute\\[id=7209cf79-23ba-421c-b1c3-925ed86c302d\\], committed the following events: Transferred FlowFiles \\[.*\\] to 'success'", - "occurrences": 1 -} +[ + { + "pattern": "o.a.n.c.r.StandardProcessSession StandardProcessSession\\[id=.*\\] for LogAttribute\\[id=7209cf79-23ba-421c-b1c3-925ed86c302d\\], committed the following events: Transferred FlowFiles \\[.*\\] to 'success'", + "occurrences": 1 + } +] diff --git a/minifi-integration-tests/src/test/resources/standalone/v1/DecompressionCircularFlow/yml/expected.json b/minifi-integration-tests/src/test/resources/standalone/v1/DecompressionCircularFlow/yml/expected.json index a48628980..f00e67565 100644 --- a/minifi-integration-tests/src/test/resources/standalone/v1/DecompressionCircularFlow/yml/expected.json +++ b/minifi-integration-tests/src/test/resources/standalone/v1/DecompressionCircularFlow/yml/expected.json @@ -1,4 +1,6 @@ -{ - "pattern": "o.a.n.c.r.StandardProcessSession StandardProcessSession\\[id=.*\\] for LogAttribute\\[id=59198b81-12fc-374f-bead-7fcd1d5d4823\\], committed the following events: Transferred FlowFiles \\[.*\\] to 'success'", - "occurrences": 1 -} +[ + { + "pattern": "o.a.n.c.r.StandardProcessSession StandardProcessSession\\[id=.*\\] for LogAttribute\\[id=59198b81-12fc-374f-bead-7fcd1d5d4823\\], committed the following events: Transferred FlowFiles \\[.*\\] to 'success'", + "occurrences": 1 + } +] diff --git a/minifi-integration-tests/src/test/resources/standalone/v1/MiNiFiTailLogAttribute/xml/expected.json b/minifi-integration-tests/src/test/resources/standalone/v1/MiNiFiTailLogAttribute/xml/expected.json index 57ad2d172..5091f4de2 100644 --- a/minifi-integration-tests/src/test/resources/standalone/v1/MiNiFiTailLogAttribute/xml/expected.json +++ b/minifi-integration-tests/src/test/resources/standalone/v1/MiNiFiTailLogAttribute/xml/expected.json @@ -1,4 +1,6 @@ -{ - "pattern": "o.a.n.c.r.StandardProcessSession StandardProcessSession\\[id=.*\\] for LogAttribute\\[id=5a36371c-4884-41cb-8792-073deb6c256a\\], committed the following events: Transferred FlowFiles \\[.*\\] to 'success'", - "occurrences": 20 -} +[ + { + "pattern": "o.a.n.c.r.StandardProcessSession StandardProcessSession\\[id=.*\\] for LogAttribute\\[id=5a36371c-4884-41cb-8792-073deb6c256a\\], committed the following events: Transferred FlowFiles \\[.*\\] to 'success'", + "occurrences": 20 + } +] diff --git a/minifi-integration-tests/src/test/resources/standalone/v1/MiNiFiTailLogAttribute/yml/expected.json b/minifi-integration-tests/src/test/resources/standalone/v1/MiNiFiTailLogAttribute/yml/expected.json index f98095b10..a6021487b 100644 --- a/minifi-integration-tests/src/test/resources/standalone/v1/MiNiFiTailLogAttribute/yml/expected.json +++ b/minifi-integration-tests/src/test/resources/standalone/v1/MiNiFiTailLogAttribute/yml/expected.json @@ -1,4 +1,6 @@ -{ - "pattern": "o.a.n.c.r.StandardProcessSession StandardProcessSession\\[id=.*\\] for LogAttribute\\[id=59198b81-12fc-374f-bead-7fcd1d5d4823\\], committed the following events: Transferred FlowFiles \\[.*\\] to 'success'", - "occurrences": 20 -} +[ + { + "pattern": "o.a.n.c.r.StandardProcessSession StandardProcessSession\\[id=.*\\] for LogAttribute\\[id=59198b81-12fc-374f-bead-7fcd1d5d4823\\], committed the following events: Transferred FlowFiles \\[.*\\] to 'success'", + "occurrences": 20 + } +] diff --git a/minifi-integration-tests/src/test/resources/standalone/v1/ReplaceTextExpressionLanguageCSVReformatting/xml/expected.json b/minifi-integration-tests/src/test/resources/standalone/v1/ReplaceTextExpressionLanguageCSVReformatting/xml/expected.json index 70303df4b..208e649a2 100644 --- a/minifi-integration-tests/src/test/resources/standalone/v1/ReplaceTextExpressionLanguageCSVReformatting/xml/expected.json +++ b/minifi-integration-tests/src/test/resources/standalone/v1/ReplaceTextExpressionLanguageCSVReformatting/xml/expected.json @@ -1,4 +1,6 @@ -{ - "pattern": "o.a.n.c.r.StandardProcessSession StandardProcessSession\\[id=.*\\] for UpdateAttribute\\[id=bfa4fb38-096b-455d-a10f-2a1ed044bd49\\], committed the following events: Transferred FlowFiles \\[.*\\] to 'success'", - "occurrences": 2 -} +[ + { + "pattern": "o.a.n.c.r.StandardProcessSession StandardProcessSession\\[id=.*\\] for UpdateAttribute\\[id=bfa4fb38-096b-455d-a10f-2a1ed044bd49\\], committed the following events: Transferred FlowFiles \\[.*\\] to 'success'", + "occurrences": 2 + } +] diff --git a/minifi-integration-tests/src/test/resources/standalone/v1/ReplaceTextExpressionLanguageCSVReformatting/yml/expected.json b/minifi-integration-tests/src/test/resources/standalone/v1/ReplaceTextExpressionLanguageCSVReformatting/yml/expected.json index 30eb6c7ad..45c444667 100644 --- a/minifi-integration-tests/src/test/resources/standalone/v1/ReplaceTextExpressionLanguageCSVReformatting/yml/expected.json +++ b/minifi-integration-tests/src/test/resources/standalone/v1/ReplaceTextExpressionLanguageCSVReformatting/yml/expected.json @@ -1,4 +1,6 @@ -{ - "pattern": "o.a.n.c.r.StandardProcessSession StandardProcessSession\\[id=.*\\] for UpdateAttribute\\[id=8c6c54be-7db6-333f-8c3b-d6e7b02411ff\\], committed the following events: Transferred FlowFiles \\[.*\\] to 'success'", - "occurrences": 2 -} +[ + { + "pattern": "o.a.n.c.r.StandardProcessSession StandardProcessSession\\[id=.*\\] for UpdateAttribute\\[id=8c6c54be-7db6-333f-8c3b-d6e7b02411ff\\], committed the following events: Transferred FlowFiles \\[.*\\] to 'success'", + "occurrences": 2 + } +] diff --git a/minifi-integration-tests/src/test/resources/standalone/v2/MultipleRelationships/xml/expected.json b/minifi-integration-tests/src/test/resources/standalone/v2/MultipleRelationships/xml/expected.json index 6110f19bc..02b1c066d 100644 --- a/minifi-integration-tests/src/test/resources/standalone/v2/MultipleRelationships/xml/expected.json +++ b/minifi-integration-tests/src/test/resources/standalone/v2/MultipleRelationships/xml/expected.json @@ -1,4 +1,6 @@ -{ - "pattern": "o.a.n.c.r.StandardProcessSession StandardProcessSession\\[.*\\] for LogAttribute\\[id=7c75ab71-0157-1000-0000-000000000000\\], committed the following events: Transferred FlowFiles \\[.*\\] to 'success'", - "occurrences": 20 -} +[ + { + "pattern": "o.a.n.c.r.StandardProcessSession StandardProcessSession\\[.*\\] for LogAttribute\\[id=7c75ab71-0157-1000-0000-000000000000\\], committed the following events: Transferred FlowFiles \\[.*\\] to 'success'", + "occurrences": 20 + } +] diff --git a/minifi-integration-tests/src/test/resources/standalone/v2/MultipleRelationships/yml/expected.json b/minifi-integration-tests/src/test/resources/standalone/v2/MultipleRelationships/yml/expected.json index 6110f19bc..02b1c066d 100644 --- a/minifi-integration-tests/src/test/resources/standalone/v2/MultipleRelationships/yml/expected.json +++ b/minifi-integration-tests/src/test/resources/standalone/v2/MultipleRelationships/yml/expected.json @@ -1,4 +1,6 @@ -{ - "pattern": "o.a.n.c.r.StandardProcessSession StandardProcessSession\\[.*\\] for LogAttribute\\[id=7c75ab71-0157-1000-0000-000000000000\\], committed the following events: Transferred FlowFiles \\[.*\\] to 'success'", - "occurrences": 20 -} +[ + { + "pattern": "o.a.n.c.r.StandardProcessSession StandardProcessSession\\[.*\\] for LogAttribute\\[id=7c75ab71-0157-1000-0000-000000000000\\], committed the following events: Transferred FlowFiles \\[.*\\] to 'success'", + "occurrences": 20 + } +] diff --git a/minifi-integration-tests/src/test/resources/standalone/v2/ProcessGroups/xml/expected.json b/minifi-integration-tests/src/test/resources/standalone/v2/ProcessGroups/xml/expected.json index a1e4e4337..7f9e81c19 100644 --- a/minifi-integration-tests/src/test/resources/standalone/v2/ProcessGroups/xml/expected.json +++ b/minifi-integration-tests/src/test/resources/standalone/v2/ProcessGroups/xml/expected.json @@ -1,4 +1,6 @@ -{ - "pattern": "o.a.n.c.r.StandardProcessSession StandardProcessSession\\[id=.*\\] for LogAttribute\\[id=e25e0e6e-0157-1000-0000-000000000000\\], committed the following events: Transferred FlowFiles \\[.*\\] to 'success'", - "occurrences": 20 -} +[ + { + "pattern": "o.a.n.c.r.StandardProcessSession StandardProcessSession\\[id=.*\\] for LogAttribute\\[id=e25e0e6e-0157-1000-0000-000000000000\\], committed the following events: Transferred FlowFiles \\[.*\\] to 'success'", + "occurrences": 20 + } +] diff --git a/minifi-integration-tests/src/test/resources/standalone/v2/ProcessGroups/yml/expected.json b/minifi-integration-tests/src/test/resources/standalone/v2/ProcessGroups/yml/expected.json index a1e4e4337..7f9e81c19 100644 --- a/minifi-integration-tests/src/test/resources/standalone/v2/ProcessGroups/yml/expected.json +++ b/minifi-integration-tests/src/test/resources/standalone/v2/ProcessGroups/yml/expected.json @@ -1,4 +1,6 @@ -{ - "pattern": "o.a.n.c.r.StandardProcessSession StandardProcessSession\\[id=.*\\] for LogAttribute\\[id=e25e0e6e-0157-1000-0000-000000000000\\], committed the following events: Transferred FlowFiles \\[.*\\] to 'success'", - "occurrences": 20 -} +[ + { + "pattern": "o.a.n.c.r.StandardProcessSession StandardProcessSession\\[id=.*\\] for LogAttribute\\[id=e25e0e6e-0157-1000-0000-000000000000\\], committed the following events: Transferred FlowFiles \\[.*\\] to 'success'", + "occurrences": 20 + } +] diff --git a/minifi-integration-tests/src/test/resources/standalone/v2/StressTestFramework/xml/expected.json b/minifi-integration-tests/src/test/resources/standalone/v2/StressTestFramework/xml/expected.json index 5f3740bdb..558903068 100644 --- a/minifi-integration-tests/src/test/resources/standalone/v2/StressTestFramework/xml/expected.json +++ b/minifi-integration-tests/src/test/resources/standalone/v2/StressTestFramework/xml/expected.json @@ -1,4 +1,6 @@ -{ - "pattern": "o.a.n.c.r.StandardProcessSession StandardProcessSession\\[id=.*\\] for RouteOnAttribute\\[id=397a4910-cc01-4c6b-0000-000000000000\\], committed the following events: Transferred FlowFiles \\[.*\\] to 'unmatched'", - "occurrences": 20 -} +[ + { + "pattern": "o.a.n.c.r.StandardProcessSession StandardProcessSession\\[id=.*\\] for RouteOnAttribute\\[id=397a4910-cc01-4c6b-0000-000000000000\\], committed the following events: Transferred FlowFiles \\[.*\\] to 'unmatched'", + "occurrences": 20 + } +] diff --git a/minifi-integration-tests/src/test/resources/standalone/v2/StressTestFramework/yml/expected.json b/minifi-integration-tests/src/test/resources/standalone/v2/StressTestFramework/yml/expected.json index 5f3740bdb..558903068 100644 --- a/minifi-integration-tests/src/test/resources/standalone/v2/StressTestFramework/yml/expected.json +++ b/minifi-integration-tests/src/test/resources/standalone/v2/StressTestFramework/yml/expected.json @@ -1,4 +1,6 @@ -{ - "pattern": "o.a.n.c.r.StandardProcessSession StandardProcessSession\\[id=.*\\] for RouteOnAttribute\\[id=397a4910-cc01-4c6b-0000-000000000000\\], committed the following events: Transferred FlowFiles \\[.*\\] to 'unmatched'", - "occurrences": 20 -} +[ + { + "pattern": "o.a.n.c.r.StandardProcessSession StandardProcessSession\\[id=.*\\] for RouteOnAttribute\\[id=397a4910-cc01-4c6b-0000-000000000000\\], committed the following events: Transferred FlowFiles \\[.*\\] to 'unmatched'", + "occurrences": 20 + } +] diff --git a/minifi-integration-tests/src/test/resources/tailFileServer.py b/minifi-integration-tests/src/test/resources/tailFileServer.py index bec2ccf0c..6bfd92ea1 100644 --- a/minifi-integration-tests/src/test/resources/tailFileServer.py +++ b/minifi-integration-tests/src/test/resources/tailFileServer.py @@ -31,7 +31,9 @@ def do_GET(self): self.send_response(200) self.send_header('Content-type','text/plain') self.end_headers() - p = Popen(['tail', '-f', '-n', '+1', TAIL_FILE], stdout=PIPE) + args = ['tail', '-f', '-n', '+1'] + args.extend(TAIL_FILES) + p = Popen(args, stdout=PIPE) try: for line in iter(p.stdout.readline, b''): self.wfile.write(line) @@ -43,7 +45,7 @@ def do_GET(self): if __name__ == '__main__': logging.basicConfig(level=logging.DEBUG) parser = ArgumentParser(description='Tail file over http') - parser.add_argument('--file') + parser.add_argument('--file', action='append') parser.add_argument('--port', type=int, default=8000) logging.debug('About to parse arguments') @@ -52,10 +54,10 @@ def do_GET(self): if not args.file: raise Exception('Must specify --file') - global TAIL_FILE - TAIL_FILE = args.file + global TAIL_FILES + TAIL_FILES = args.file - logging.debug('Serving tail of ' + TAIL_FILE + ' via HTTP at port ' + str(args.port)) + logging.debug('Serving tail of ' + str(TAIL_FILES) + ' via HTTP at port ' + str(args.port)) server = ThreadedHTTPServer(('', args.port), TailHTTPRequestHandler) server.serve_forever() \ No newline at end of file From d19d70f6f4fa1f821c316b3f3b66f8e551db1f12 Mon Sep 17 00:00:00 2001 From: Bryan Rosander Date: Wed, 26 Apr 2017 12:19:01 -0400 Subject: [PATCH 2/3] MINIFI-272 - Proxy auth, caching fixes --- .../delegating/DelegatingConfigurationProvider.java | 12 +++++++++++- .../nifi/minifi/c2/provider/util/HttpConnector.java | 2 +- .../apache/nifi/minifi/c2/service/ConfigService.java | 5 ++++- .../minifi/c2/service/ConfigurationProviderKey.java | 8 ++++++++ 4 files changed, 24 insertions(+), 3 deletions(-) diff --git a/minifi-c2/minifi-c2-provider/minifi-c2-provider-delegating/src/main/java/org/apache/nifi/minifi/c2/provider/delegating/DelegatingConfigurationProvider.java b/minifi-c2/minifi-c2-provider/minifi-c2-provider-delegating/src/main/java/org/apache/nifi/minifi/c2/provider/delegating/DelegatingConfigurationProvider.java index 6e0fec386..04f6017c0 100644 --- a/minifi-c2/minifi-c2-provider/minifi-c2-provider-delegating/src/main/java/org/apache/nifi/minifi/c2/provider/delegating/DelegatingConfigurationProvider.java +++ b/minifi-c2/minifi-c2-provider/minifi-c2-provider-delegating/src/main/java/org/apache/nifi/minifi/c2/provider/delegating/DelegatingConfigurationProvider.java @@ -28,6 +28,8 @@ import org.apache.nifi.minifi.c2.api.cache.WriteableConfiguration; import org.apache.nifi.minifi.c2.api.security.authorization.AuthorizationException; import org.apache.nifi.minifi.c2.provider.util.HttpConnector; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.InputStream; @@ -45,6 +47,7 @@ public class DelegatingConfigurationProvider implements ConfigurationProvider { public static final Pattern errorPattern = Pattern.compile("^Server returned HTTP response code: ([0-9]+) for URL:.*"); + private static final Logger logger = LoggerFactory.getLogger(DelegatingConfigurationProvider.class); private final ConfigurationCache configurationCache; private final HttpConnector httpConnector; private final ObjectMapper objectMapper; @@ -64,7 +67,11 @@ public List getContentTypes() throws ConfigurationProviderException { try { HttpURLConnection httpURLConnection = httpConnector.get("/c2/config/contentTypes"); try { - return objectMapper.readValue(httpURLConnection.getInputStream(), List.class); + List contentTypes = objectMapper.readValue(httpURLConnection.getInputStream(), List.class); + if (logger.isDebugEnabled()) { + logger.debug("Got content types: " + contentTypes); + } + return contentTypes; } finally { httpURLConnection.disconnect(); } @@ -80,6 +87,9 @@ public Configuration getConfiguration(String contentType, Integer version, Map configurationProviders, Authori cacheBuilder = cacheBuilder.maximumSize(maximumCacheSize); } if (cacheTtlMillis >= 0) { - cacheBuilder = cacheBuilder.expireAfterAccess(cacheTtlMillis, TimeUnit.MILLISECONDS); + cacheBuilder = cacheBuilder.refreshAfterWrite(cacheTtlMillis, TimeUnit.MILLISECONDS); } this.configurationCache = cacheBuilder .build(new CacheLoader() { @@ -102,6 +102,9 @@ public ConfigurationProviderValue load(ConfigurationProviderKey key) throws Exce } public ConfigurationProviderValue initConfigurationProviderValue(ConfigurationProviderKey key) { + if (logger.isDebugEnabled()) { + logger.debug("Attempting to load and cache configuration with key " + key); + } try { List acceptValues = key.getAcceptValues(); Pair providerPair = getProvider(acceptValues); diff --git a/minifi-c2/minifi-c2-service/src/main/java/org/apache/nifi/minifi/c2/service/ConfigurationProviderKey.java b/minifi-c2/minifi-c2-service/src/main/java/org/apache/nifi/minifi/c2/service/ConfigurationProviderKey.java index ca75d0012..3bc8919b6 100644 --- a/minifi-c2/minifi-c2-service/src/main/java/org/apache/nifi/minifi/c2/service/ConfigurationProviderKey.java +++ b/minifi-c2/minifi-c2-service/src/main/java/org/apache/nifi/minifi/c2/service/ConfigurationProviderKey.java @@ -52,6 +52,14 @@ public boolean equals(Object o) { return parameters.equals(that.parameters); } + @Override + public String toString() { + return "ConfigurationProviderKey{" + + "acceptValues=" + acceptValues + + ", parameters=" + parameters + + '}'; + } + @Override public int hashCode() { int result = acceptValues.hashCode(); From 202481334aeb48402f833c391c7e279432901eb2 Mon Sep 17 00:00:00 2001 From: Bryan Rosander Date: Fri, 12 May 2017 17:44:39 -0400 Subject: [PATCH 3/3] MINIFI-272 - Added C2 readme --- minifi-c2/README.md | 33 ++ minifi-c2/c2-integration-test.graphml | 630 ++++++++++++++++++++++++++ minifi-c2/c2-integration-test.png | Bin 0 -> 26942 bytes 3 files changed, 663 insertions(+) create mode 100644 minifi-c2/README.md create mode 100644 minifi-c2/c2-integration-test.graphml create mode 100644 minifi-c2/c2-integration-test.png diff --git a/minifi-c2/README.md b/minifi-c2/README.md new file mode 100644 index 000000000..24409471f --- /dev/null +++ b/minifi-c2/README.md @@ -0,0 +1,33 @@ + +## Apache NiFi MiNiFi Command and Control (C2) Server +MiNiFi agents allow us to push data flows down to smaller devices on the edge of the network. This provides many of the niceties of processing data with NiFi in a smaller package. One big challenge with many disparate agents running on all sorts of devices is coordinating their work and pushing out revised flows. + +The C2 server is the beginning of an attempt to address this usecase. It provides an endpoint for existing PullHttpChangeIngestor functionality that is intended to facilitate distributing appropriate flow definitions to each class of agent. + +In the assumed usecase one or more class of MiNiFi agent polls the C2 server periodically for updates to its flow. When there is a new version available, the C2 server will send it back to the agent at which point the agent will attempt to restart itself with the new flow, rolling back if there is a problem starting. + +The C2 server is intended to be extensible and flexibly configurable. The ConfigurationProvider interface is the main extension point where arbitrary logic should be able to be used to get updated flows. The server supports bidirectional TLS authentication and configurable authorization. + +### Configuration Providers: +There are three ConfigurationProvider implementations provided out of the box. +1. The [CacheConfigurationProvider](./minifi-c2-assembly/src/main/resources/conf/minifi-c2-context.xml) looks at directory on the filesystem. +2. The [DelegatingConfigurationProvider](./minifi-c2-integration-tests/src/test/resources/c2-unsecure-delegating/conf/minifi-c2-context.xml) delegates to another C2 server to allow for hierarchical C2 structures to help with scaling and/or bridging networks. +3. The [NiFiRestConfigurationProvider](./minifi-c2-integration-tests/src/test/resources/c2-unsecure-rest/conf/minifi-c2-context.xml) pulls templates from a NiFi instance over its REST API. (Note: sensitive values are NOT included in templates so this is unsuitable for flows with sensitive configuration currently) + +### Example network diagram: +Below is a network diagram showing the different configurations tested by [our hierarchical integration test docker-compose file.](../minifi-integration-tests/src/test/resources/docker-compose-c2-hierarchical.yml) It consists of a "cluster" network where real processing might occur as well as 3 "edge" networks that can get configuration from the cluster network a few different ways. The edge1 instance can directly access the authoritative C2 server via HTTPS. The edge2 instance is representative of a segmented network where the MiNiFi agents can talk to a local delegating C2 server over HTTP which asks the authoritative C2 server over HTTPS. The edge 3 instance can talk to the authoritative C2 server through a Squid proxy over HTTPS. + +![Network diagram](./c2-integration-test.png) diff --git a/minifi-c2/c2-integration-test.graphml b/minifi-c2/c2-integration-test.graphml new file mode 100644 index 000000000..82fbbf514 --- /dev/null +++ b/minifi-c2/c2-integration-test.graphml @@ -0,0 +1,630 @@ + + + + + + + + + + + + + + + + + + + + + + + + + Cluster Network + + + + + + + + + + + + Edge1 Network + + + + + + + + + + + + Edge2 Network + + + + + + + + + + + + C2-Authoritative + + + + + + + + + + + + + + + + + + + + + MiNiFi-Edge1 + + + + + + + + + + + + + + + + + + + + + C2-Edge2 + + + + + + + + + + + + + + + + + + + + + Edge3 Network + + + + + + + + + + + + Squid-Edge3 + + + + + + + + + + + + + + + + + + + + + MiNiFi-Edge2 + + + + + + + + + + + + + + + + + + + + + MiNiFi-Edge3 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + https + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + http + + + + + + + + + + + + + + + + + + + https + + + + + + + + + + + + + + + + + + + https through http proxy + + + + + + + + + + + + + + + <?xml version="1.0" encoding="utf-8"?> +<svg version="1.1" + xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" + x="0px" y="0px" width="41px" height="48px" viewBox="-0.875 -0.887 41 48" enable-background="new -0.875 -0.887 41 48" + xml:space="preserve"> +<defs> +</defs> +<linearGradient id="SVGID_1_" gradientUnits="userSpaceOnUse" x1="642.8008" y1="-979.1445" x2="682.0508" y2="-979.1445" gradientTransform="matrix(1 0 0 -1 -642.8008 -939.4756)"> + <stop offset="0" style="stop-color:#3C89C9"/> + <stop offset="0.1482" style="stop-color:#60A6DD"/> + <stop offset="0.3113" style="stop-color:#81C1F0"/> + <stop offset="0.4476" style="stop-color:#95D1FB"/> + <stop offset="0.5394" style="stop-color:#9CD7FF"/> + <stop offset="0.636" style="stop-color:#98D4FD"/> + <stop offset="0.7293" style="stop-color:#8DCAF6"/> + <stop offset="0.8214" style="stop-color:#79BBEB"/> + <stop offset="0.912" style="stop-color:#5EA5DC"/> + <stop offset="1" style="stop-color:#3C89C9"/> +</linearGradient> +<path fill="url(#SVGID_1_)" d="M19.625,36.763C8.787,36.763,0,34.888,0,32.575v10c0,2.313,8.787,4.188,19.625,4.188 + c10.839,0,19.625-1.875,19.625-4.188v-10C39.25,34.888,30.464,36.763,19.625,36.763z"/> +<linearGradient id="SVGID_2_" gradientUnits="userSpaceOnUse" x1="642.8008" y1="-973.1445" x2="682.0508" y2="-973.1445" gradientTransform="matrix(1 0 0 -1 -642.8008 -939.4756)"> + <stop offset="0" style="stop-color:#9CD7FF"/> + <stop offset="0.0039" style="stop-color:#9DD7FF"/> + <stop offset="0.2273" style="stop-color:#BDE5FF"/> + <stop offset="0.4138" style="stop-color:#D1EEFF"/> + <stop offset="0.5394" style="stop-color:#D9F1FF"/> + <stop offset="0.6155" style="stop-color:#D5EFFE"/> + <stop offset="0.6891" style="stop-color:#C9E7FA"/> + <stop offset="0.7617" style="stop-color:#B6DAF3"/> + <stop offset="0.8337" style="stop-color:#9AC8EA"/> + <stop offset="0.9052" style="stop-color:#77B0DD"/> + <stop offset="0.9754" style="stop-color:#4D94CF"/> + <stop offset="1" style="stop-color:#3C89C9"/> +</linearGradient> +<path fill="url(#SVGID_2_)" d="M19.625,36.763c10.839,0,19.625-1.875,19.625-4.188l-1.229-2c0,2.168-8.235,3.927-18.396,3.927 + c-9.481,0-17.396-1.959-18.396-3.927l-1.229,2C0,34.888,8.787,36.763,19.625,36.763z"/> +<path fill="#3C89C9" d="M19.625,26.468c10.16,0,19.625,2.775,19.625,2.775c-0.375,2.721-5.367,5.438-19.554,5.438 + c-12.125,0-18.467-2.484-19.541-4.918C-0.127,29.125,9.465,26.468,19.625,26.468z"/> +<linearGradient id="SVGID_3_" gradientUnits="userSpaceOnUse" x1="642.8008" y1="-965.6948" x2="682.0508" y2="-965.6948" gradientTransform="matrix(1 0 0 -1 -642.8008 -939.4756)"> + <stop offset="0" style="stop-color:#3C89C9"/> + <stop offset="0.1482" style="stop-color:#60A6DD"/> + <stop offset="0.3113" style="stop-color:#81C1F0"/> + <stop offset="0.4476" style="stop-color:#95D1FB"/> + <stop offset="0.5394" style="stop-color:#9CD7FF"/> + <stop offset="0.636" style="stop-color:#98D4FD"/> + <stop offset="0.7293" style="stop-color:#8DCAF6"/> + <stop offset="0.8214" style="stop-color:#79BBEB"/> + <stop offset="0.912" style="stop-color:#5EA5DC"/> + <stop offset="1" style="stop-color:#3C89C9"/> +</linearGradient> +<path fill="url(#SVGID_3_)" d="M19.625,23.313C8.787,23.313,0,21.438,0,19.125v10c0,2.313,8.787,4.188,19.625,4.188 + c10.839,0,19.625-1.875,19.625-4.188v-10C39.25,21.438,30.464,23.313,19.625,23.313z"/> +<linearGradient id="SVGID_4_" gradientUnits="userSpaceOnUse" x1="642.8008" y1="-959.6948" x2="682.0508" y2="-959.6948" gradientTransform="matrix(1 0 0 -1 -642.8008 -939.4756)"> + <stop offset="0" style="stop-color:#9CD7FF"/> + <stop offset="0.0039" style="stop-color:#9DD7FF"/> + <stop offset="0.2273" style="stop-color:#BDE5FF"/> + <stop offset="0.4138" style="stop-color:#D1EEFF"/> + <stop offset="0.5394" style="stop-color:#D9F1FF"/> + <stop offset="0.6155" style="stop-color:#D5EFFE"/> + <stop offset="0.6891" style="stop-color:#C9E7FA"/> + <stop offset="0.7617" style="stop-color:#B6DAF3"/> + <stop offset="0.8337" style="stop-color:#9AC8EA"/> + <stop offset="0.9052" style="stop-color:#77B0DD"/> + <stop offset="0.9754" style="stop-color:#4D94CF"/> + <stop offset="1" style="stop-color:#3C89C9"/> +</linearGradient> +<path fill="url(#SVGID_4_)" d="M19.625,23.313c10.839,0,19.625-1.875,19.625-4.188l-1.229-2c0,2.168-8.235,3.926-18.396,3.926 + c-9.481,0-17.396-1.959-18.396-3.926l-1.229,2C0,21.438,8.787,23.313,19.625,23.313z"/> +<path fill="#3C89C9" d="M19.476,13.019c10.161,0,19.625,2.775,19.625,2.775c-0.375,2.721-5.367,5.438-19.555,5.438 + c-12.125,0-18.467-2.485-19.541-4.918C-0.277,15.674,9.316,13.019,19.476,13.019z"/> +<linearGradient id="SVGID_5_" gradientUnits="userSpaceOnUse" x1="642.8008" y1="-952.4946" x2="682.0508" y2="-952.4946" gradientTransform="matrix(1 0 0 -1 -642.8008 -939.4756)"> + <stop offset="0" style="stop-color:#3C89C9"/> + <stop offset="0.1482" style="stop-color:#60A6DD"/> + <stop offset="0.3113" style="stop-color:#81C1F0"/> + <stop offset="0.4476" style="stop-color:#95D1FB"/> + <stop offset="0.5394" style="stop-color:#9CD7FF"/> + <stop offset="0.636" style="stop-color:#98D4FD"/> + <stop offset="0.7293" style="stop-color:#8DCAF6"/> + <stop offset="0.8214" style="stop-color:#79BBEB"/> + <stop offset="0.912" style="stop-color:#5EA5DC"/> + <stop offset="1" style="stop-color:#3C89C9"/> +</linearGradient> +<path fill="url(#SVGID_5_)" d="M19.625,10.113C8.787,10.113,0,8.238,0,5.925v10c0,2.313,8.787,4.188,19.625,4.188 + c10.839,0,19.625-1.875,19.625-4.188v-10C39.25,8.238,30.464,10.113,19.625,10.113z"/> +<linearGradient id="SVGID_6_" gradientUnits="userSpaceOnUse" x1="642.8008" y1="-946.4946" x2="682.0508" y2="-946.4946" gradientTransform="matrix(1 0 0 -1 -642.8008 -939.4756)"> + <stop offset="0" style="stop-color:#9CD7FF"/> + <stop offset="0.0039" style="stop-color:#9DD7FF"/> + <stop offset="0.2273" style="stop-color:#BDE5FF"/> + <stop offset="0.4138" style="stop-color:#D1EEFF"/> + <stop offset="0.5394" style="stop-color:#D9F1FF"/> + <stop offset="0.6155" style="stop-color:#D5EFFE"/> + <stop offset="0.6891" style="stop-color:#C9E7FA"/> + <stop offset="0.7617" style="stop-color:#B6DAF3"/> + <stop offset="0.8337" style="stop-color:#9AC8EA"/> + <stop offset="0.9052" style="stop-color:#77B0DD"/> + <stop offset="0.9754" style="stop-color:#4D94CF"/> + <stop offset="1" style="stop-color:#3C89C9"/> +</linearGradient> +<path fill="url(#SVGID_6_)" d="M19.625,10.113c10.839,0,19.625-1.875,19.625-4.188l-1.229-2c0,2.168-8.235,3.926-18.396,3.926 + c-9.481,0-17.396-1.959-18.396-3.926L0,5.925C0,8.238,8.787,10.113,19.625,10.113z"/> +<linearGradient id="SVGID_7_" gradientUnits="userSpaceOnUse" x1="644.0293" y1="-943.4014" x2="680.8223" y2="-943.4014" gradientTransform="matrix(1 0 0 -1 -642.8008 -939.4756)"> + <stop offset="0" style="stop-color:#9CD7FF"/> + <stop offset="1" style="stop-color:#3C89C9"/> +</linearGradient> +<ellipse fill="url(#SVGID_7_)" cx="19.625" cy="3.926" rx="18.396" ry="3.926"/> +<path opacity="0.24" fill="#FFFFFF" enable-background="new " d="M31.04,45.982c0,0-4.354,0.664-7.29,0.781 + c-3.125,0.125-8.952,0-8.952,0l-2.384-10.292l0.044-2.108l-1.251-1.154L9.789,23.024l-0.082-0.119L9.5,20.529l-1.65-1.254 + L5.329,8.793c0,0,4.213,0.903,7.234,1.07s8.375,0.25,8.375,0.25l3,9.875l-0.25,1.313l1.063,2.168l2.312,9.645l-0.521,1.416 + l1.46,1.834L31.04,45.982z"/> +</svg> + + <?xml version="1.0" encoding="utf-8"?> +<svg version="1.1" + xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" + x="0px" y="0px" width="68px" height="46px" viewBox="-0.064 -0.075 68 46" enable-background="new -0.064 -0.075 68 46" + xml:space="preserve"> +<defs> +</defs> +<radialGradient id="SVGID_1_" cx="478.8413" cy="1991.2729" r="21.6001" gradientTransform="matrix(1.15 0 0 1 -526.6598 -1982.4023)" gradientUnits="userSpaceOnUse"> + <stop offset="0" style="stop-color:#F2F2F2"/> + <stop offset="1" style="stop-color:#8D8D8D"/> +</radialGradient> +<path fill="url(#SVGID_1_)" d="M10.263,1.903c0-0.987,0.807-1.794,1.794-1.794h43.279c0.986,0,1.795,0.807,1.795,1.794v26.978 + c0,0.987-0.809,1.794-1.795,1.794h-43.28c-0.987,0-1.794-0.807-1.794-1.794L10.263,1.903L10.263,1.903z"/> +<path display="none" fill="none" stroke="#3C89C9" stroke-width="0.2185" stroke-linecap="round" stroke-linejoin="round" stroke-miterlimit="10" d=" + M10.263,1.903c0-0.987,0.807-1.794,1.794-1.794h43.279c0.986,0,1.795,0.807,1.795,1.794v26.978c0,0.987-0.809,1.794-1.795,1.794 + h-43.28c-0.987,0-1.794-0.807-1.794-1.794L10.263,1.903L10.263,1.903z"/> +<radialGradient id="SVGID_2_" cx="455.894" cy="1983.9624" r="47.8462" fx="493.5344" fy="1977.5143" gradientTransform="matrix(1.1935 0 0 1 -509.6731 -1982.4023)" gradientUnits="userSpaceOnUse"> + <stop offset="0" style="stop-color:#4D4D4D"/> + <stop offset="1" style="stop-color:#999999"/> +</radialGradient> +<path fill="url(#SVGID_2_)" d="M11.18,2.819c0-0.987,0.807-1.794,1.794-1.794h41.649c0.986,0,1.795,0.807,1.795,1.794v24.943 + c0,0.985-0.809,1.794-1.795,1.794H12.974c-0.987,0-1.794-0.809-1.794-1.794V2.819z"/> +<radialGradient id="SVGID_3_" cx="456.8843" cy="1984.0386" r="30.6699" gradientTransform="matrix(1.1923 0 0 1 -510.1314 -1982.4023)" gradientUnits="userSpaceOnUse"> + <stop offset="0" style="stop-color:#9CD7FF"/> + <stop offset="1" style="stop-color:#3C89C9"/> +</radialGradient> +<path fill="url(#SVGID_3_)" d="M11.689,3.228c0-0.987,0.807-1.794,1.794-1.794h40.633c0.986,0,1.795,0.807,1.795,1.794v24.126 + c0,0.986-0.809,1.794-1.795,1.794H13.483c-0.987,0-1.794-0.809-1.794-1.794V3.228z"/> +<path opacity="0.24" fill="#F2F2F2" d="M11.689,21.472V3.228c0-0.987,0.807-1.794,1.794-1.794h40.633 + c0.986,0,1.795,0.807,1.795,1.794v9.454c0,0.987-10.518,5.21-18.256,6.795C29.917,21.062,11.689,21.472,11.689,21.472z"/> +<linearGradient id="SVGID_4_" gradientUnits="userSpaceOnUse" x1="131.3501" y1="-212.2612" x2="131.3501" y2="-170.6274" gradientTransform="matrix(1 0 0 -1 -97.6001 -165.0498)"> + <stop offset="0" style="stop-color:#4D4D4D"/> + <stop offset="0.0667" style="stop-color:#717171"/> + <stop offset="0.069" style="stop-color:#757575"/> + <stop offset="0.0831" style="stop-color:#8C8C8C"/> + <stop offset="0.0996" style="stop-color:#9E9E9E"/> + <stop offset="0.1196" style="stop-color:#AAAAAA"/> + <stop offset="0.1466" style="stop-color:#B1B1B1"/> + <stop offset="0.2121" style="stop-color:#B3B3B3"/> + <stop offset="1" style="stop-color:#C5C5C5"/> +</linearGradient> +<path fill="url(#SVGID_4_)" d="M58.385,32.234c-0.689-0.856-2.154-1.559-3.254-1.559H12c-1.1,0-2.552,0.711-3.227,1.579 + l-7.546,9.717C0,43.55,0.042,43.785,0,44.55v0.125c0.056,0.393,0.311,1.002,1.248,1.002h64.75c0.75,0.002,1.5-0.252,1.5-1.002V44.55 + c0-0.813,0-0.813-1.254-2.559L58.385,32.234z"/> +<path opacity="0.23" fill="#F2F2F2" enable-background="new " d="M58.385,32.234c-0.689-0.856-2.154-1.559-3.254-1.559H12 + c-1.1,0-2.552,0.711-3.227,1.579l-7.055,9.085l58.098-7.33L58.385,32.234z"/> +<path fill="#6E6E6E" d="M59.75,37.205c0.344,0.431,0.172,0.783-0.377,0.783H8.249c-0.55,0-0.742-0.369-0.427-0.819l2.786-3.986 + c0.315-0.45,1.023-0.819,1.573-0.819h42.729c0.551,0,1.279,0.354,1.623,0.783L59.75,37.205z"/> +<path fill="#6E6E6E" d="M40.43,41.906c0.072,0.217-0.057,0.395-0.285,0.395H26.727c-0.229,0-0.364-0.18-0.3-0.398l0.822-2.826 + c0.064-0.221,0.303-0.399,0.532-0.399h11.167c0.229,0,0.475,0.179,0.547,0.396L40.43,41.906z"/> +</svg> + + <?xml version="1.0" encoding="utf-8"?> +<svg version="1.1" + xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" + x="0px" y="0px" width="36px" height="57px" viewBox="0 -0.741 36 57" enable-background="new 0 -0.741 36 57" + xml:space="preserve"> +<defs> +</defs> +<linearGradient id="SVGID_1_" gradientUnits="userSpaceOnUse" x1="230.1768" y1="798.6021" x2="180.3346" y2="798.6021" gradientTransform="matrix(1 0 0 1 -195.2002 -770.8008)"> + <stop offset="0" style="stop-color:#4D4D4D"/> + <stop offset="1" style="stop-color:#8D8D8D"/> +</linearGradient> +<rect y="0.943" fill="url(#SVGID_1_)" width="34.977" height="53.716"/> +<linearGradient id="SVGID_2_" gradientUnits="userSpaceOnUse" x1="224.6807" y1="798.6021" x2="200.6973" y2="798.6021" gradientTransform="matrix(1 0 0 1 -195.2002 -770.8008)"> + <stop offset="0.0319" style="stop-color:#848484"/> + <stop offset="0.1202" style="stop-color:#8C8C8C"/> + <stop offset="0.308" style="stop-color:#969696"/> + <stop offset="0.5394" style="stop-color:#999999"/> + <stop offset="0.5501" style="stop-color:#9C9C9C"/> + <stop offset="0.6256" style="stop-color:#B0B0B0"/> + <stop offset="0.7118" style="stop-color:#BEBEBE"/> + <stop offset="0.8178" style="stop-color:#C7C7C7"/> + <stop offset="1" style="stop-color:#C9C9C9"/> +</linearGradient> +<path fill="url(#SVGID_2_)" d="M5.497,0.943c7.945-1.258,16.04-1.258,23.983,0c0,17.905,0,35.811,0,53.716 + c-7.943,1.258-16.039,1.258-23.983,0C5.497,36.753,5.497,18.848,5.497,0.943z"/> +<path fill="#515151" d="M5.497,14.621c7.995,0,15.989,0,23.983,0c0,13.346,0,26.693,0,40.037c-7.943,1.258-16.039,1.258-23.983,0 + C5.497,41.314,5.497,27.967,5.497,14.621z"/> +<path opacity="0.43" fill="#565656" d="M5.497,4.745c7.982-0.628,16.001-0.628,23.983,0c0,2.707,0,5.413,0,8.12 + c-7.994,0-15.989,0-23.983,0C5.497,10.158,5.497,7.452,5.497,4.745z"/> +<path opacity="0.43" fill="none" stroke="#4D4D4D" stroke-width="0.0999" stroke-miterlimit="10" d="M5.497,4.745 + c7.982-0.628,16.001-0.628,23.983,0c0,2.707,0,5.413,0,8.12c-7.994,0-15.989,0-23.983,0C5.497,10.158,5.497,7.452,5.497,4.745z"/> +<polygon opacity="0.43" fill="#565656" stroke="#4D4D4D" stroke-width="0.0135" stroke-miterlimit="10" enable-background="new " points=" + 6.496,5.746 9.869,5.606 9.869,6.661 6.496,6.799 "/> +<rect x="31.307" y="2.517" fill="#E7ED00" stroke="#717171" stroke-width="0.1926" stroke-miterlimit="10" width="3.692" height="1.505"/> +<rect x="31.307" y="5.8" fill="#C8FF00" stroke="#717171" stroke-width="0.1926" stroke-miterlimit="10" width="3.692" height="1.507"/> +<linearGradient id="SVGID_3_" gradientUnits="userSpaceOnUse" x1="29.4414" y1="35.1235" x2="5.4995" y2="35.1235"> + <stop offset="0" style="stop-color:#808080"/> + <stop offset="0.1907" style="stop-color:#828282"/> + <stop offset="0.2955" style="stop-color:#8A8A8A"/> + <stop offset="0.3795" style="stop-color:#989898"/> + <stop offset="0.4524" style="stop-color:#ACACAC"/> + <stop offset="0.5175" style="stop-color:#C5C5C5"/> + <stop offset="0.5273" style="stop-color:#C9C9C9"/> + <stop offset="0.5914" style="stop-color:#C9C9C9"/> + <stop offset="0.9681" style="stop-color:#C9C9C9"/> +</linearGradient> +<path fill="url(#SVGID_3_)" d="M5.5,14.822c0,13.22,0,26.438,0,39.66c7.931,1.256,16.012,1.256,23.941,0c0-13.222,0-26.439,0-39.66 + C21.461,14.822,13.48,14.822,5.5,14.822z M28.396,18.703c-0.74,0.01-1.482,0.02-2.225,0.029c0-0.951,0-1.901-0.001-2.85 + c0.742-0.003,1.483-0.005,2.224-0.008C28.396,16.817,28.396,17.76,28.396,18.703z M16.354,42.496c0-0.961,0-1.924,0-2.885 + c0.744,0.006,1.489,0.006,2.233,0c0,0.961,0,1.924,0,2.885C17.843,42.503,17.098,42.503,16.354,42.496z M18.587,43.568 + c0,0.955,0,1.91,0,2.866c-0.744,0.009-1.489,0.009-2.234,0c0-0.956,0-1.911,0-2.866C17.098,43.574,17.843,43.574,18.587,43.568z + M18.586,27.742c0,0.961,0,1.922,0,2.886c-0.744,0.004-1.488,0.004-2.231,0c0-0.964,0-1.925,0-2.886 + C17.099,27.746,17.842,27.746,18.586,27.742z M16.354,26.671c0-0.955,0-1.91,0-2.865c0.743,0.002,1.487,0.002,2.23,0 + c0,0.955,0,1.91,0,2.865C17.842,26.675,17.099,26.675,16.354,26.671z M16.354,34.583c0-0.961,0-1.924,0-2.885 + c0.744,0.004,1.488,0.004,2.231,0c0,0.961,0,1.924,0,2.885C17.842,34.588,17.099,34.588,16.354,34.583z M18.586,35.656 + c0,0.961,0,1.924,0.001,2.885c-0.745,0.008-1.489,0.008-2.233,0c0-0.961,0-1.924,0-2.885C17.099,35.66,17.842,35.66,18.586,35.656z + M15.307,30.619c-0.742-0.01-1.484-0.021-2.227-0.039c0-0.957,0-1.916,0-2.875c0.742,0.014,1.485,0.023,2.226,0.029 + C15.307,28.695,15.307,29.656,15.307,30.619z M15.307,31.689c0,0.961,0,1.924,0,2.885c-0.742-0.012-1.485-0.025-2.227-0.047 + c0-0.959,0.001-1.92,0.001-2.877C13.822,31.667,14.565,31.68,15.307,31.689z M15.307,35.644c0,0.959,0,1.922-0.001,2.883 + c-0.742-0.012-1.485-0.031-2.228-0.056c0-0.959,0.001-1.918,0.001-2.877C13.821,35.617,14.564,35.633,15.307,35.644z M15.306,39.597 + c0,0.96,0,1.922,0,2.883c-0.742-0.016-1.486-0.037-2.228-0.064c0-0.959,0-1.916,0.001-2.877 + C13.82,39.564,14.563,39.585,15.306,39.597z M19.637,39.597c0.742-0.012,1.484-0.033,2.227-0.059c0,0.959,0,1.918,0,2.875 + c-0.741,0.029-1.483,0.052-2.227,0.064C19.637,41.519,19.637,40.559,19.637,39.597z M19.637,38.527c0-0.961,0-1.924,0-2.883 + c0.74-0.012,1.482-0.027,2.225-0.05c0,0.959,0,1.918,0.002,2.876C21.121,38.496,20.377,38.515,19.637,38.527z M19.637,34.572 + c0-0.961,0-1.922-0.002-2.883c0.741-0.01,1.483-0.021,2.225-0.039c0.002,0.957,0.002,1.916,0.002,2.875 + C21.119,34.547,20.376,34.564,19.637,34.572z M19.635,30.619c0-0.963,0-1.924,0-2.885c0.74-0.006,1.483-0.017,2.225-0.029 + c0,0.959,0,1.916,0,2.875C21.118,30.599,20.376,30.609,19.635,30.619z M19.633,26.666c0-0.955,0-1.909,0-2.864 + c0.741-0.005,1.483-0.013,2.227-0.021c0,0.951,0,1.903,0,2.856C21.118,26.65,20.375,26.66,19.633,26.666z M19.633,22.732 + c-0.001-0.963-0.001-1.924-0.001-2.885c0.741-0.002,1.483-0.006,2.226-0.012c0,0.959,0.002,1.918,0.002,2.877 + C21.116,22.72,20.374,22.728,19.633,22.732z M18.586,22.736c-0.744,0.002-1.487,0.002-2.23,0c0-0.963,0-1.924,0-2.887 + c0.743,0.002,1.487,0.002,2.23,0C18.586,20.813,18.586,21.773,18.586,22.736z M15.309,22.732c-0.742-0.004-1.483-0.012-2.226-0.02 + c0-0.959,0.001-1.918,0.001-2.877c0.742,0.006,1.484,0.01,2.226,0.012C15.31,20.808,15.309,21.769,15.309,22.732z M15.309,23.801 + c0,0.955,0,1.91,0,2.864c-0.742-0.006-1.483-0.016-2.227-0.027c0-0.953,0-1.906,0-2.859C13.825,23.789,14.566,23.796,15.309,23.801z + M12.036,26.617c-0.742-0.017-1.483-0.033-2.225-0.055c0-0.947,0-1.895,0.001-2.841c0.741,0.019,1.483,0.031,2.225,0.042 + C12.037,24.716,12.036,25.666,12.036,26.617z M12.035,27.683c0,0.957,0,1.916,0,2.873c-0.742-0.021-1.483-0.047-2.225-0.076 + c0-0.953,0-1.904,0-2.857C10.552,27.646,11.293,27.667,12.035,27.683z M12.035,31.621c0,0.957-0.001,1.914-0.001,2.871 + c-0.742-0.023-1.483-0.055-2.224-0.092c0-0.953,0-1.906,0-2.859C10.551,31.572,11.292,31.6,12.035,31.621z M12.033,35.56 + c0,0.956-0.001,1.914-0.001,2.871c-0.742-0.031-1.484-0.066-2.225-0.111c0-0.953,0.001-1.906,0.001-2.858 + C10.549,35.5,11.291,35.533,12.033,35.56z M12.031,39.498c0,0.955,0,1.914-0.001,2.869c-0.742-0.035-1.484-0.078-2.225-0.129 + c0-0.953,0-1.904,0.001-2.857C10.547,39.426,11.289,39.465,12.031,39.498z M12.03,43.435c0,0.951-0.001,1.901-0.001,2.854 + c-0.742-0.041-1.484-0.09-2.225-0.149c0-0.944,0.001-1.892,0.001-2.838C10.546,43.353,11.288,43.4,12.03,43.435z M13.077,43.482 + c0.743,0.031,1.486,0.053,2.228,0.067c0,0.956,0,1.91,0,2.864c-0.742-0.016-1.486-0.041-2.229-0.074 + C13.077,45.389,13.077,44.435,13.077,43.482z M15.305,47.486c0,0.961,0,1.922,0,2.883c-0.743-0.019-1.487-0.047-2.23-0.084 + c0-0.959,0-1.918,0.001-2.875C13.818,47.443,14.562,47.468,15.305,47.486z M16.353,47.504c0.745,0.009,1.49,0.009,2.234,0 + c0.001,0.96,0.001,1.924,0.001,2.883c-0.745,0.011-1.49,0.011-2.235,0C16.353,49.427,16.353,48.464,16.353,47.504z M19.639,47.486 + c0.741-0.018,1.483-0.043,2.227-0.076c0,0.957,0.002,1.916,0.002,2.875c-0.742,0.037-1.486,0.065-2.229,0.084 + C19.639,49.406,19.639,48.447,19.639,47.486z M19.637,46.414c0-0.954,0-1.908,0-2.864c0.742-0.015,1.484-0.036,2.229-0.067 + c0,0.953,0,1.905,0,2.857C21.122,46.373,20.379,46.398,19.637,46.414z M22.911,43.435c0.741-0.035,1.483-0.082,2.224-0.135 + c0,0.945,0,1.895,0.002,2.838c-0.74,0.059-1.482,0.107-2.226,0.15C22.911,45.336,22.911,44.386,22.911,43.435z M22.911,42.369 + c-0.001-0.957-0.001-1.914-0.002-2.871c0.741-0.032,1.483-0.069,2.225-0.117c0,0.954,0.001,1.906,0.001,2.857 + C24.395,42.289,23.652,42.333,22.911,42.369z M22.909,38.431c0-0.957-0.001-1.915-0.001-2.871c0.742-0.027,1.482-0.061,2.224-0.098 + c0.001,0.951,0.001,1.904,0.001,2.857C24.393,38.363,23.65,38.4,22.909,38.431z M22.908,34.494c0-0.957-0.002-1.916-0.002-2.871 + c0.742-0.021,1.482-0.051,2.225-0.079c0,0.952,0,1.903,0.001,2.856C24.391,34.437,23.648,34.468,22.908,34.494z M22.906,30.556 + c0-0.957,0-1.916-0.002-2.873c0.742-0.016,1.484-0.037,2.226-0.061c0,0.953,0.001,1.904,0.001,2.857 + C24.391,30.509,23.648,30.535,22.906,30.556z M22.904,26.617c0-0.951,0-1.901,0-2.854c0.74-0.011,1.482-0.025,2.224-0.042 + c0,0.946,0.001,1.894,0.001,2.841C24.389,26.583,23.646,26.601,22.904,26.617z M22.902,22.699c0-0.957,0-1.916,0-2.874 + c0.742-0.007,1.482-0.014,2.225-0.023c0.001,0.953,0.001,1.906,0.001,2.859C24.387,22.676,23.646,22.689,22.902,22.699z + M22.902,18.76C22.9,17.802,22.9,16.845,22.9,15.887c0.742,0,1.481-0.003,2.225-0.004c0.001,0.953,0.001,1.906,0.002,2.858 + C24.385,18.75,23.643,18.756,22.902,18.76z M21.855,18.767c-0.742,0.004-1.482,0.007-2.225,0.009c0-0.961,0-1.922,0-2.884 + c0.741,0,1.482-0.001,2.225-0.002C21.855,16.849,21.855,17.808,21.855,18.767z M18.585,18.779c-0.743,0.001-1.486,0.001-2.229,0 + c0-0.961,0-1.923,0-2.885c0.742,0,1.486,0,2.229,0C18.585,16.855,18.585,17.817,18.585,18.779z M15.31,18.777 + c-0.742-0.002-1.483-0.005-2.225-0.009c0-0.959,0-1.918,0-2.877c0.742,0,1.483,0.001,2.225,0.002 + C15.31,16.854,15.31,17.815,15.31,18.777z M12.039,18.76c-0.742-0.005-1.483-0.011-2.225-0.019c0-0.953,0-1.905,0.001-2.858 + c0.742,0.001,1.483,0.004,2.224,0.004C12.039,16.845,12.039,17.803,12.039,18.76z M12.039,19.827c0,0.957-0.001,1.915-0.001,2.872 + c-0.741-0.01-1.483-0.021-2.224-0.035c0-0.953,0-1.906,0-2.859C10.555,19.813,11.296,19.819,12.039,19.827z M8.768,22.64 + c-0.741-0.018-1.482-0.035-2.223-0.057c0-0.943,0-1.887,0-2.831c0.741,0.013,1.482,0.025,2.223,0.036 + C8.768,20.739,8.768,21.689,8.768,22.64z M8.767,23.697c0,0.944,0,1.89,0,2.832c-0.741-0.024-1.482-0.053-2.223-0.084 + c0-0.938,0-1.873,0-2.811C7.284,23.658,8.026,23.679,8.767,23.697z M8.766,27.587c0,0.949-0.001,1.898-0.001,2.85 + c-0.74-0.033-1.481-0.068-2.222-0.111c0-0.942,0-1.887,0-2.83C7.284,27.529,8.025,27.56,8.766,27.587z M8.765,31.494 + c0,0.951-0.001,1.9-0.001,2.852c-0.74-0.04-1.481-0.087-2.221-0.139c0-0.943,0-1.887,0-2.831C7.283,31.42,8.023,31.459,8.765,31.494 + z M8.763,35.404c0,0.949,0,1.899,0,2.851c-0.741-0.052-1.481-0.104-2.22-0.168c0-0.942,0-1.886,0-2.829 + C7.282,35.31,8.022,35.361,8.763,35.404z M8.762,39.312c0,0.949,0,1.899-0.001,2.852c-0.741-0.059-1.48-0.123-2.219-0.195 + c0-0.943,0-1.889,0-2.83C7.281,39.203,8.021,39.26,8.762,39.312z M8.76,43.219c0,0.944,0,1.888-0.001,2.832 + c-0.74-0.065-1.479-0.14-2.218-0.224c0-0.938,0-1.875,0-2.812C7.281,43.092,8.02,43.16,8.76,43.219z M8.759,47.109 + c0,0.951,0,1.9,0,2.851c-0.741-0.073-1.48-0.158-2.219-0.253c0-0.942,0-1.887,0-2.828C7.279,46.964,8.019,47.039,8.759,47.109z + M9.804,47.201c0.741,0.06,1.483,0.111,2.224,0.154c0,0.955,0,1.912,0,2.868c-0.742-0.045-1.484-0.103-2.225-0.166 + C9.804,49.107,9.804,48.154,9.804,47.201z M12.027,51.291c0,0.957,0,1.916,0,2.873c-0.742-0.053-1.484-0.114-2.225-0.188 + c0-0.951,0.001-1.904,0.001-2.857C10.544,51.187,11.285,51.244,12.027,51.291z M13.075,51.353c0.743,0.039,1.486,0.067,2.229,0.086 + c0,0.961,0,1.922,0,2.885c-0.743-0.021-1.487-0.053-2.229-0.094C13.075,53.269,13.075,52.312,13.075,51.353z M16.353,51.459 + c0.745,0.009,1.49,0.009,2.235,0c0,0.961,0,1.924,0,2.885c-0.745,0.013-1.491,0.013-2.235,0 + C16.353,53.382,16.353,52.42,16.353,51.459z M19.639,51.439c0.741-0.019,1.485-0.049,2.229-0.086c0,0.959,0,1.92,0.001,2.877 + c-0.743,0.041-1.485,0.072-2.229,0.094C19.639,53.361,19.639,52.4,19.639,51.439z M22.913,51.291 + c0.743-0.047,1.483-0.104,2.226-0.172c0,0.953,0,1.906,0,2.857c-0.74,0.073-1.481,0.135-2.224,0.188 + C22.914,53.205,22.914,52.248,22.913,51.291z M22.913,50.224c-0.001-0.956-0.001-1.912-0.001-2.869 + c0.742-0.043,1.484-0.095,2.225-0.154c0,0.953,0,1.906,0.002,2.857C24.396,50.123,23.654,50.179,22.913,50.224z M26.184,47.109 + c0.739-0.066,1.479-0.145,2.217-0.229c0,0.942,0,1.887,0,2.83c-0.736,0.092-1.478,0.177-2.217,0.252 + C26.184,49.009,26.184,48.06,26.184,47.109z M26.184,46.051c-0.002-0.944-0.002-1.888-0.002-2.832 + c0.739-0.06,1.48-0.127,2.219-0.202c0,0.938,0,1.873,0,2.811C27.662,45.912,26.923,45.986,26.184,46.051z M26.182,42.162 + c0-0.95-0.002-1.9-0.002-2.85c0.74-0.052,1.48-0.109,2.219-0.176c0.002,0.943,0.002,1.887,0.002,2.83 + C27.662,42.039,26.921,42.105,26.182,42.162z M26.18,38.253c0-0.95,0-1.9-0.002-2.852c0.742-0.041,1.482-0.093,2.221-0.146 + c0,0.942,0,1.887,0,2.829C27.66,38.15,26.92,38.203,26.18,38.253z M26.178,34.345c0-0.949,0-1.898,0-2.852 + c0.74-0.034,1.481-0.073,2.221-0.117c0,0.943,0,1.887,0,2.83C27.659,34.258,26.918,34.305,26.178,34.345z M26.177,30.437 + c0-0.949,0-1.9-0.001-2.85c0.741-0.027,1.481-0.059,2.221-0.092c0,0.943,0.002,1.888,0.002,2.83 + C27.659,30.367,26.918,30.404,26.177,30.437z M26.176,26.529c-0.001-0.942-0.001-1.888-0.001-2.832 + c0.742-0.018,1.482-0.039,2.222-0.063c0,0.938,0,1.873,0,2.811C27.657,26.476,26.917,26.503,26.176,26.529z M26.174,22.64 + c0-0.951-0.001-1.901-0.001-2.851c0.741-0.01,1.483-0.022,2.224-0.035c0,0.943,0,1.886,0,2.831 + C27.657,22.605,26.915,22.623,26.174,22.64z M8.769,15.881c0,0.95,0,1.9-0.001,2.85c-0.741-0.008-1.482-0.018-2.223-0.028 + c0-0.943,0-1.887,0-2.83C7.286,15.876,8.028,15.878,8.769,15.881z M6.54,50.758c0.738,0.097,1.478,0.183,2.218,0.258 + c0,0.95,0,1.901,0,2.853c-0.741-0.084-1.48-0.178-2.218-0.28C6.54,52.646,6.54,51.701,6.54,50.758z M26.184,53.869 + c0-0.95,0-1.899,0-2.853c0.739-0.075,1.479-0.163,2.217-0.259c0.002,0.941,0.002,1.889,0.002,2.83 + C27.663,53.693,26.925,53.785,26.184,53.869z"/> +<path id="highlight_2_" opacity="0.17" fill="#FFFFFF" enable-background="new " d="M0,0.943h5.497c0,0,6.847-0.943,11.974-0.943 + C22.6,0,29.48,0.943,29.48,0.943h5.496v41.951c0,0-12.076-0.521-18.623-2.548C9.807,38.32,0,30.557,0,30.557V0.943z"/> +</svg> + + + + diff --git a/minifi-c2/c2-integration-test.png b/minifi-c2/c2-integration-test.png new file mode 100644 index 0000000000000000000000000000000000000000..404063e906525e4357e7eb955fcf54b4cf3e8e66 GIT binary patch literal 26942 zcmeFZWmJ^!_cuCz5kW~wC8R+Znjs_whZ?#&rMo*6qy?lK6zLecVGse49O*9U&Y|Pn ze1E_Hv!1u-oORYZFP>o*7fjsOHM{p`?>!O9iqbgG$ew{fARJj4s457ARsjP2LxlMc z@QBZc=vH7sbCr-)$Hc^(T~=BG{(b2tsqLobXyN8*;$jZ6ba1pcXLEh;Vs7r>YUSv5 zfYvSw0#Soxq2lUZnY;78P85^=92bjk=1+c-Kb8riMSX?BCa~H1W9<6*TvN_wn45Cv zUpMXR*qdr?MjLqal|;Mv8$`%yep6vjTB$au2W!4zDUIxds~#(sPE1D=2$7rT>sw}B z5Vr}?Mdn?AjHsXD$94>yp03=S#b&=80HGPb#SWJa6f5}F@wKqf(&^91-e|k4E+@|; z;P9YN@%jfi4}XBP`DHpER(!+%H|rxX+sK5EV^=is&lGGBC)Efi=#%P@d_<(iYKKOF zEST++Oj?tsR`}sF(4PQcBYe>}JV||N`X<^tFSu~%cui@S{{fXz<(YsmDf=m8&AFve zdf)dw4o`luPc=K1*QV~M;O^t?r|Shc#K8(Hl1Qv;MhNHWJ;>LcCdv+8Ie7hT))@3{ z&P*#gW7!x;ENU^BoEB~f?c?Dn@3<^Ne&^`jw&B`K96w)4#N> z2D0;iU<(^Jcjh;@_gmt@cI+wS#nD509pMcjYXod%eVm`|_((r3%j1W+PTQ5w3T5+7ojFP7QbNQHg8pX&G_{6OxQCN7}W-iR2X^VB{~4Uk~}as#st@L)b$9N zo@1-lyQ;4bzupQ$-;gVa4*k-)i?+>QQl+1puaO@Q;WkBQ=1-E>rK5wScqLc|+kk5a z;&erv6AGoMpM1c7NA~2yp=|nenTf^@KI?nK2HTO^(3W!v<49c4tJ zIy#jE8VMGB84z0(4%S2~eJ91x5K4z%Z#<|_j$9V@gt_6)!yc#ZXAHzVnSxsDX;-S+ zKOhMi&J!O;v+A@2n|^T>^=4@=8ijr8iqBa#N^K@r}1{kzUl>+mqJs9LaZV$_XD zg!e^XIWlj)=C#F>O=~h{>!3AtXetS@)xOAcmD8Nm*Ku(uclBhQu+1o)eO==bSi_{7 z;rZ3MG@Y$-7i%D`xH%<-M~#_h(u*oC$?zuRNAk~pYx9>N%<_U3%w2wCHoN0r$z0*T ze-HSlPz<^JN^Jo*dM-4$;uQ<$o-Oueq53&$LYu?*7Lb{gihRDg>$7~0CbV=ln}p@v zLY13>J0w=i=oVoKN%9B!B%xne1x#M^so>x(7>o7%pM-)mGOhvrAqZXUoX*{taXHz>B?RPsy_g9N00cSNiBFDe!+ONhd0&aJVs6>2D%NrI11MhBDSIwIA zafiM*1OO(U1Clbxn82Ld^z6f~6W@HYa3+i~LsZ2UtHA^3(i=%n{_$YP{3JhF| z-y0`uF4OgUx^I@UJQuuC+Zqi4|#pLWPo#4_<2 zlJO0g@GwB#sb^XpBYwY7Oqa-u2>*-Ebi`Y3wPX_)Wi~rqsXorpY1mPvb-|~!fm}-g zNLTend=T9(c?~b9atksqY25y({CeNBT@EUqY`*^O+c$L<2J`$c6{0vT@Ew7nR*3vK zy$A=+H6n0-u4#ID8jnJ_-E|`oUzlg~dKOQ(U6kWkP{-QjCHQIG4z*dyRA-s4H-NiP^i({{J#1AH zT}-H_1W$7zM<;gb?A|2nWHe4u(9pHjnZ6%_)adQ;cEl?)KLYiGn;{OR)_T#Tw=eYu z6UXhQIhsmVOE?ft%k6ia=0?m$RjPej6q8%aO!w_QfqtU!D0tDXoK@S?HV!xr3hJ;G z%S2o4QcdtR&85k2`6qhG4%?Z(*u}QDVR=}iCcpeRBIjyL()P9sn%IvkGmRWL^6EJd zZiH9Z?{JPJ63|LR6Dfu)>gAov^iAmBWZLWcW9t#KFp$YBDLtse&neaq=5|`V=+pwOZa?B9Jhy=dR{CLml_Rh3!07 z%kS>z5&qY?!<$zKCVbsd+^mcv4iYQ7i5%y|9p>)c%iT*@@lxqA(KSLfa%Z&C^h_it zEyCNG>`BM?^&QYxN-I zOI>o6phEPr;t~WAu+m)E$uqKKYUVtA37L{>YbQPY=UmJd`P$5yNd>>UqDZRyJO2R2X}&* zLx@yiD(CjMILzq0?5uf=kDt=Q^Ds25RYk%1GF5-6X&*~ljwf$uJ^A3&6Qjoj^Ok07 z?*Q3*HbKS6zdzM$%vO!mXhC8-qYi$u7u}zXmuejkQBltMs~MFIDxqcQu#!Y|&rHxO zwh4Zf^rMZDA~@~vC?__4_uu14VCGB8ZbDC*OT0QnT*XWY;g_TM|Zd zE#Ff<{IaCdU21%8SU~r?6?!_+&g>kWut?P zvEDgYM-8U`os?0wFuWz^!7?S#fAhJ7LXufLJ_6kt1%A#<60iNdx9>>9NnX>MbNrdR z1+518-teE=qM4bXj1I=Hi3AR{JP8iHSXVEg(usm}@u_yP$Mw~-RVh0P&?kbm_E~XO zP;g>f>z;QcyHd(*r48uAq4>Y7L=SqN(m@>q3w{=GPHXipC?ZHQJWbN+a~@F~QM`b_ zIQ8Rk+`%A|l%x_@3r+PVm0N2k2rs{ILh#01nZh%^y1<*YEdwDRj{^$)X7-RjkZ*F+ z#D#lV0|W6>CrK+;F@<(WPhk&z_nXt{tQHS&a*3`z$>_!4?yzV3PL_%SJuNrMa}{QH z1sR1#(P_95t8aHT)2HyA>J~8%(*d1>NgPn6HBCbq*~p<^P|?ii5)4%e3U-YlT+TSt zc&NMtSJZY@qH31}W=D`0{BLwgA+ZgHw)kkt*f+H`50p$)sia^A$_&lU1zr)5PJR+& z_{K?Y@uS`qh7TbtL|g*)?ddmLdRf@>;m2|`dDOBu9sU?@3eRZ8>A`_`yKTNLi6bY| z=ShWAsKk|;JsTJ{IlE z%zS)t2A{7UxF0ugYit@yYs@APn#i|Ks64bE(d(DHS9et>qn2=@W})1c>y;Lt_p`pH z{#nDRW4X=r;^{>6GF-AWd4KY#UqVe(rk8D0xof2I@=l3ugHjWB`s4pr#iz4gF!a<= zR;X((++c-?Pj#DC?aek!7B0`&6?V991%XEfZTa;*%`Gf&lP4kiez+2#CiC$xp&3Ql zEcmc5t6$FwJW{(OWmvwMxp%eEG66A45xWN`l%*9awOGn|PaVu_4G911r` z{XW4(F^8Y}o9ySKCiA2(p&O-lGqTU+3i?>-ltw1{p~ zG&Hmnb_14_{@dHz6b_?$d!bjcEpR=Sq)+$}P*V>NP3H0O@w&=_e~fp;9dtzUmP1(D zwPXh(wVyHKK64<7Q_nN&v=>kMmQE`$OzpK?=wIiSRV2ul&~yw*vTbu(GmdKV5Pnig zGnK8{umAC#I)?QZ>Thz(x%z~fG^utMr(2`h88?c;}T@q6@n2m z`5!rJXlaP~A&lax@w5b@Pi0xPot*4DyZe7{JaZ^%C{;&&fltg(7>p@h@yp%7f}Q9Ya18OL!+`;L2pLM4(SJFpEi#b6(=Wu7IL4 zYt=#p+LF}Uf2s0BKSPD1sK(^mYk}@RQfRmJPdh=KVi(+--0>+>vG1wQ#`}< zkTU*-K#|J`|qzYpW ze$Dy`6Ok+b&rcBGhrWc80@@XsPc%rLTvVIaCuZ7_Bu6)-U??)#+!#z1o|Qy)o~9 zK2OT7pPH7owX;)NK`!dPowyMrD<0)w#bGK0&W;Da8R!lrxfzo1tcP>XGM-R zK`ZW@qOr}>X??JP!(cEaO?EV|G_fIGB#WJeG+$4C8pU{$pC+-jzF?J7(C!iN>mr&` zuI$fA>-(MgZiTc!C5@Z&wU^14KK9$;!%m-NC%!uU!U#SHvc%1T{N+`s`Rw_DrEH9kTU0utM~TL zwP{tFdP}|B7CN$MtE%uS`U-qudC&5Ws62rZuecjbOYOgn)i&^8MZ2 z-Rqkv1j#j6RWfMY`B$t(ZUVP7B- z_|+;``FyJ0|LWA57RHzWgzM#l{KA5Qf}EUx#>X{E$zM@X<)ozzyU#mk8Y7myghfR} z_Ph-p9S^FN+|~!u3IR$3!7{bw6mKBe3|J$#}|=-FheMRO&0= z-73o*|FfIR6OPuypI98Nm5z%Dr2oEa&gFpR)zwuHx<%yB;_Z!_qt=c8HO9H!!E>Ka zGIE=`Mq2W8`w^;8m&%@#q+?;p0X3dWFIvMrV`^GcT8_qD3Rn5bPekyI7Ez&WBNz2v zRFt_Gt)!>^Xuhy>os)C5{_M}<;^M-bfUDxdY(ej1GbiNI5;Zq$>UDl(_urq*%fPeL z7Zcl|SRoR{>NdLkt*2@?t4w0v+r@>3Ud!mnQlh5774pT+?QPv_Qj9-}zuD|RLgwBw zule59ZV*$Z$Y9dsVKNrMYhRU|2KFx1Q`3|fm2QxyQ@(U|FEa1U^LNG2bJy}HhE!~C zgJz0LOLt}$@9IrnbTlw2W#5dXAWknWxbO`E^a3rEazgtBXVJc0S zZkA$gp=r=tDluKxZ`fP9`!*0&Y9xXhQxm&c#FR>+$eFiYhCilkoQ zs2zCwar;Qqegv^C`D15Bvy+R2hv%Lu@TC2IWAU!s-J6P9-EKDELH_|G00?@5NIS|X zB0NnvUteiWZa<=Gx`r^{3K|7H&G5->qx6x6?tI?LQh6C09?qzgCE$Vln8z5rk3j&9 z$>opDY2BI-`eo0hM{>#C@~S2Kz^CuXujfS0eCv(Z-v~qoZ zkdvOSd+{Am&Nl=RW#iz-+B|_bJ%OzUBKOw8tI6)(7&HG^H;QvI$EudmC;VL!WJ~H~ z73VIJ*s!7WIjrM0Fj^Bs-81Dj7sG7qOFZ@*HA`GtW-3$1=g>Ru3m5Ol@DTTH+1c6T z_o6>Y&TdYSlUJc=&2X{PKkss`=QeOD1y7B##bnh>?u$*&hb67rTrB;H)Q!!`_4Spzgsi``4jJnCF_*Qv*Oz&46%z~;Leoe zckFn7=4fjPN0W}Bc;k0*nBGcS$>n$6vY6$mRIDx-aMQTqec&_M64bmrG&E$sL7ttN zSvHOX&L`OyK4?7|I`BX61q5YvR7p&Pk5BXd`@1mQw$l+g+iSr`6F+P#+RpxRxRBE~ z*ezHyaBlerFKO+7HDQ;L{@n;9!qljE1|Yi;Gx6NvbcLV!qEK$6GgNMhNoY9r)-Yfu!RrsxenG`cCS#6iYn?w9G<|Q>tx@5Q# zzKo~#BT9&Jj5P|M?)Sp|V(ivwZ?T z&v>Nwhfqn_d41p|tsvmM;^?G-K>r+oZc#wMV0*MY1Y;jAdpfcj8TR5}x~2y*=(;f^ zaz5u8hD%ACdvtWvLlx*}Zu?Q8+d0$w{Pgs64p8j_-^FWyOTPO}mQGGi8$(%uytc0; z>j94m?KjdDkuOxemll3?u(|mHh*9_U#v%5Adl&ND*EfA|6?cc7c=v#2_^%~ti2?{O zM(~Ey9D2s{|0ujNOl6H!#M4;aS~Xygv-HkNcJTJM7NVjx^jBU`!AdD3PU(msn13C2 zDd>G!XblNy&uMq6>cJo%E_XV%9{mE;ZL~8Cui}&i4cs=P3^m+fmzqR{Q3YJ&rx^OI zMDsuF`CxS_5Leso%weKspK+!`SAU^Np=;(O({i=C+ z$^CQR(L>-4*3_~j@Bw};YP@sy2w`Ow+p?7k)ALz6szZ8 zC|KQ@BV?896vh-5v@V_fOLJ{Pe4HMoezo7)MMe0jlM|GFCIGnyJsnG~cbhS1bYx)a zbemRH66d7qCIEKdZ@<4w&LQEWYG)ydLT?eDbPq->n^tKXJsfgpgPc7HhH~!&0Ez64 z@=aYgUU6sKAB;i&QG=`NYhT~y1CJihQk^h2ahC<}zb6p?BX7dO!icpJ)gy#f^Ru690QK?hliW{O$NTF>c)2ZC=0gI=U>b>xHYtX ztTboNx104an5^I1-`~dzI1H}0H(K=ToNaI=PVG6}EZ> zmg$uEg81#W8?&K=oT&*JF}7S?m~RBVQ1{B#Vo$NOuwXYrlf5DNe|i;MeHI54jN>N%?TW%1PC2et>G&KKKl_Y8^b*atF^8-bVoM%&GIaL=|K zAZJG$oS&U3Dk;Ur#Nc9MySclMZ-@5u^Z=hGsGnC!K|%8SFks!&(q1l}J3oClHSsS???4*AvryvWxkID!>qJ=o zSBEKg1BKDl>2ob^QtBV;pXNrSGOYC`6|1ycot7oODGRu+4;-!Z$mNe5$LhRSg&|KI z+HcQJHjvV^aUVZ^G%n)(k$yEE(Ql5p$`F>}WMq+EX%4=WPW`EvVG$f2E=7S%TU{s2*~HEJfU9xKf|xlvsjXqj;DFB)DawbnG)y?#QqDZ=Z_||jEok00D>5+>$4<1 z6ZD~r`wirR&G4OOgzJVP?TkLvFAx||N5d%{`RFsr;_P%Wm;o&mO z?#HXL29qL4to304sF;P~VDT{<9bMXn@MtM{dO{Ogd=KX4Ys!V-*3Gmu=nT2KMwaR+ zWBF=OI&u2Pl=8UN>50%>UD|Q$ux2`I0s6gaWY-!{tBbC zaWYz3lZ;s{kNKV(Rc*L-c*JIBh&yC#RI)p8xsJ1|L&<3p6$2I6S7A54|4ccZw}Yxl7XG7c2f+H9<8d==S0`z%M=)R zul)9#Hn2O3+6(ppT;n{L>IP)+(?pGxB=(g^N#fO?FwxWX3`d{r$;HLMy_vevLZyS5 zIxtBTrHD^rny-%!$pnt4o7>|Pob&VZit`9sV|?ERl?gs@(@wVm}L{bd*$7 zj)L^AlxruK0AM6T?{f6F+e;V$)GUGD3*_h~-%>@oN3`gn3ug-mT|xdes5@J(?YsFn zcxr*IM1pGgN3fP3a|m_31SWI%?}KI(%2{(@VD;+nb{UIc-_g`rB61b1TGBg5geP+P zM+>MJa{$JoqajXab#wT;lJHSi&h2zg9k&j*-AmOzZPsMuMnsYF>#2GJ_LQjZze!*% zhysA!i1?nB9b9|PyH9pUkw$f!dU`hOH0P@_Z@$um*(bkZ{y+@ik96M@I{~-({{4HP zDD0dG)G1?PVkVGf&hr%h9U4DVv>O8~5|UgD8njtGJUoEsJlx#o+(s2L9Euq29(8nw z29^55y6ch7-4d9L6Ob()E{wLH^ud=?A=nm-nvoGYDP7PK?v6FpyBp*^P`6$BrwP7; zIQm5%h|&r~RmJ-60vgoW+4;MRW_Wh?Ixg0)%Q;hq{y3xkG9yI7#LDV5;N`>s&8>4hxo5i{0*SE+xMG z7yzgSZ`rqL{+`*g3p3#PE7)zyl{#n{5lXC%)c3Rna17PHrN>ZKl39d*OTl0cI?0lh zi@zrL4KL}qRnyMkAeX`kP^iCJ#tW2zR%!17Y2z7?n0q2lvZx8g&Zgcsvd<@5&U*Z5 z%S(I~`5^>9PA0FVrKPZti7eV-w!U1Q#X5arcz9VL1}MVcnpV9S!&j1*_sVAa1XSYd zPJVZ&ckSHvf#6gw{WDLk$vh}FeuiMxy_C&M1YFn242}j~cF>6fiho;+myeH6P*9hh z!)K43Hd5+4P}>JyQaVSvadN3vsEBUc>HSt)8mB(|iqwo35^|=^9IS zgMwe6Z3lqnRm{IM`z!s*d-VZ)DHKH&`(K~Y z5ISo7TOIiNo2H1|gQ8HIRZ(6Z`dy{R8Lr>-S0HA?8wlf+UwdVf$_uLO^o_Vv`pQgk zf6<%J-5lZM0od_5CI)b?EcrHrn^zN>>?wftwP@7$=>_z`czpt;JwtiMX@(gH;L73M9T&goTCzaV`+N1H~60Q;(7Q4x}SM zR<*kelsry_9RVqR@Le;Ye+g>&E5i}-68o*;0gg`jl@IKL$8g@%Jn8lA`Tk9tr=yr> zx!(wk7LJ2(gLVOZ-{!s{YApGC8%IP!u5LsD;NL2E`+reuk#PQC)RygCv9v2Sx(fD|Ej5rLmpJ*VE0 z{d)uQsMkcldU^^A3j7vxGBdFq%77@~!{8(97xnSp&tFEo+1&>WrgxgaYsnBC_RU+I zq0(enK9C6O@3;f1>Egm&M2_%5+1&bdK7duhkISreSxMdj9Xz$K-I54^{$v0k5Omx8 z)6da zwt{$(b;Awf&rBlGbP*8Zj;Eu|U3Hom$hQ&ckP5|-8w;GucH%~lthoZ2mu)QAMT@H&Nmm%C5BpjU#)eO!NNe% zP(;LlB`N&3`#ccX0hKVeSFy-}fvI}R2xviJVTFEEwf!t#iN{ksZ0yGoZ^)u;(lr;~ zmF-+TKp>q)RG_j#P0`pt{MZ>Oyz%{<9yw6c#h@|h=HNJ`-7iM}#!mB<>A!8&P^fEcnI4^;xj5rpL*R7-0H5zKwl6{kf%Kr_T-%g<;v>z6sSoMlg<;X4oq&a_p9s%OOAKqi3=*pQ@YHAu0mKf@X_ujw_51iB#~ z2YXg`PM;R|$hlgSSkIQUv{%XU``)NxJ)UV?(YxRuzdC`B-|_$M0$fWAYC=!!@FN~I zA=;1TnfRuB6kY+UYHgJBd7Sw~`@x&hvir4(*bMH^W8u@Q{ELN}DS z;p5n9`m;1KKon764FvB0q`d%5%qRD=;ZVujt&`ju>oNfy^}i}j2J47{p8c1o&XVoe zFW9aEw(K`JZ1c?Slm?ovVKU4-v%88l0a>mXw&Z0fJK@kbnF>hCp`q*K@XyP6I&YTp zv_3|o%aSjM&NF9Q zC^@@fw)}iqNLLm&BObsg3GJYg_t#^{wV^0y6XvLxtQdu11((?bo;mSl6`=R4GmJ(* zLX7k(?OneYXMFMjRoCs>pGzgb!$@Q;U*to|@%Aq-G2z3b?IRepErqJHby%6_ z;?i|a&uFqYNaBc9w8hllR}J)cJy=Nq7G2XQ!-LP-zIZODV)s2!u8^qZg$#uLK96v z^Ow^TO$Af5Y1>g|OXPtEwJ8s-sc6 z)io)qBednpBp12PI|0jd05PTq_HMaas0O%{2&t*t5lQVed4D7nd|fkcjsFQhIgV@n z$qbTe3y&rqQlrxe8|VN0wo*+uSJ_VdO!%(+mB@Fn&KJ*o1EFwZb`KrtZ-jB_D|nBL4#4 z!2=3{avW`^0FdxCIl_L{@L*4u-+j_g$1Zblgv2*dydN)B&(ut5)Z{JO=T_)E=UZ1j zrUq&pM{0Kia7L5Dqfg`C-i6tuJ#+;iMX&9wt|H2tX|ho7%t**BYfileyI+k*{^uxT zZvWwKrMO+%+(T>hA4PFSQGi^FH-n3Fz(-oVUO2a~w4C()K9L?*RpJlr;2kQZijMjW z2#TTWkm+|rWpXxX*lCm8>4#zUYj=13u(!2)!(;v>42yfFj+etz+GUhH;M ztnF;Z7I@|DaZ0-|A7A|fIkAMEWfvZ(mXw77WUQggTty2cPyRPYJw|><-9Qy0@F>ct zb+3MQ!Z(VK5W)x2W>$v*P0*}B|N6zl)_vfL&#|!&oX)x3wN%O$ZUS%@p!5A%d?T*}r=^9s4958rI=PNCF5RA2#`k1*|_`+ZI42`vz3%n5JM@G%Wn< zfwmK`2!v_zS!r4Nfgq(I71V(HvRbL*9d3e~kW}n}hZJda$Oi!(mDIp^ zNG)v=Ow*B3tr+%%512TE00aT?aXqk0)2yb>bXlJ-L%x7z@$op(!1S^mnfD8j@yP>e ztRx+}%FGP@6I#$#<9`756?%BvhkwV!81PQoDHYR`A~E~S2EW?<{QW}HJvSnbXgd^m z9ZknTKkNUJJ=6X@{<7LiY{P`gmbHaaUZR*^Pe6NPeeD1E1DyNc#}tbaK_4_adUrH_ z`i=|!F9`r%!ixUC(er=k`v1pO&gfwVbE?`dmdPGrQQ8|XrlWdU-IsD3Uuu_P*`K`! zpt{%h8~4Y*@7=|jiQ{MsRG_N*qy{R}5IUqR>Uf`co`!ne~!H&XSJ(&-~U(`|ORFyOVqqOZ+qJm`;_cS}-Ylni@Q#N6-U=W`>Y6HJdlm~@|W(Lh$g zrF_vT5k-SEUNjbG#7abS!?tH+V#J`guODQ$Y~sk8EGs|$-BW|Ex;i$a&4ci{n!=U_ zx~uzF8T$Pjz-6!=Y(&{anylxq(V)8!FX%72AYyOPjedQ%nLj#n&&Qt#nxbX-&=w?# zWHUE~wYz^WFLrl)T91ekc{h6+gUdMb=?}ypcDQ~>M)|N*yD8g6#Jr{N z%&+JD>`{NV@<60HXOSXP%$5Wx(Xga^3NUc8Es-JtTL!cHmaJZMR~wzp6)}3AL3;{r zQ33M+>mC79i9c+b5Q07)$40Tyi=tz$z=+Gc+pKgr&r^jjj3Fe$kZ-c$@iGbY=ub_V zUtcg49VdmS%5Ysr2EUxv&oNL+647uIR&o}mU1<*AKHlZM-Zir}ZEhXnAO{)Cy?xM9 zG$ZCoEz61l!?b!K!r?5DVUgRTJ&~ikAsJeD4@5zRKthH&Rpv`ko=Hib33GUG1YN2d zO`@4nJR?!uX{!DNd`8&tFsj~G$gp^~MU;7f* zx8&yiaT?Jv2k1w72evH;w6}4wTd&;rL!CME5A-)X7z0#k@~>^n*{Ip5_$sEAlJ`Ck za8<7=&(Q(=UaM-QbONK%857AN4&)bIdwiv%#~?Qt1SdA z?UW?L!ff4SuV5oi*!jf3b|12s0mj{SAM6DlW(f)A>fX-b!F%>2VPWXj<$LjI5yXM3 zLw89IOn_zj^l}0O(gat~0&`ubaWV(J!{810z*HnJvv7WEl6*8H`c}}VuidZy`rOHP zWT(|#o0>`&wsM*`*j_&Z{d^cHHm-|Ny5fQF9&CBt3puv+up4EizVYGQ@IT5z z$TXQR1+oSS68Le{;z8kQOqN+uYHPVGJkIiyR|Hq}@XfPN0lr4LbQ5&>PQvZ07d(;; zOz@pgU>6Xm_CMl2NR-u3@tN6qZJ`bA`utS#y9xn!LH~XuJOZwmCFn+LN+Sb|=sv-!hjKh>*ultDr^O|jz(-ThVSwp|M$bx zeykWX>%E??0#{^a@4}#Gb~{ro-)()BYU$uTZfv<&z`64jRjq<#2;`U_S5cSTeEww0 z4tDW71Ag*}r2U_dn#;1}bVK7fhP-O(^;}wm_A1N}!1n}b69M}A)|a!!)Wu)pwXY=; z>^+YnJ?;1B{$F{xr---0Z46Y}=op1^Ra0mSqKX)KnS)-lqI)-?h!y{*7eGEni6NG; z-*jd@=v{q3yigd>U?j?K7G%Xh-h&;Wr3nGL z!{WIc+sRyYQu3_jMV~*_b;#78TG%K!WCZ7fGfo%J=R_ijSf^@>FY@dJ{O)C4* zLE3teS`+@ovhh>=XrQ^*Q!Bu9e|cYHLy~@?VlFNTXi2IglK{kt@jVWX3{r8!N$s^% zWvfsi%&R;0Resi9kt7kv1*i*P2ypO8;ho?Oi~Hs1QjVNdUAsrb1DL~P3SZc*w!W4U z9guHd1s?Y6KBW)_y;Xa#MWyaQTY#`+;mz+O*GY<=QC``+agHTs!{Kpmh-^BwW5;cF z#c_xW!q{`u?K}pd;V1wC0gXeY|BcSdqS7pXr z3^7iniv76ArKZ1u$wx?Qgtj%8Qp??$jjf?wh^8)P9JedZIqU826z|(`pbpPs^m-?K zcWd9PXRqZhEck%)ZqxwdV4Dd;bRMx?w03u(dgxQeF7V#W8kGF`81y zb!}>^ieD+(9JntcPOl=ODkxeGsT{NKyN8GG&q^BtZ${_DZfy4pD>8jh7u^vNTlQSd zmY$Q>`$xS3mc$L$)+g;`1(7c6WR0|pQ(zZ&KY%)`L{tIzK8og9ccf@TXy~APM$+UTFMO)j>Lfei zAL+-$Mp?{l7gr%0hN5RD{8E;Th2M7>Au-$BCARaF!v?d0XM0;p3w(!gq58$Z^FJME zEuzQp`F8)s!TXhKhr6uv#hX9Zj(79t^EWLA?UxOK?~#Fl$UlyZo&ksOy@Oe?)aE8f z{n|Rc`1$(|Dz6&i zCqf)_odY+7qhI^r;EaA1xJSlkHvWuV}%_bH| zEruQ;ck{EHSi^w3Md_UT^Wi{0)K6PW#$SvUatzW<*+UdbLdi(Vdmk zU%Y~X9Zj}we(5m=D-G~sEN=~`-i-z@J`)|&f=ZKzm;(vUXu5W zNkNTMh?%~LCE8lSbmr~j9(tNIw$M;1qJ+?i!lw!{|2$34M<=In>R@ATWO$^uyZ?g1 z>FfGy&; z!A3_2!V}+r`G2+d-SJfa@855!C@YeZ2t~3-Mx-PfM`q@+LPxUKA!HPm>)=RH_6i|o{I0jY_v8EhJs$Vtci;E_w?CcndcCjLwO-ftd|s#4#FF>hcW0)g zn-dI@wtb<#?rPFjSe(%K_?J|J({_SRVH34p>!BYOeb(jLiWjP8^P#)3HKSJ#I3@nr z_}h>1+-pr%$|)Ky7j;6*jNdp6kp4dEk$V-D&8|4yoTABQhnpqmX0-dW`M_}Yg~p;~ z-E*Z<4+{C(IQW#j^_=8>rZ6uEy(krbzNE(4_v9$Oq`*e{1jnyTHp{uR>56=9Yf1TL zF{L58XmM-aF=(WcOuU3P-K?ypps|a><6`SK#{oaODP6zOQRy;y{auBQ`JbR~m%??I z58%dy&J3H&>JBeky9jM@NlqTd^e>~*$#@+Xn@FupE@?Ik6^Kb=o(%BZB#n*|XC+cp zb)`zu$z-kh5t3_8o={(mT50;Nd4RmiMyqAoe~GKIxRoQ1d#!b~RU^oZk0%J&M~uX@ zQkF$)`<6cYmL&el6t$T5nQd>zTWzz&C zI$go(@eA0-Tqh0KOysLE-pK5MI?wgY$zjYibfpLDeRt4_2GHnd3@j2}IuM8t@FJFq z3J7FqnLO!N;OflYUtv(^wKG5Q+4TOeOZg021x3#~Rl+X3AwUqP)4X?ul^@fLdHtts zZD*rT3Kle}_iw)>NS_!% z%$=-}LtMgy zujw??ZehAHv_v-A+OtAzJ>7%(vfUUZIK}i_l@=~?nM^m zP{?z4bCdFvp?f<@$H|h!ggm7NXeSJgfB_pMuoZ`7S$qz$iNbHy-!zIsi7%O_;m=|{ z3|;sHCVz2fb0vbKO{fL^Z03H1aZK^md4q9%S%=Tx%URjkVQ_#KG**FH^`b|z;a>|B zGAWa_>j;P6XvVvVP2kjYI}X=+(mmGx_IYx4VF6I*u6OTF_U9Qse*BnZ&53k{Lxx&t zt4ux@R<>pk8Echf+xO0YJZ|Y(Z|khJjSgZmQQaAFg)fsceDLVhYkDM zb6c^+4}SC4Y}+_K^bX7Tdc7+XdLMuI@lmh!7FRbndV2bcpd^N6wi^Ia*ZXCSSlPo( zTmKY@f<0G*y4&iQZ|zq$GJqXeR3&Z}u=WrtiDZ-Z_?2bXJa^L~lBw@jr|-6lukQ|k zWm?Cble{SWEYaanxMIRa%K$$}p5c|*rtzJ|#Q2ZQ2S*9zfSYuARbKl}yC%-P;2NA9 zpoey#z6>p%2*9RTn<8DO(bF)Bk=OWeRYgi>(6(#BCWr7H&u(_^Md|6ge;CO{E-eXc zwex|tI?4lVZ}~^$uL6{&$l^UfyB2K+i^pF*xrluvzvu1mtLRSJ1_J%Z_tqYa9FMlF zt-*d~IZ++sX#1t`4jK~bizFy@=OecN+?{hg8W~+4^JhW8g1iQXCfMfrXTDsoRszu{ zRfsY8QA^KapM;j(3wa|I{Uv(I-hkB8hU%d@9r)axl+66?Ew0B<;s-^ z4tk{bN!wL~+X(Xx!+y!9rXILR;QRqFG%{5r_^^phK~Rkj;cnj7Yj)>Ob1?zJ%_Ay? z5uZQv5gcl&s0$^+q!ayU@E0JmwVyxHa|F5@`ppE|dOpfvjSEfIo8T{?9U2R$I5{~v zHu;!i=qEMV_9KLcFt7t9XQ5ny6wPCZ_2^L?!Weot%Mg%O?(V4RvdKS9Km}%Va9~_Vw+sgBFX@a(!fL0Lp039uD_8JqrG?Ow3MM~jq#B_jBl#YNv`AW?d+4Wi#aa}8>J zy%wFx@&Lmts9DCmqS}MxHilDyJd*%4Lk4yc-qn@xk)Eb3`$2|rR4UWlooKD? zM;l*;MpbI`kheL{;GtCHVD1q#q3#294US53DE)=N(9jEMml$Jafr6D7eUH4-oz&mt z^CC;86i6=EA0lW5(6{N#lJ+B}&w^)UGQKZkT5 zg#{||T1YvIBG|o9jnn7>JQ*-;9{@}U4!M?S!T8Ml(;CUpWiC;FuzgIO$`6MY)yC); z7%lR~P|5d~wwDA7 zYPzS2?k5tQ1=&B<7>dz5AVpd5b>?$JrL&Oucuc3q9+y9aR8&?RWk+t;0ic(bfg$CB zhj`M2#U6#QZ^ev$)UnUGxPhZH>e4CiKGqxp9$nf~&?e;HhGJCVXa@QQxTD{IDI!l5 z5q1}v0%iZzI$n%3fcBpVR^%Y1g3x9fEvMp=;HVo6T#}Xb1Y3`>Jv#AY|DoJ{o$iRA zEKm(50o4s08>^3~dj1F#_oHrk(+niEUx2CF-d?Z&(#U2vY`trT&@jB=4j^1Wy@83A zIWq6}!-7YCY6X)30L}^Y)18f!oxv%Hew2=b)CvJA9}oNl^dTUyPh7ZPI8F{cj~dyM z%Sq%mBcM^?(Jy*ejlU8_dK*Pr5pL#ACY+U(6##}Z)l+=_+LGa}hxi>t!5I?t__0tM zaJOzNrv1k!%z+=#79#|BwAXfIu!X#)f9C^=KiAj8K2$H(k*6LGYoX5|X151L!s5N^|N|E`W5$-b3|_8A)*cJmCPMP(oA7R*`Lk<%?7K3s=rCbwAg;_bgI znUfL3?cP~=Ci#9E(SpJcAMn@F*_%NKKhDlB?>ePn?ofi}9=eL1A;?oc6r2l=9EhNC z1OFlK7+07mRaaMc_>7wENKGZt;@)Xgb>Kw(bO9hdL?W#NftB~dzmJsA0w78F1WH>1 z@{R3J{o_wi4FSvTDh42B9(oHg^{pjRLh=+g&5U#<(1Y4Tx(Kl#F5U$xq!wnQ2zESM ziAy~1l2K5w2q@DMn#cbuY+ySE1_nR_=ugNfz%lbI{Z)+qLde-cBNsncmlG)sOn2Wd z1-apPg!IUvXW-_fJt1p45r?Hf^1cBG;v^vYLJpsaJ5PnQHBU1k*ygUTE+F2igXm#u z2C6DOsSu{H4RA?>_0|@@>;O6RoTnhtbu&&2;cPtjidRxzz7{TZ?>W4}-A0xFBbR&) zxKI(J3Q-xE>(xKF(ant296&neLCSKV!H__NG$I;QYDq{}GcXHHAVR>=Lh!x%WFL~i z^kxc4HSmy_MEwPs42Fa#yh#f=s1We3l~-I|9{4Zi9xD?tQY{lsgdSC&zuEjbM$1om z%xE}Fyh4$>0gp%pp$>GG_aNn4ej=Dj$0S532sjoKClCV1-;i4K09ZYcl3m*I6}cHx z@|sM03J0@0`L3`+ULQ5Oqj#236H~&%$bN_j!#7szlv<;KE9oR)--7>)Ngwmh*5(J& ziTw|)9pBS=rTJ0yUHNsZKOUioh78r0$A7CAkL+gtJW@#3`(M9)O^pj2eVLAo-QGz?2`KL5FPFU4$6EmZ-&e}?Su*v7f zfM}H2v9pVYSJM4Moqi=or9?)X9ShRp9{a6qeu+-cJAyoQ$x*t>(3s~* z72kaon=8dQ?tRo&SFE#lMcPIP6UO3XoEJ?}V%%JE&rl9qGhy3VYxt8co#px`$M-mL zoev87y;shT^9+j;X1=OF_1bFAEeU4Q3;RdOhS>U~CVv*M8sIdv$k#tK8}*~9PXIZ9+k39>&~kHg9nK>c>VcOWh-yjOy83lJRm4R^>VklhF0tjafy@59=pZ{D8~+2l-9vPq9#!!+izTxrI)h(-xu zL-^+BPw}noKTo3Lu+rQ7DR7xS((}ybQvx;Qum{mx`bb`>Zd%QLY7vgvuW~|Ik8!hk zYg}+e8eRQ0#k1+rQ-yEm7WqH)*!N10=5aDchw>a8cJbsx5IjUEg`}w9tSn=hDgQ%P zO4b!hj+ulN@-_A4a|P|yF(YCGG}CgP6X>f1YIN(C8WJRL$3{=hzT^lRRN)UJzSj%J* zIFR?Lv7uL%)HfcQEL1IR`FVO;%j1!kitl~IiVM~BTfH^2(GTeupUo(o___aDZ6hWm zSPM(FGRd-k%<-Azvz)X-cL{n@XT!qH!6d z9IBozUf!|d!AyUxV<+0pO?14Xaa)K@*;+6o_3`M%@-NsLsDG-0Eoimmf!URgia(>@ zP2T)GwF{QNf_Bie1-5jVAW}+3^Q#9Rmy-`I$@0|5;rS%bd<@#EZy36lA5)yGazx&c z*u&#q{QXl5eh^=4dB9hg-2k8fD2ip3x~VX?bTl+K%9IN%%wd$eKy>a%2wv1fO-bMg z6yy3rn~XifsBD&E!k>ob-WBl7d7X(_uvOOc=Xq?LR~vJH{efNYr9F_3qS|2bNtWR< z_1Ye%u9N>LE8V-aPXO$opS7PqPH{3iw52*EiA7IEc^2JNywGfXadKWo^bEd#*s8Sj z_A|*wQG{?1JznN^hn|LIMl&6Kfppq*qa(wnS8SFj9s80dOAgvk5Crn0jF)7&G5Pi3 z2>~>^dmWew(sRDEM}kVf;3<$Ss=vO-&A42f_y$Ls_0urfyQb2F-I%?0U$t>0q51OW z>UOZUkuh=Xv$}Bb$IJCfeL@MZ*zERRK5ullqi+ZY5uSjeC1yse?9<0a;)H#3R7PYO zgTzbfwFA1D)+oJ32W+_3>i2syIUO;+M5|JfrF5Sy(yD0Gv7^+Ag}nvz- z-yHAJGGJ}hXw}F!yI(YL_t(0aZi9UMR_SSf(G%_IFAd47zx|rL9H%zQtxPo7`qLnz z)DS`C*`)HyWF?A|-2-IIbLP!2k4ji;dz)6Oi(w7wzU;14QWr2) zsKPBs-#M5UP$Xkyq!?EJAt88;6h-~2kfeT@44aNXb+QY~RyE9vJJN1(3lsj;lFf{l z{$-GI8lMqfL^gYYEwU;oS5N~LLC8#{Yir)IbM%x7wdl4Pnnb0J+s9U2vP z_Jw=D(ZG-pcK2bQ#mSwSoba2|Znd1@K^=#HY~XL#rz+Jr@pXpEW&76Z2by*t6{TKl z0vWef=DH!^K2q!S?Oo}gQH1j{?55BhI}3aZxBxj<0$r5lA{U|ecP=ee0XEK8&!H)Z z4q*6f()K?+CHk$)k*9~yFu+Q}I}BGi0i6n7u^lS;i2wX9@=TJNx|*6vrE@y)>c72< z*lUjHGr~>@?ZAoo@ZrOFTkN%K*8sM!1@pz_l2uv;r_m{3Uool*f4KG@}r$jo? z!kvJqgoMk+@;KlX=a$FY;Gze#{dzWeZcU6T@YaM_?d@ZpR>AqGq*d2u-pQHn$&v2bY2G@ z$#iWvAo#m45aH24!W)zD7Hw}whO1%Dgz|`Rk{UWUCU^07QgSi}E9*T6hx$JY)$y8C zNEjOWRr6WqV&*DGUlpgv#OOy=}s}Bo!S9-`s=sdbm8(muvE4wXVOsTnp06XxgTMLbIl^P8cMYl$dDrg8nf| z6$0j^toM6%cY==t()Aq9`OqSa9${x?tpu`jwl;Qa1e4zVE`n_ldM`XF@oJgMr^_TO z>0xHxP@x%`CR(GxzXr%c1u%ZW$H_np3* z-%bK~_2mUpZ=CYK9=ZmqI6&!#zpD&)N@M1<_xqi;SW-q9S?(y)e1jWpn4~*k2A>*% zJ|W2Iqp&Q3d{l-q1ES!J7AXGPO9_E5bPO_yj}Si=G#h&QLWgGAvP6jXm74BZ+)> z^@R1rulJq>s>iKx@K1cS`N!(zsL5nFE5ZqYQa_Iq)w1F&c5?0Mx+ar(UyE=Pb8*es ztrLsnr{&%whs{%1FM6yd5)4WnC9w#;KYfVuExM~LOFDfuW*2t3yV%R3Nw@k-=N|gk ziW&$HwikBWKh7$#849e3a9gE+#r-D5b24Bt+|IDY96*Ao*|(cn=5bE2*j0<(CS`S7 z+LGH1q$|7sx0T??=?{*HH{aD{pW6So;94nOHjlq7Rp3kzs&VZ{s9v0v6Nk5+Zf@o(59f=77Bm-X5fM z{vO;r_nbN}&6qp9KI~?kghad|*RV&CMiyE%w%|{Gp`!#6Lh};v_b(k|qc@j*jv?1MQI8mjV_;6yc|ST(XZCR*u;!Pq z#ohgcA+!qox0e@_5)=ESjur`dRni~9LFu=u2sYo5Vf8XbM%yQLx4eMfnqAZ60%Y9O z2!c=|oQbjj3F;$&!?kF)&T@0`eMjT-rN1|3R-th8rY#Fl+yACh|AV^y|CjzR?ehOc zP}5f#fC82lW&INXQ5E>8{wksc90_v#ksA724*w??@OLRceKlQN(p82rUaP(on%$e> z({s?9c#|<*2sR(7T7B%#f=ihQhx&m6#L)$;;q=pJQr~ia!u;UJ1ZBAahX&{NI0ah+ z%TK(c&{qK1rk{p3NpfB>V%|-k5E0t0`Mq>mXHeFHtXrgCdur##nnHs~-1DOgJp!~| zI~4|JMqStObXRMPbxqCqO_TT6lks)I%-i=9KX-&gNhV& zmD(o}eLbCqB!ak$rOB*+QuewjErNy1tiqoE=m))5ZsVX9H=kQO&$jCfHf4-TI&96} zhIA(AUaU_wHlOykyxBvZkcq?@j9T=xH`D7RJ|Oy*S9WlMAZaKAOXnCV{CL6dK2<34 zZVEIPyB^K`T1JLc$N)Sv@&s)XnaK*V91OFQmWUs^wUkJ}avuz9E>RJzQcuCftBtsw zXe(|m7JOf{63H4c8h9mQzUL5Bo-Kv#{ZAsN(M#k7z#5pX_*+bdhS_!A$x|%L@tocJ zh1u>Fj;EerKkPBjB)M+T(SBWc(*4xS`0y8PH=^!OGDyyyiS)36oFSMRgvH&mMyK*l z_?VbPGU{-f1Hjfy(6eq{6qn2?)SxqTl@wWee E0g-@{ZvX%Q literal 0 HcmV?d00001