From 190cb4b52c0636543b9b415cb975c2c4cddebd6a Mon Sep 17 00:00:00 2001 From: Bryan Rosander Date: Tue, 6 Dec 2016 09:59:31 -0500 Subject: [PATCH 1/4] MINIFI-153 - Adding proxy settings to remote process group --- .../bootstrap/util/ConfigTransformer.java | 9 + .../bootstrap/util/ConfigTransformerTest.java | 22 +- ...eHttpMiNiFiProxyNoPasswordTemplateTest.yml | 302 ++++++++++++++++++ ...okeHttpMiNiFiProxyPasswordTemplateTest.yml | 302 ++++++++++++++++++ .../minifi/commons/schema/ConfigSchema.java | 31 +- .../schema/RemoteProcessGroupSchema.java | 75 ++++- .../commons/schema/common/BaseSchema.java | 21 +- .../common/BaseSchemaWithIdAndName.java | 2 +- .../schema/common/CollectionOverlap.java | 56 ++++ .../commons/schema/v1/ConfigSchemaV1.java | 9 +- .../commons/schema/v2/ConfigSchemaV2.java | 94 +----- .../schema/v2/ProcessGroupSchemaV2.java | 26 +- .../schema/v2/RemoteProcessGroupSchemaV2.java | 101 ++++++ .../schema/RemoteProcessGroupSchemaTest.java | 22 +- .../v2/RemoteProcessGroupSchemaV2Test.java | 82 +++++ .../src/main/markdown/System_Admin_Guide.md | 7 +- .../dto/RemoteProcessGroupSchemaFunction.java | 4 + .../toolkit/configuration/ConfigMainTest.java | 32 +- .../InvokeHttpMiNiFiTemplateTest-v2.yml | 297 +++++++++++++++++ .../InvokeHttpMiNiFiTemplateTest.xml | 7 +- .../InvokeHttpMiNiFiTemplateTest.yml | 6 +- ...ProcessGroupsAndRemoteProcessGroups-v2.yml | 287 +++++++++++++++++ .../ProcessGroupsAndRemoteProcessGroups.yml | 8 + .../test/resources/SimpleTailFileToRPG-v2.yml | 113 +++++++ .../test/resources/SimpleTailFileToRPG.yml | 4 + 25 files changed, 1757 insertions(+), 162 deletions(-) create mode 100644 minifi-bootstrap/src/test/resources/InvokeHttpMiNiFiProxyNoPasswordTemplateTest.yml create mode 100644 minifi-bootstrap/src/test/resources/InvokeHttpMiNiFiProxyPasswordTemplateTest.yml create mode 100644 minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/common/CollectionOverlap.java create mode 100644 minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/v2/RemoteProcessGroupSchemaV2.java create mode 100644 minifi-commons/minifi-commons-schema/src/test/java/org/apache/nifi/minifi/commons/schema/v2/RemoteProcessGroupSchemaV2Test.java create mode 100644 minifi-toolkit/minifi-toolkit-configuration/src/test/resources/InvokeHttpMiNiFiTemplateTest-v2.yml create mode 100644 minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ProcessGroupsAndRemoteProcessGroups-v2.yml create mode 100644 minifi-toolkit/minifi-toolkit-configuration/src/test/resources/SimpleTailFileToRPG-v2.yml diff --git a/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java b/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java index 55d32853e..b724019b2 100644 --- a/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java +++ b/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java @@ -19,6 +19,7 @@ import org.apache.commons.io.output.ByteArrayOutputStream; +import org.apache.commons.lang3.StringUtils; import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeException; import org.apache.nifi.minifi.bootstrap.exception.InvalidConfigurationException; import org.apache.nifi.minifi.commons.schema.ComponentStatusRepositorySchema; @@ -569,6 +570,14 @@ protected static void addRemoteProcessGroup(final Element parentElement, RemoteP addTextElement(element, "yieldPeriod", remoteProcessGroupProperties.getYieldPeriod()); addTextElement(element, "transmitting", "true"); addTextElement(element, "transportProtocol", remoteProcessGroupProperties.getTransportProtocol()); + addTextElement(element, "proxyHost", remoteProcessGroupProperties.getProxyHost()); + if (remoteProcessGroupProperties.getProxyPort() != null) { + addTextElement(element, "proxyPort", Integer.toString(remoteProcessGroupProperties.getProxyPort())); + } + addTextElement(element, "proxyUser", remoteProcessGroupProperties.getProxyUser()); + if (!StringUtils.isEmpty(remoteProcessGroupProperties.getProxyPassword())) { + addTextElement(element, "proxyPassword", remoteProcessGroupProperties.getProxyPassword()); + } List remoteInputPorts = remoteProcessGroupProperties.getInputPorts(); for (RemoteInputPortSchema remoteInputPortSchema : remoteInputPorts) { diff --git a/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerTest.java b/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerTest.java index be88cb4ba..2cb3a9c74 100644 --- a/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerTest.java +++ b/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerTest.java @@ -119,6 +119,21 @@ public void testFunnelAndRpgTransform() throws Exception { testConfigFileTransform("config-funnel-and-rpg.yml"); } + @Test + public void testRpgTransform() throws Exception { + testConfigFileTransform("config-multiple-RPGs.yml"); + } + + @Test + public void testRpgProxyNoPassTransform() throws Exception { + testConfigFileTransform("InvokeHttpMiNiFiProxyNoPasswordTemplateTest.yml"); + } + + @Test + public void testRpgProxyPassTransform() throws Exception { + testConfigFileTransform("InvokeHttpMiNiFiProxyPasswordTemplateTest.yml"); + } + public void testConfigFileTransform(String configFile) throws Exception { ConfigSchema configSchema = SchemaLoader.loadConfigSchemaFromYaml(ConfigTransformerTest.class.getClassLoader().getResourceAsStream(configFile)); @@ -215,7 +230,12 @@ private void testRemoteProcessGroups(Element element, RemoteProcessGroupSchema r assertEquals(remoteProcessingGroupSchema.getUrl(), getText(element, "url")); assertEquals(remoteProcessingGroupSchema.getTimeout(), getText(element, "timeout")); assertEquals(remoteProcessingGroupSchema.getYieldPeriod(), getText(element, "yieldPeriod")); - + assertEquals(remoteProcessingGroupSchema.getTransportProtocol(), getText(element, "transportProtocol")); + assertEquals(remoteProcessingGroupSchema.getProxyHost(), getText(element, "proxyHost")); + String proxyPortText = getText(element, "proxyPort"); + assertEquals(remoteProcessingGroupSchema.getProxyPort(), StringUtil.isNullOrEmpty(proxyPortText) ? null : Integer.parseInt(proxyPortText)); + assertEquals(remoteProcessingGroupSchema.getProxyUser(), getText(element, "proxyUser")); + assertEquals(remoteProcessingGroupSchema.getProxyPassword(), getText(element, "proxyPassword")); NodeList inputPortElements = (NodeList) xPathFactory.newXPath().evaluate("inputPort", element, XPathConstants.NODESET); assertEquals(remoteProcessingGroupSchema.getInputPorts().size(), inputPortElements.getLength()); diff --git a/minifi-bootstrap/src/test/resources/InvokeHttpMiNiFiProxyNoPasswordTemplateTest.yml b/minifi-bootstrap/src/test/resources/InvokeHttpMiNiFiProxyNoPasswordTemplateTest.yml new file mode 100644 index 000000000..084da2d37 --- /dev/null +++ b/minifi-bootstrap/src/test/resources/InvokeHttpMiNiFiProxyNoPasswordTemplateTest.yml @@ -0,0 +1,302 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the \"License\"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +MiNiFi Config Version: 3 +Flow Controller: + name: InvokeHttpMiNiFiTemplateTest2 + comment: '' +Core Properties: + flow controller graceful shutdown period: 10 sec + flow service write delay interval: 500 ms + administrative yield duration: 30 sec + bored yield duration: 10 millis + max concurrent threads: 1 +FlowFile Repository: + partitions: 256 + checkpoint interval: 2 mins + always sync: false + Swap: + threshold: 20000 + in period: 5 sec + in threads: 1 + out period: 5 sec + out threads: 4 +Content Repository: + content claim max appendable size: 10 MB + content claim max flow files: 100 + always sync: false +Provenance Repository: + provenance rollover time: 1 min +Component Status Repository: + buffer size: 1440 + snapshot frequency: 1 min +Security Properties: + keystore: '' + keystore type: '' + keystore password: '' + key password: '' + truststore: '' + truststore type: '' + truststore password: '' + ssl protocol: '' + Sensitive Props: + key: + algorithm: PBEWITHMD5AND256BITAES-CBC-OPENSSL + provider: BC +Processors: +- id: 0a73c5e4-7216-4cdf-9008-ace353478d55 + name: LogAttribute + class: org.apache.nifi.processors.standard.LogAttribute + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: + - success + Properties: + Attributes to Ignore: + Attributes to Log: + Log Level: info + Log Payload: 'false' + Log prefix: +- id: 0bda7282-7a4f-4d39-83e5-ea86aa63f1dc + name: LogAttribute2 + class: org.apache.nifi.processors.standard.LogAttribute + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: + - success + Properties: + Attributes to Ignore: + Attributes to Log: + Log Level: info + Log Payload: 'false' + Log prefix: +- id: dcff217a-a25c-424f-b9f1-d82c55bfa61b + name: Route On Status Code + class: org.apache.nifi.processors.standard.RouteOnAttribute + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: [] + Properties: + '200': ${invokehttp.status.code:equals(200)} + Routing Strategy: Route to Property name +- id: 66649998-3b08-4b41-9f07-a51d999743f6 + name: Search Google + class: org.apache.nifi.processors.standard.InvokeHTTP + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: [] + Properties: + Add Response Headers to Request: 'false' + Always Output Response: 'false' + Attributes to Send: + Basic Authentication Password: + Basic Authentication Username: + Connection Timeout: 5 secs + Content-Type: ${mime.type} + Digest Authentication: 'false' + Follow Redirects: 'True' + HTTP Method: GET + Include Date Header: 'True' + Max Length To Put In Attribute: '256' + Penalize on "No Retry": 'false' + Proxy Host: + Proxy Port: + Put Response Body In Attribute: + Read Timeout: 15 secs + Remote URL: http://www.google.com/search?q=${q}&rct=j + SSL Context Service: + Trusted Hostname: + Use Chunked Encoding: 'false' + invokehttp-proxy-password: + invokehttp-proxy-user: + send-message-body: 'true' +- id: fcb378d7-6438-41d3-8485-7da85e870ca4 + name: Source + class: org.apache.nifi.processors.standard.GenerateFlowFile + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 30 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: [] + Properties: + Batch Size: '1' + Data Format: Binary + File Size: 10 b + Unique FlowFiles: 'false' +- id: 6638cd15-0aaf-4bba-a18a-722cf85038b6 + name: q=nifi + class: org.apache.nifi.processors.attributes.UpdateAttribute + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: [] + Properties: + Delete Attributes Expression: + q: nifi +Controller Services: [] +Process Groups: [] +Input Ports: [] +Output Ports: [] +Funnels: [] +Connections: +- id: 3039718a-bb40-4811-9b74-ecbe926daae8 + name: Route On Status Code/200/LogAttribute + source id: dcff217a-a25c-424f-b9f1-d82c55bfa61b + source relationship names: + - '200' + destination id: 0a73c5e4-7216-4cdf-9008-ace353478d55 + max work queue size: 0 + max work queue data size: 0 MB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- id: 4a7bdaf1-eba0-4317-92f2-93cef6240c28 + name: Route On Status Code/200/da15f83d-0158-1000-7f8d-eb4d2652f870 + source id: dcff217a-a25c-424f-b9f1-d82c55bfa61b + source relationship names: + - '200' + destination id: da15f83d-0158-1000-7f8d-eb4d2652f870 + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- id: 9f948a25-afb4-4598-9a01-cfb7e3aa5920 + name: Route On Status Code/unmatched/LogAttribute2 + source id: dcff217a-a25c-424f-b9f1-d82c55bfa61b + source relationship names: + - unmatched + destination id: 0bda7282-7a4f-4d39-83e5-ea86aa63f1dc + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- id: 7299c143-46f6-4a94-aa24-ec47db16f73e + name: Route On Status Code/unmatched/da15f83d-0158-1000-7f8d-eb4d2652f870 + source id: dcff217a-a25c-424f-b9f1-d82c55bfa61b + source relationship names: + - unmatched + destination id: da15f83d-0158-1000-7f8d-eb4d2652f870 + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- id: 451b153a-8a4e-4bed-99ab-5159e499bcf1 + name: Search Google/Failure/Route On Status Code + source id: 66649998-3b08-4b41-9f07-a51d999743f6 + source relationship names: + - Failure + destination id: dcff217a-a25c-424f-b9f1-d82c55bfa61b + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- id: ddbc1711-8cda-4418-b6ee-895763ef1a97 + name: Search Google/No Retry/Route On Status Code + source id: 66649998-3b08-4b41-9f07-a51d999743f6 + source relationship names: + - No Retry + destination id: dcff217a-a25c-424f-b9f1-d82c55bfa61b + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- id: b97044aa-3401-49df-86ab-bc440cd8f6c8 + name: Search Google/Original/Route On Status Code + source id: 66649998-3b08-4b41-9f07-a51d999743f6 + source relationship names: + - Original + destination id: dcff217a-a25c-424f-b9f1-d82c55bfa61b + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- id: c6a06d31-51fb-488f-bcde-495102f8c93f + name: Search Google/Response/Route On Status Code + source id: 66649998-3b08-4b41-9f07-a51d999743f6 + source relationship names: + - Response + destination id: dcff217a-a25c-424f-b9f1-d82c55bfa61b + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- id: 3172a4b6-c35b-4d2f-a7c1-68eb16cc1b37 + name: Search Google/Retry/Route On Status Code + source id: 66649998-3b08-4b41-9f07-a51d999743f6 + source relationship names: + - Retry + destination id: dcff217a-a25c-424f-b9f1-d82c55bfa61b + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- id: 2ef71cd8-eb24-4653-8526-34ab84e79328 + name: Source/success/q=nifi + source id: fcb378d7-6438-41d3-8485-7da85e870ca4 + source relationship names: + - success + destination id: 6638cd15-0aaf-4bba-a18a-722cf85038b6 + max work queue size: 0 + max work queue data size: 0 MB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- id: ee5ee5bf-a552-43cf-9514-c14d40350bff + name: q=nifi/success/Search Google + source id: 6638cd15-0aaf-4bba-a18a-722cf85038b6 + source relationship names: + - success + destination id: 66649998-3b08-4b41-9f07-a51d999743f6 + max work queue size: 0 + max work queue data size: 0 MB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +Remote Process Groups: +- id: 5b0a5c18-a02e-4a85-8080-0187ee679763 + name: '' + url: http://nifi.minifi2:8080/nifi + comment: '' + timeout: 30 sec + yield period: 10 sec + transport protocol: HTTP + proxy host: squidnp.minifi + proxy port: 3128 + proxy user: '' + proxy password: '' + Input Ports: + - id: da15f83d-0158-1000-7f8d-eb4d2652f870 + name: response + comment: '' + max concurrent tasks: 1 + use compression: false diff --git a/minifi-bootstrap/src/test/resources/InvokeHttpMiNiFiProxyPasswordTemplateTest.yml b/minifi-bootstrap/src/test/resources/InvokeHttpMiNiFiProxyPasswordTemplateTest.yml new file mode 100644 index 000000000..348219bd8 --- /dev/null +++ b/minifi-bootstrap/src/test/resources/InvokeHttpMiNiFiProxyPasswordTemplateTest.yml @@ -0,0 +1,302 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the \"License\"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +MiNiFi Config Version: 3 +Flow Controller: + name: InvokeHttpMiNiFiTemplateTest2 + comment: '' +Core Properties: + flow controller graceful shutdown period: 10 sec + flow service write delay interval: 500 ms + administrative yield duration: 30 sec + bored yield duration: 10 millis + max concurrent threads: 1 +FlowFile Repository: + partitions: 256 + checkpoint interval: 2 mins + always sync: false + Swap: + threshold: 20000 + in period: 5 sec + in threads: 1 + out period: 5 sec + out threads: 4 +Content Repository: + content claim max appendable size: 10 MB + content claim max flow files: 100 + always sync: false +Provenance Repository: + provenance rollover time: 1 min +Component Status Repository: + buffer size: 1440 + snapshot frequency: 1 min +Security Properties: + keystore: '' + keystore type: '' + keystore password: '' + key password: '' + truststore: '' + truststore type: '' + truststore password: '' + ssl protocol: '' + Sensitive Props: + key: + algorithm: PBEWITHMD5AND256BITAES-CBC-OPENSSL + provider: BC +Processors: +- id: 0a73c5e4-7216-4cdf-9008-ace353478d55 + name: LogAttribute + class: org.apache.nifi.processors.standard.LogAttribute + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: + - success + Properties: + Attributes to Ignore: + Attributes to Log: + Log Level: info + Log Payload: 'false' + Log prefix: +- id: 0bda7282-7a4f-4d39-83e5-ea86aa63f1dc + name: LogAttribute2 + class: org.apache.nifi.processors.standard.LogAttribute + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: + - success + Properties: + Attributes to Ignore: + Attributes to Log: + Log Level: info + Log Payload: 'false' + Log prefix: +- id: dcff217a-a25c-424f-b9f1-d82c55bfa61b + name: Route On Status Code + class: org.apache.nifi.processors.standard.RouteOnAttribute + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: [] + Properties: + '200': ${invokehttp.status.code:equals(200)} + Routing Strategy: Route to Property name +- id: 66649998-3b08-4b41-9f07-a51d999743f6 + name: Search Google + class: org.apache.nifi.processors.standard.InvokeHTTP + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: [] + Properties: + Add Response Headers to Request: 'false' + Always Output Response: 'false' + Attributes to Send: + Basic Authentication Password: + Basic Authentication Username: + Connection Timeout: 5 secs + Content-Type: ${mime.type} + Digest Authentication: 'false' + Follow Redirects: 'True' + HTTP Method: GET + Include Date Header: 'True' + Max Length To Put In Attribute: '256' + Penalize on "No Retry": 'false' + Proxy Host: + Proxy Port: + Put Response Body In Attribute: + Read Timeout: 15 secs + Remote URL: http://www.google.com/search?q=${q}&rct=j + SSL Context Service: + Trusted Hostname: + Use Chunked Encoding: 'false' + invokehttp-proxy-password: + invokehttp-proxy-user: + send-message-body: 'true' +- id: fcb378d7-6438-41d3-8485-7da85e870ca4 + name: Source + class: org.apache.nifi.processors.standard.GenerateFlowFile + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 30 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: [] + Properties: + Batch Size: '1' + Data Format: Binary + File Size: 10 b + Unique FlowFiles: 'false' +- id: 6638cd15-0aaf-4bba-a18a-722cf85038b6 + name: q=nifi + class: org.apache.nifi.processors.attributes.UpdateAttribute + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: [] + Properties: + Delete Attributes Expression: + q: nifi +Controller Services: [] +Process Groups: [] +Input Ports: [] +Output Ports: [] +Funnels: [] +Connections: +- id: 3039718a-bb40-4811-9b74-ecbe926daae8 + name: Route On Status Code/200/LogAttribute + source id: dcff217a-a25c-424f-b9f1-d82c55bfa61b + source relationship names: + - '200' + destination id: 0a73c5e4-7216-4cdf-9008-ace353478d55 + max work queue size: 0 + max work queue data size: 0 MB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- id: 4a7bdaf1-eba0-4317-92f2-93cef6240c28 + name: Route On Status Code/200/da15f83d-0158-1000-7f8d-eb4d2652f870 + source id: dcff217a-a25c-424f-b9f1-d82c55bfa61b + source relationship names: + - '200' + destination id: da15f83d-0158-1000-7f8d-eb4d2652f870 + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- id: 9f948a25-afb4-4598-9a01-cfb7e3aa5920 + name: Route On Status Code/unmatched/LogAttribute2 + source id: dcff217a-a25c-424f-b9f1-d82c55bfa61b + source relationship names: + - unmatched + destination id: 0bda7282-7a4f-4d39-83e5-ea86aa63f1dc + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- id: 7299c143-46f6-4a94-aa24-ec47db16f73e + name: Route On Status Code/unmatched/da15f83d-0158-1000-7f8d-eb4d2652f870 + source id: dcff217a-a25c-424f-b9f1-d82c55bfa61b + source relationship names: + - unmatched + destination id: da15f83d-0158-1000-7f8d-eb4d2652f870 + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- id: 451b153a-8a4e-4bed-99ab-5159e499bcf1 + name: Search Google/Failure/Route On Status Code + source id: 66649998-3b08-4b41-9f07-a51d999743f6 + source relationship names: + - Failure + destination id: dcff217a-a25c-424f-b9f1-d82c55bfa61b + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- id: ddbc1711-8cda-4418-b6ee-895763ef1a97 + name: Search Google/No Retry/Route On Status Code + source id: 66649998-3b08-4b41-9f07-a51d999743f6 + source relationship names: + - No Retry + destination id: dcff217a-a25c-424f-b9f1-d82c55bfa61b + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- id: b97044aa-3401-49df-86ab-bc440cd8f6c8 + name: Search Google/Original/Route On Status Code + source id: 66649998-3b08-4b41-9f07-a51d999743f6 + source relationship names: + - Original + destination id: dcff217a-a25c-424f-b9f1-d82c55bfa61b + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- id: c6a06d31-51fb-488f-bcde-495102f8c93f + name: Search Google/Response/Route On Status Code + source id: 66649998-3b08-4b41-9f07-a51d999743f6 + source relationship names: + - Response + destination id: dcff217a-a25c-424f-b9f1-d82c55bfa61b + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- id: 3172a4b6-c35b-4d2f-a7c1-68eb16cc1b37 + name: Search Google/Retry/Route On Status Code + source id: 66649998-3b08-4b41-9f07-a51d999743f6 + source relationship names: + - Retry + destination id: dcff217a-a25c-424f-b9f1-d82c55bfa61b + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- id: 2ef71cd8-eb24-4653-8526-34ab84e79328 + name: Source/success/q=nifi + source id: fcb378d7-6438-41d3-8485-7da85e870ca4 + source relationship names: + - success + destination id: 6638cd15-0aaf-4bba-a18a-722cf85038b6 + max work queue size: 0 + max work queue data size: 0 MB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- id: ee5ee5bf-a552-43cf-9514-c14d40350bff + name: q=nifi/success/Search Google + source id: 6638cd15-0aaf-4bba-a18a-722cf85038b6 + source relationship names: + - success + destination id: 66649998-3b08-4b41-9f07-a51d999743f6 + max work queue size: 0 + max work queue data size: 0 MB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +Remote Process Groups: +- id: 5b0a5c18-a02e-4a85-8080-0187ee679763 + name: '' + url: http://nifi.minifi2:8080/nifi + comment: '' + timeout: 30 sec + yield period: 10 sec + transport protocol: HTTP + proxy host: squidp.minifi + proxy port: 3128 + proxy user: username + proxy password: password + Input Ports: + - id: da15f83d-0158-1000-7f8d-eb4d2652f870 + name: response + comment: '' + max concurrent tasks: 1 + use compression: false diff --git a/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/ConfigSchema.java b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/ConfigSchema.java index 7a9bf09fc..39e5a2f80 100644 --- a/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/ConfigSchema.java +++ b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/ConfigSchema.java @@ -18,18 +18,16 @@ package org.apache.nifi.minifi.commons.schema; import org.apache.nifi.minifi.commons.schema.common.BaseSchema; +import org.apache.nifi.minifi.commons.schema.common.CollectionOverlap; import org.apache.nifi.minifi.commons.schema.common.ConvertableSchema; import org.apache.nifi.minifi.commons.schema.common.StringUtil; import org.apache.nifi.minifi.commons.schema.common.WritableSchema; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.stream.Collectors; import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.COMPONENT_STATUS_REPO_KEY; @@ -121,29 +119,24 @@ public ConfigSchema(Map map, List validationIssues) { checkForDuplicates(this::addValidationIssue, FOUND_THE_FOLLOWING_DUPLICATE_OUTPUT_PORT_IDS, allOutputPortIds); // Potential connection sources and destinations need to have unique ids - OverlapResults overlapResults = findOverlap(new HashSet<>(allProcessorIds), new HashSet<>(allRemoteInputPortIds), new HashSet<>(allInputPortIds), new HashSet<>(allOutputPortIds), - new HashSet<>(allFunnelIds)); - if (overlapResults.duplicates.size() > 0) { - addValidationIssue(FOUND_THE_FOLLOWING_DUPLICATE_IDS + overlapResults.duplicates.stream().sorted().collect(Collectors.joining(", "))); + CollectionOverlap overlapResults = new CollectionOverlap<>(new HashSet<>(allProcessorIds), new HashSet<>(allRemoteInputPortIds), new HashSet<>(allInputPortIds), + new HashSet<>(allOutputPortIds), new HashSet<>(allFunnelIds)); + if (overlapResults.getDuplicates().size() > 0) { + addValidationIssue(FOUND_THE_FOLLOWING_DUPLICATE_IDS + overlapResults.getDuplicates().stream().sorted().collect(Collectors.joining(", "))); } allConnectionSchemas.forEach(c -> { String destinationId = c.getDestinationId(); - if (!StringUtil.isNullOrEmpty(destinationId) && !overlapResults.seen.contains(destinationId)) { + if (!StringUtil.isNullOrEmpty(destinationId) && !overlapResults.getElements().contains(destinationId)) { addValidationIssue(CONNECTION_WITH_ID + c.getId() + HAS_INVALID_DESTINATION_ID + destinationId); } String sourceId = c.getSourceId(); - if (!StringUtil.isNullOrEmpty(sourceId) && !overlapResults.seen.contains(sourceId)) { + if (!StringUtil.isNullOrEmpty(sourceId) && !overlapResults.getElements().contains(sourceId)) { addValidationIssue(CONNECTION_WITH_ID + c.getId() + HAS_INVALID_SOURCE_ID + sourceId); } }); } - protected static OverlapResults findOverlap(Collection... collections) { - Set seen = new HashSet<>(); - return new OverlapResults<>(seen, Arrays.stream(collections).flatMap(c -> c.stream()).sequential().filter(s -> !seen.add(s)).collect(Collectors.toSet())); - } - public static List getAllProcessGroups(ProcessGroupSchema processGroupSchema) { List result = new ArrayList<>(); addProcessGroups(processGroupSchema, result); @@ -215,14 +208,4 @@ public int getVersion() { public ConfigSchema convert() { return this; } - - private static class OverlapResults { - private final Set seen; - private final Set duplicates; - - private OverlapResults(Set seen, Set duplicates) { - this.seen = seen; - this.duplicates = duplicates; - } - } } diff --git a/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/RemoteProcessGroupSchema.java b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/RemoteProcessGroupSchema.java index ccec3896e..53705f244 100644 --- a/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/RemoteProcessGroupSchema.java +++ b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/RemoteProcessGroupSchema.java @@ -30,22 +30,16 @@ public class RemoteProcessGroupSchema extends BaseSchemaWithIdAndName { public static final String URL_KEY = "url"; public static final String TIMEOUT_KEY = "timeout"; public static final String TRANSPORT_PROTOCOL_KEY = "transport protocol"; + public static final String PROXY_HOST_KEY = "proxy host"; + public static final String PROXY_PORT_KEY = "proxy port"; + public static final String PROXY_USER_KEY = "proxy user"; + public static final String PROXY_PASSWORD_KEY = "proxy password"; - private enum transportProtocolOptions { - RAW("RAW"), HTTP("HTTP"); - - private final String stringValue; - - private transportProtocolOptions(final String s) { - stringValue = s; - } - - public String toString() { - return stringValue; - } + public enum TransportProtocolOptions { + RAW, HTTP; public static boolean valid(String input) { - return RAW.stringValue.equals(input) || HTTP.stringValue.equals(input); + return RAW.name().equals(input) || HTTP.name().equals(input); } } @@ -53,6 +47,10 @@ public static boolean valid(String input) { public static final String DEFAULT_TIMEOUT = "30 secs"; public static final String DEFAULT_YIELD_PERIOD = "10 sec"; public static final String DEFAULT_TRANSPORT_PROTOCOL= "RAW"; + public static final String DEFAULT_PROXY_HOST = ""; + public static final Integer DEFAULT_PROXY_PORT = null; + public static final String DEFAULT_PROXY_USER = ""; + public static final String DEFAULT_PROXY_PASSWORD = ""; private String url; private List inputPorts; @@ -61,6 +59,10 @@ public static boolean valid(String input) { private String timeout = DEFAULT_TIMEOUT; private String yieldPeriod = DEFAULT_YIELD_PERIOD; private String transportProtocol = DEFAULT_TRANSPORT_PROTOCOL; + private String proxyHost = DEFAULT_PROXY_HOST; + private Integer proxyPort = DEFAULT_PROXY_PORT; + private String proxyUser = DEFAULT_PROXY_USER; + private String proxyPassword = DEFAULT_PROXY_PASSWORD; public RemoteProcessGroupSchema(Map map) { super(map, "RemoteProcessGroup(id: {id}, name: {name})"); @@ -78,9 +80,14 @@ public RemoteProcessGroupSchema(Map map) { yieldPeriod = getOptionalKeyAsType(map, YIELD_PERIOD_KEY, String.class, wrapperName, DEFAULT_YIELD_PERIOD); transportProtocol = getOptionalKeyAsType(map, TRANSPORT_PROTOCOL_KEY, String.class, wrapperName, DEFAULT_TRANSPORT_PROTOCOL); - if (!transportProtocolOptions.valid(transportProtocol)){ + if (!TransportProtocolOptions.valid(transportProtocol)){ addValidationIssue(TRANSPORT_PROTOCOL_KEY, wrapperName, "it must be either 'RAW' or 'HTTP' but is '" + transportProtocol + "'"); } + + proxyHost = getOptionalKeyAsType(map, PROXY_HOST_KEY, String.class, wrapperName, DEFAULT_PROXY_HOST); + proxyPort = getOptionalKeyAsType(map, PROXY_PORT_KEY, Integer.class, wrapperName, DEFAULT_PROXY_PORT); + proxyUser = getOptionalKeyAsType(map, PROXY_USER_KEY, String.class, wrapperName, DEFAULT_PROXY_USER); + proxyPassword = getOptionalKeyAsType(map, PROXY_PASSWORD_KEY, String.class, wrapperName, DEFAULT_PROXY_PASSWORD); } @Override @@ -91,6 +98,10 @@ public Map toMap() { result.put(TIMEOUT_KEY, timeout); result.put(YIELD_PERIOD_KEY, yieldPeriod); result.put(TRANSPORT_PROTOCOL_KEY, transportProtocol); + result.put(PROXY_HOST_KEY, proxyHost); + result.put(PROXY_PORT_KEY, proxyPort == null ? "" : proxyPort); + result.put(PROXY_USER_KEY, proxyUser); + result.put(PROXY_PASSWORD_KEY, proxyPassword); putListIfNotNull(result, INPUT_PORTS_KEY, inputPorts); return result; } @@ -118,4 +129,40 @@ public List getInputPorts() { public String getTransportProtocol() { return transportProtocol; } + + public void setProxyHost(String proxyHost) { + this.proxyHost = proxyHost; + } + + public void setProxyPort(Integer proxyPort) { + this.proxyPort = proxyPort; + } + + public void setProxyUser(String proxyUser) { + this.proxyUser = proxyUser; + } + + public void setProxyPassword(String proxyPassword) { + this.proxyPassword = proxyPassword; + } + + public void setTransportProtocol(String transportProtocol) { + this.transportProtocol = transportProtocol; + } + + public String getProxyHost() { + return proxyHost; + } + + public Integer getProxyPort() { + return proxyPort; + } + + public String getProxyUser() { + return proxyUser; + } + + public String getProxyPassword() { + return proxyPassword; + } } diff --git a/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/common/BaseSchema.java b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/common/BaseSchema.java index 7cd82f730..919bba326 100644 --- a/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/common/BaseSchema.java +++ b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/common/BaseSchema.java @@ -27,20 +27,15 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Set; -import java.util.TreeSet; import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Supplier; -import java.util.regex.Pattern; import java.util.stream.Collectors; public abstract class BaseSchema implements Schema { public static final String IT_WAS_NOT_FOUND_AND_IT_IS_REQUIRED = "it was not found and it is required"; public static final String EMPTY_NAME = "empty_name"; - public static final Pattern ID_REPLACE_PATTERN = Pattern.compile("[^A-Za-z0-9_-]"); - protected final Supplier> mapSupplier; public BaseSchema() { @@ -99,7 +94,7 @@ public T getRequiredKeyAsType(Map valueMap, String key, Class targetClass T getKeyAsType(Map valueMap, String key, Class targetClass, String wrapperName, boolean required, T defaultValue) { Object value = valueMap.get(key); - if (value == null) { + if (value == null || (targetClass != String.class && "".equals(value))) { if (defaultValue != null) { return defaultValue; } else if(required) { @@ -175,7 +170,7 @@ private OutputT interpretValueAsType(Class inputType, } private T interpretValueAsType(Object obj, String key, Class targetClass, String wrapperName, boolean required, boolean instantiateIfNull) { - if (obj == null) { + if (obj == null || (targetClass != String.class && "".equals(obj))) { if (required){ addValidationIssue(key, wrapperName, "it is a required property but was not found"); } else { @@ -220,15 +215,9 @@ public static void putListIfNotNull(Map valueMap, String key, List duplicateMessageConsumer, String errorMessagePrefix, List strings) { if (strings != null) { - Set seen = new HashSet<>(); - Set duplicates = new TreeSet<>(); - for (String string : strings) { - if (!seen.add(string)) { - duplicates.add(String.valueOf(string)); - } - } - if (duplicates.size() > 0) { - duplicateMessageConsumer.accept(errorMessagePrefix + duplicates.stream().collect(Collectors.joining(", "))); + CollectionOverlap collectionOverlap = new CollectionOverlap<>(strings); + if (collectionOverlap.getDuplicates().size() > 0) { + duplicateMessageConsumer.accept(errorMessagePrefix + collectionOverlap.getDuplicates().stream().collect(Collectors.joining(", "))); } } } diff --git a/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/common/BaseSchemaWithIdAndName.java b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/common/BaseSchemaWithIdAndName.java index 5c9f8c7de..344200d25 100644 --- a/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/common/BaseSchemaWithIdAndName.java +++ b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/common/BaseSchemaWithIdAndName.java @@ -23,7 +23,7 @@ import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.NAME_KEY; -public abstract class BaseSchemaWithIdAndName extends BaseSchemaWithId implements WritableSchema { +public class BaseSchemaWithIdAndName extends BaseSchemaWithId implements WritableSchema { private String name; public BaseSchemaWithIdAndName(Map map, String wrapperName) { diff --git a/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/common/CollectionOverlap.java b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/common/CollectionOverlap.java new file mode 100644 index 000000000..358c216c7 --- /dev/null +++ b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/common/CollectionOverlap.java @@ -0,0 +1,56 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one or more + * * contributor license agreements. See the NOTICE file distributed with + * * this work for additional information regarding copyright ownership. + * * The ASF licenses this file to You under the Apache License, Version 2.0 + * * (the "License"); you may not use this file except in compliance with + * * the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package org.apache.nifi.minifi.commons.schema.common; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class CollectionOverlap { + private final Set elements; + private final Set duplicates; + + public CollectionOverlap(Collection... collections) { + this(Arrays.stream(collections).map(c -> c.stream())); + } + + public CollectionOverlap(Stream... streams) { + this(Arrays.stream(streams).map(Function.identity())); + } + + public CollectionOverlap(Stream> streams) { + Set elements = new HashSet<>(); + this.duplicates = Collections.unmodifiableSet(streams.flatMap(Function.identity()).sequential().filter(s -> !elements.add(s)).collect(Collectors.toSet())); + this.elements = Collections.unmodifiableSet(elements); + } + + public Set getElements() { + return elements; + } + + public Set getDuplicates() { + return duplicates; + } +} diff --git a/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/v1/ConfigSchemaV1.java b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/v1/ConfigSchemaV1.java index 37f34b120..1989b884e 100644 --- a/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/v1/ConfigSchemaV1.java +++ b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/v1/ConfigSchemaV1.java @@ -33,6 +33,7 @@ import org.apache.nifi.minifi.commons.schema.RemoteProcessGroupSchema; import org.apache.nifi.minifi.commons.schema.SecurityPropertiesSchema; import org.apache.nifi.minifi.commons.schema.common.BaseSchema; +import org.apache.nifi.minifi.commons.schema.common.CollectionOverlap; import org.apache.nifi.minifi.commons.schema.common.ConvertableSchema; import org.apache.nifi.minifi.commons.schema.common.StringUtil; @@ -160,13 +161,7 @@ protected List getConnectionSchemas(List proc if (processors != null) { processors.stream().forEachOrdered(p -> processorNameToIdMap.put(p.getName(), p.getId())); - - Set processorNames = new HashSet<>(); - processors.stream().map(ProcessorSchema::getName).forEachOrdered(n -> { - if (!processorNames.add(n)) { - duplicateProcessorNames.add(n); - } - }); + duplicateProcessorNames = new CollectionOverlap<>(processors.stream().map(ProcessorSchema::getName)).getDuplicates(); } Set remoteInputPortIds = new HashSet<>(); diff --git a/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/v2/ConfigSchemaV2.java b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/v2/ConfigSchemaV2.java index 20a9da2bb..d73d8ae53 100644 --- a/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/v2/ConfigSchemaV2.java +++ b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/v2/ConfigSchemaV2.java @@ -33,18 +33,15 @@ import org.apache.nifi.minifi.commons.schema.RemoteProcessGroupSchema; import org.apache.nifi.minifi.commons.schema.SecurityPropertiesSchema; import org.apache.nifi.minifi.commons.schema.common.BaseSchema; +import org.apache.nifi.minifi.commons.schema.common.CollectionOverlap; import org.apache.nifi.minifi.commons.schema.common.ConvertableSchema; import org.apache.nifi.minifi.commons.schema.common.StringUtil; -import org.apache.nifi.minifi.commons.schema.common.WritableSchema; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.stream.Collectors; import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.COMPONENT_STATUS_REPO_KEY; @@ -56,7 +53,7 @@ import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.PROVENANCE_REPO_KEY; import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.SECURITY_PROPS_KEY; -public class ConfigSchemaV2 extends BaseSchema implements WritableSchema, ConvertableSchema { +public class ConfigSchemaV2 extends BaseSchema implements ConvertableSchema { public static final int CONFIG_VERSION = 2; public static final String VERSION = "MiNiFi Config Version"; public static final String FOUND_THE_FOLLOWING_DUPLICATE_REMOTE_INPUT_PORT_IDS = "Found the following duplicate remote input port ids: "; @@ -71,6 +68,7 @@ public class ConfigSchemaV2 extends BaseSchema implements WritableSchema, Conver public static final String FOUND_THE_FOLLOWING_DUPLICATE_FUNNEL_IDS = "Found the following duplicate funnel ids: "; public static final String FOUND_THE_FOLLOWING_DUPLICATE_REMOTE_PROCESS_GROUP_NAMES = "Found the following duplicate remote process group names: "; public static String TOP_LEVEL_NAME = "top level"; + private FlowControllerSchema flowControllerProperties; private CorePropertiesSchema coreProperties; private FlowFileRepositorySchema flowfileRepositoryProperties; @@ -79,7 +77,6 @@ public class ConfigSchemaV2 extends BaseSchema implements WritableSchema, Conver private SecurityPropertiesSchema securityProperties; private ProcessGroupSchemaV2 processGroupSchema; private ProvenanceReportingSchema provenanceReportingProperties; - private ProvenanceRepositorySchema provenanceRepositorySchema; public ConfigSchemaV2(Map map) { @@ -133,29 +130,24 @@ public ConfigSchemaV2(Map map, List validationIssues) { checkForDuplicates(this::addValidationIssue, FOUND_THE_FOLLOWING_DUPLICATE_OUTPUT_PORT_IDS, allOutputPortIds); // Potential connection sources and destinations need to have unique ids - OverlapResults overlapResults = findOverlap(new HashSet<>(allProcessorIds), new HashSet<>(allRemoteInputPortIds), new HashSet<>(allInputPortIds), new HashSet<>(allOutputPortIds), - new HashSet<>(allFunnelIds)); - if (overlapResults.duplicates.size() > 0) { - addValidationIssue(FOUND_THE_FOLLOWING_DUPLICATE_IDS + overlapResults.duplicates.stream().sorted().collect(Collectors.joining(", "))); + CollectionOverlap overlapResults = new CollectionOverlap<>(new HashSet<>(allProcessorIds), new HashSet<>(allRemoteInputPortIds), new HashSet<>(allInputPortIds), + new HashSet<>(allOutputPortIds), new HashSet<>(allFunnelIds)); + if (overlapResults.getDuplicates().size() > 0) { + addValidationIssue(FOUND_THE_FOLLOWING_DUPLICATE_IDS + overlapResults.getDuplicates().stream().sorted().collect(Collectors.joining(", "))); } allConnectionSchemas.forEach(c -> { String destinationId = c.getDestinationId(); - if (!StringUtil.isNullOrEmpty(destinationId) && !overlapResults.seen.contains(destinationId)) { + if (!StringUtil.isNullOrEmpty(destinationId) && !overlapResults.getElements().contains(destinationId)) { addValidationIssue(CONNECTION_WITH_ID + c.getId() + HAS_INVALID_DESTINATION_ID + destinationId); } String sourceId = c.getSourceId(); - if (!StringUtil.isNullOrEmpty(sourceId) && !overlapResults.seen.contains(sourceId)) { + if (!StringUtil.isNullOrEmpty(sourceId) && !overlapResults.getElements().contains(sourceId)) { addValidationIssue(CONNECTION_WITH_ID + c.getId() + HAS_INVALID_SOURCE_ID + sourceId); } }); } - protected static OverlapResults findOverlap(Collection... collections) { - Set seen = new HashSet<>(); - return new OverlapResults<>(seen, Arrays.stream(collections).flatMap(c -> c.stream()).sequential().filter(s -> !seen.add(s)).collect(Collectors.toSet())); - } - public static List getAllProcessGroups(ProcessGroupSchemaV2 processGroupSchema) { List result = new ArrayList<>(); addProcessGroups(processGroupSchema, result); @@ -167,7 +159,13 @@ private static void addProcessGroups(ProcessGroupSchemaV2 processGroupSchema, Li processGroupSchema.getProcessGroupSchemas().forEach(p -> addProcessGroups(p, result)); } - public Map toMap() { + @Override + public int getVersion() { + return CONFIG_VERSION; + } + + @Override + public ConfigSchema convert() { Map result = mapSupplier.get(); result.put(VERSION, getVersion()); putIfNotNull(result, FLOW_CONTROLLER_PROPS_KEY, flowControllerProperties); @@ -179,64 +177,6 @@ public Map toMap() { putIfNotNull(result, SECURITY_PROPS_KEY, securityProperties); result.putAll(processGroupSchema.toMap()); putIfNotNull(result, PROVENANCE_REPORTING_KEY, provenanceReportingProperties); - return result; - } - - public FlowControllerSchema getFlowControllerProperties() { - return flowControllerProperties; - } - - public CorePropertiesSchema getCoreProperties() { - return coreProperties; - } - - public FlowFileRepositorySchema getFlowfileRepositoryProperties() { - return flowfileRepositoryProperties; - } - - public ContentRepositorySchema getContentRepositoryProperties() { - return contentRepositoryProperties; - } - - public SecurityPropertiesSchema getSecurityProperties() { - return securityProperties; - } - - public ProcessGroupSchemaV2 getProcessGroupSchema() { - return processGroupSchema; - } - - public ProvenanceReportingSchema getProvenanceReportingProperties() { - return provenanceReportingProperties; - } - - public ComponentStatusRepositorySchema getComponentStatusRepositoryProperties() { - return componentStatusRepositoryProperties; - } - - public ProvenanceRepositorySchema getProvenanceRepositorySchema() { - return provenanceRepositorySchema; - } - - @Override - public int getVersion() { - return CONFIG_VERSION; - } - - @Override - public ConfigSchema convert() { - Map map = this.toMap(); - List validationIssues = getValidationIssues(); - return new ConfigSchema(map, validationIssues); - } - - private static class OverlapResults { - private final Set seen; - private final Set duplicates; - - private OverlapResults(Set seen, Set duplicates) { - this.seen = seen; - this.duplicates = duplicates; - } + return new ConfigSchema(result, getValidationIssues()); } } diff --git a/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/v2/ProcessGroupSchemaV2.java b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/v2/ProcessGroupSchemaV2.java index a9f04499d..2e8559a3a 100644 --- a/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/v2/ProcessGroupSchemaV2.java +++ b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/v2/ProcessGroupSchemaV2.java @@ -1,20 +1,18 @@ /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * - * * Licensed to the Apache Software Foundation (ASF) under one or more - * * contributor license agreements. See the NOTICE file distributed with - * * this work for additional information regarding copyright ownership. - * * The ASF licenses this file to You under the Apache License, Version 2.0 - * * (the "License"); you may not use this file except in compliance with - * * the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.apache.nifi.minifi.commons.schema.v2; diff --git a/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/v2/RemoteProcessGroupSchemaV2.java b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/v2/RemoteProcessGroupSchemaV2.java new file mode 100644 index 000000000..3e1b762ec --- /dev/null +++ b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/v2/RemoteProcessGroupSchemaV2.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.commons.schema.v2; + +import org.apache.nifi.minifi.commons.schema.RemoteInputPortSchema; +import org.apache.nifi.minifi.commons.schema.RemoteProcessGroupSchema; +import org.apache.nifi.minifi.commons.schema.RemoteProcessGroupSchema.TransportProtocolOptions; +import org.apache.nifi.minifi.commons.schema.common.BaseSchema; +import org.apache.nifi.minifi.commons.schema.common.BaseSchemaWithIdAndName; +import org.apache.nifi.minifi.commons.schema.common.ConvertableSchema; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.apache.nifi.minifi.commons.schema.RemoteProcessGroupSchema.DEFAULT_COMMENT; +import static org.apache.nifi.minifi.commons.schema.RemoteProcessGroupSchema.DEFAULT_TIMEOUT; +import static org.apache.nifi.minifi.commons.schema.RemoteProcessGroupSchema.DEFAULT_TRANSPORT_PROTOCOL; +import static org.apache.nifi.minifi.commons.schema.RemoteProcessGroupSchema.DEFAULT_YIELD_PERIOD; +import static org.apache.nifi.minifi.commons.schema.RemoteProcessGroupSchema.TIMEOUT_KEY; +import static org.apache.nifi.minifi.commons.schema.RemoteProcessGroupSchema.TRANSPORT_PROTOCOL_KEY; +import static org.apache.nifi.minifi.commons.schema.RemoteProcessGroupSchema.URL_KEY; +import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.COMMENT_KEY; +import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.INPUT_PORTS_KEY; +import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.YIELD_PERIOD_KEY; + +public class RemoteProcessGroupSchemaV2 extends BaseSchema implements ConvertableSchema { + private BaseSchemaWithIdAndName idAndName; + private String url; + private List inputPorts; + + private String comment = DEFAULT_COMMENT; + private String timeout = DEFAULT_TIMEOUT; + private String yieldPeriod = DEFAULT_YIELD_PERIOD; + private String transportProtocol = DEFAULT_TRANSPORT_PROTOCOL; + + public RemoteProcessGroupSchemaV2(Map map) { + idAndName = new BaseSchemaWithIdAndName(map, "RemoteProcessGroup(id: {id}, name: {name})"); + + String wrapperName = idAndName.getWrapperName(); + url = getRequiredKeyAsType(map, URL_KEY, String.class, wrapperName); + inputPorts = convertListToType(getRequiredKeyAsType(map, INPUT_PORTS_KEY, List.class, wrapperName), "input port", RemoteInputPortSchema.class, INPUT_PORTS_KEY); + if (inputPorts != null) { + for (RemoteInputPortSchema remoteInputPortSchema: inputPorts) { + addIssuesIfNotNull(remoteInputPortSchema); + } + } + + comment = getOptionalKeyAsType(map, COMMENT_KEY, String.class, wrapperName, DEFAULT_COMMENT); + timeout = getOptionalKeyAsType(map, TIMEOUT_KEY, String.class, wrapperName, DEFAULT_TIMEOUT); + yieldPeriod = getOptionalKeyAsType(map, YIELD_PERIOD_KEY, String.class, wrapperName, DEFAULT_YIELD_PERIOD); + transportProtocol = getOptionalKeyAsType(map, TRANSPORT_PROTOCOL_KEY, String.class, wrapperName, DEFAULT_TRANSPORT_PROTOCOL); + + if (!TransportProtocolOptions.valid(transportProtocol)){ + addValidationIssue(TRANSPORT_PROTOCOL_KEY, wrapperName, "it must be either 'RAW' or 'HTTP' but is '" + transportProtocol + "'"); + } + } + + @Override + public RemoteProcessGroupSchema convert() { + Map result = idAndName.toMap(); + result.put(URL_KEY, url); + result.put(COMMENT_KEY, comment); + result.put(TIMEOUT_KEY, timeout); + result.put(YIELD_PERIOD_KEY, yieldPeriod); + result.put(TRANSPORT_PROTOCOL_KEY, transportProtocol); + putListIfNotNull(result, INPUT_PORTS_KEY, inputPorts); + return new RemoteProcessGroupSchema(result); + } + + @Override + public List getValidationIssues() { + List validationIssues = new ArrayList<>(idAndName.getValidationIssues()); + validationIssues.addAll(super.getValidationIssues()); + return validationIssues; + } + + @Override + public int getVersion() { + return ConfigSchemaV2.CONFIG_VERSION; + } + + public String getTransportProtocol() { + return transportProtocol; + } +} diff --git a/minifi-commons/minifi-commons-schema/src/test/java/org/apache/nifi/minifi/commons/schema/RemoteProcessGroupSchemaTest.java b/minifi-commons/minifi-commons-schema/src/test/java/org/apache/nifi/minifi/commons/schema/RemoteProcessGroupSchemaTest.java index aa7ef6127..4d4630771 100644 --- a/minifi-commons/minifi-commons-schema/src/test/java/org/apache/nifi/minifi/commons/schema/RemoteProcessGroupSchemaTest.java +++ b/minifi-commons/minifi-commons-schema/src/test/java/org/apache/nifi/minifi/commons/schema/RemoteProcessGroupSchemaTest.java @@ -32,7 +32,12 @@ public class RemoteProcessGroupSchemaTest { @Test public void testNoPropertiesSet() { - validateIssuesNumMatches(3, new RemoteProcessGroupSchema(new HashMap<>())); + RemoteProcessGroupSchema remoteProcessGroupSchema = new RemoteProcessGroupSchema(new HashMap<>()); + validateIssuesNumMatches(3, remoteProcessGroupSchema); + assertEquals(RemoteProcessGroupSchema.DEFAULT_PROXY_HOST, remoteProcessGroupSchema.getProxyHost()); + assertEquals(RemoteProcessGroupSchema.DEFAULT_PROXY_PORT, remoteProcessGroupSchema.getProxyPort()); + assertEquals(RemoteProcessGroupSchema.DEFAULT_PROXY_USER, remoteProcessGroupSchema.getProxyUser()); + assertEquals(RemoteProcessGroupSchema.DEFAULT_PROXY_PASSWORD, remoteProcessGroupSchema.getProxyPassword()); } @Test @@ -64,6 +69,21 @@ public void testTransportProtocol() { assertEquals(second.getTransportProtocol(), "HTTP"); } + @Test + public void testProxySettings() { + Map map = new HashMap<>(); + map.put(RemoteProcessGroupSchema.PROXY_HOST_KEY, "host"); + map.put(RemoteProcessGroupSchema.PROXY_PORT_KEY, 1234); + map.put(RemoteProcessGroupSchema.PROXY_USER_KEY, "user"); + map.put(RemoteProcessGroupSchema.PROXY_PASSWORD_KEY, "password"); + RemoteProcessGroupSchema remoteProcessGroupSchema = new RemoteProcessGroupSchema(map); + + assertEquals("host", remoteProcessGroupSchema.getProxyHost()); + assertEquals(Integer.valueOf(1234), remoteProcessGroupSchema.getProxyPort()); + assertEquals("user", remoteProcessGroupSchema.getProxyUser()); + assertEquals("password", remoteProcessGroupSchema.getProxyPassword()); + } + private PortSchema createPortSchema(String id, String name, String wrapperName) { Map map = new HashMap<>(); map.put(CommonPropertyKeys.ID_KEY, id); diff --git a/minifi-commons/minifi-commons-schema/src/test/java/org/apache/nifi/minifi/commons/schema/v2/RemoteProcessGroupSchemaV2Test.java b/minifi-commons/minifi-commons-schema/src/test/java/org/apache/nifi/minifi/commons/schema/v2/RemoteProcessGroupSchemaV2Test.java new file mode 100644 index 000000000..5186d566f --- /dev/null +++ b/minifi-commons/minifi-commons-schema/src/test/java/org/apache/nifi/minifi/commons/schema/v2/RemoteProcessGroupSchemaV2Test.java @@ -0,0 +1,82 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one or more + * * contributor license agreements. See the NOTICE file distributed with + * * this work for additional information regarding copyright ownership. + * * The ASF licenses this file to You under the Apache License, Version 2.0 + * * (the "License"); you may not use this file except in compliance with + * * the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package org.apache.nifi.minifi.commons.schema.v2; + +import org.apache.nifi.minifi.commons.schema.ConfigSchema; +import org.apache.nifi.minifi.commons.schema.PortSchema; +import org.apache.nifi.minifi.commons.schema.RemoteProcessGroupSchema; +import org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys; +import org.junit.Test; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; + +public class RemoteProcessGroupSchemaV2Test { + @Test + public void testNoPropertiesSet() { + validateIssuesNumMatches(3, new RemoteProcessGroupSchemaV2(new HashMap<>())); + } + + @Test + public void testInputPortsRootGroup() { + Map map = new HashMap<>(); + map.put(CommonPropertyKeys.INPUT_PORTS_KEY, Arrays.asList(createPortSchema("f94d2469-39f8-4f07-a0d8-acd9396f639e", "testName", ConfigSchema.TOP_LEVEL_NAME).toMap())); + map.put(RemoteProcessGroupSchema.URL_KEY, "http://localhost:8080/nifi"); + map.put(CommonPropertyKeys.ID_KEY, "a58d2fab-7efe-4cb7-8224-12a60bd8003d"); + validateIssuesNumMatches(0, new RemoteProcessGroupSchemaV2(map)); + } + + @Test + public void testTransportProtocol() { + Map map = new HashMap<>(); + map.put(CommonPropertyKeys.INPUT_PORTS_KEY, Arrays.asList(createPortSchema("f94d2469-39f8-4f07-a0d8-acd9396f639e", "testName", ConfigSchema.TOP_LEVEL_NAME).toMap())); + map.put(RemoteProcessGroupSchema.URL_KEY, "http://localhost:8080/nifi"); + map.put(CommonPropertyKeys.ID_KEY, "a58d2fab-7efe-4cb7-8224-12a60bd8003d"); + map.put(RemoteProcessGroupSchema.TRANSPORT_PROTOCOL_KEY, "not valid"); + validateIssuesNumMatches(1, new RemoteProcessGroupSchemaV2(map)); + + map.put(RemoteProcessGroupSchema.TRANSPORT_PROTOCOL_KEY, "RAW"); + RemoteProcessGroupSchemaV2 first = new RemoteProcessGroupSchemaV2(map); + validateIssuesNumMatches(0,first); + assertEquals(first.getTransportProtocol(), "RAW"); + + map.put(RemoteProcessGroupSchema.TRANSPORT_PROTOCOL_KEY, "HTTP"); + RemoteProcessGroupSchemaV2 second = new RemoteProcessGroupSchemaV2(map); + validateIssuesNumMatches(0, second); + assertEquals(second.getTransportProtocol(), "HTTP"); + } + + private PortSchema createPortSchema(String id, String name, String wrapperName) { + Map map = new HashMap<>(); + map.put(CommonPropertyKeys.ID_KEY, id); + map.put(CommonPropertyKeys.NAME_KEY, name); + return new PortSchema(map, wrapperName); + } + + private void validateIssuesNumMatches(int expected, RemoteProcessGroupSchemaV2 remoteProcessGroupSchema) { + int actual = remoteProcessGroupSchema.getValidationIssues().size(); + String issues = "[" + System.lineSeparator() + remoteProcessGroupSchema.getValidationIssues().stream().collect(Collectors.joining("," + System.lineSeparator())) + "]"; + assertEquals("Expected " + expected + " issue(s), got " + actual + ": " + issues, expected, actual); + } +} diff --git a/minifi-docs/src/main/markdown/System_Admin_Guide.md b/minifi-docs/src/main/markdown/System_Admin_Guide.md index d95cd84af..a4e0f317d 100644 --- a/minifi-docs/src/main/markdown/System_Admin_Guide.md +++ b/minifi-docs/src/main/markdown/System_Admin_Guide.md @@ -349,6 +349,7 @@ parses and upconverts to the current version without issue. ### Version 2 -> Version 3 changes 1. Added support for Controller Services. +2. Added support for Site-To-Site over proxy. ## Flow Controller @@ -582,7 +583,7 @@ queue prioritizer class | This configuration option specifies the fully qualifi ## Remote Process Groups -MiNiFi can be used to send data using the Site to Site protocol (via a Remote Process Group) or a Processor. These properties configure the Remote Process Groups that use Site-To-Site to send data to a core instance. +MiNiFi can be used to send data using the Site to Site protocol (via a Remote Process Group) or a Processor. These properties configure the Remote Process Groups that use Site-To-Site to send data to a core instance. The proxy settings are for HTTP Site-To-Site. *Property* | *Description* -------------------| ------------- @@ -592,6 +593,10 @@ url | The URL of the core NiFi instance. timeout | How long MiNiFi should wait before timing out the connection. yield period | When communication with this Remote Process Group fails, it will not be scheduled again for this amount of time. transport protocol | The transport protocol to use for this Remote Process Group. Can be either "RAW" or "HTTP" +proxy host | The hostname of the proxy server +proxy port | The port to connect to on the proxy server +proxy user | The user name on the proxy server +proxy password | The password for the proxy server #### Input Ports Subsection diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/RemoteProcessGroupSchemaFunction.java b/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/RemoteProcessGroupSchemaFunction.java index 1dcc51171..e7d5fa066 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/RemoteProcessGroupSchemaFunction.java +++ b/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/RemoteProcessGroupSchemaFunction.java @@ -59,6 +59,10 @@ public RemoteProcessGroupSchema apply(RemoteProcessGroupDTO remoteProcessGroupDT map.put(RemoteProcessGroupSchema.TIMEOUT_KEY, remoteProcessGroupDTO.getCommunicationsTimeout()); map.put(CommonPropertyKeys.YIELD_PERIOD_KEY, remoteProcessGroupDTO.getYieldDuration()); map.put(RemoteProcessGroupSchema.TRANSPORT_PROTOCOL_KEY, remoteProcessGroupDTO.getTransportProtocol()); + map.put(RemoteProcessGroupSchema.PROXY_HOST_KEY, remoteProcessGroupDTO.getProxyHost()); + map.put(RemoteProcessGroupSchema.PROXY_PORT_KEY, remoteProcessGroupDTO.getProxyPort()); + map.put(RemoteProcessGroupSchema.PROXY_USER_KEY, remoteProcessGroupDTO.getProxyUser()); + map.put(RemoteProcessGroupSchema.PROXY_PASSWORD_KEY, remoteProcessGroupDTO.getProxyPassword()); return new RemoteProcessGroupSchema(map); } } diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMainTest.java b/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMainTest.java index 597546f2c..ae5936c32 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMainTest.java +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMainTest.java @@ -277,11 +277,12 @@ private void transformRoundTrip(String name) throws JAXBException, IOException, Map templateMap = ConfigMain.transformTemplateToSchema(getClass().getClassLoader().getResourceAsStream(name + ".xml")).toMap(); Map yamlMap = SchemaLoader.loadYamlAsMap(getClass().getClassLoader().getResourceAsStream(name + ".yml")); assertNoMapDifferences(templateMap, yamlMap); + testV2YmlIfPresent(name, yamlMap); testV1YmlIfPresent(name, yamlMap); } private InputStream upgradeAndReturn(String name) throws FileNotFoundException { - InputStream yamlV1Stream = getClass().getClassLoader().getResourceAsStream(name + "-v1.yml"); + InputStream yamlV1Stream = getClass().getClassLoader().getResourceAsStream(name); if (yamlV1Stream == null) { return null; } @@ -292,8 +293,30 @@ private InputStream upgradeAndReturn(String name) throws FileNotFoundException { return new ByteArrayInputStream(outputStream.toByteArray()); } + private void testV2YmlIfPresent(String name, Map yamlMap) throws IOException, SchemaLoaderException { + InputStream upgradedInputStream = upgradeAndReturn(name + "-v2.yml"); + if (upgradedInputStream != null) { + ConvertableSchema configSchemaConvertableSchema = SchemaLoader.loadConvertableSchemaFromYaml(upgradedInputStream); + ConfigSchema configSchemaUpgradedFromV2 = configSchemaConvertableSchema.convert(); + + ConfigSchema configSchemaFromCurrent = new ConfigSchema(yamlMap); + ConfigSchema.getAllProcessGroups(configSchemaFromCurrent.getProcessGroupSchema()).stream().flatMap(p -> p.getRemoteProcessGroups().stream()).forEach(r -> { + clearProxyInfo(r); + }); + + assertNoMapDifferences(configSchemaUpgradedFromV2.toMap(), configSchemaFromCurrent.toMap()); + } + } + + private void clearProxyInfo(RemoteProcessGroupSchema remoteProcessGroupSchema) { + remoteProcessGroupSchema.setProxyHost(RemoteProcessGroupSchema.DEFAULT_PROXY_HOST); + remoteProcessGroupSchema.setProxyPort(RemoteProcessGroupSchema.DEFAULT_PROXY_PORT); + remoteProcessGroupSchema.setProxyUser(RemoteProcessGroupSchema.DEFAULT_PROXY_USER); + remoteProcessGroupSchema.setProxyPassword(RemoteProcessGroupSchema.DEFAULT_PROXY_PASSWORD); + } + private void testV1YmlIfPresent(String name, Map yamlMap) throws IOException, SchemaLoaderException { - InputStream upgradedInputStream = upgradeAndReturn(name); + InputStream upgradedInputStream = upgradeAndReturn(name + "-v1.yml"); if (upgradedInputStream != null) { ConvertableSchema configSchemaConvertableSchema = SchemaLoader.loadConvertableSchemaFromYaml(upgradedInputStream); ConfigSchema configSchemaUpgradedFromV1 = configSchemaConvertableSchema.convert(); @@ -341,6 +364,11 @@ private void testV1YmlIfPresent(String name, Map yamlMap) throws v1Connection.setSourceId(v1IdToCurrentIdMap.get(v1Connection.getSourceId())); v1Connection.setDestinationId(v1IdToCurrentIdMap.get(v1Connection.getDestinationId())); } + + ConfigSchema.getAllProcessGroups(configSchemaFromCurrent.getProcessGroupSchema()).stream().flatMap(p -> p.getRemoteProcessGroups().stream()).forEach(r -> { + clearProxyInfo(r); + r.setTransportProtocol(RemoteProcessGroupSchema.TransportProtocolOptions.RAW.name()); + }); Map v1YamlMap = configSchemaUpgradedFromV1.toMap(); assertNoMapDifferences(v1YamlMap, configSchemaFromCurrent.toMap()); } diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/InvokeHttpMiNiFiTemplateTest-v2.yml b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/InvokeHttpMiNiFiTemplateTest-v2.yml new file mode 100644 index 000000000..7981d1258 --- /dev/null +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/InvokeHttpMiNiFiTemplateTest-v2.yml @@ -0,0 +1,297 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the \"License\"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +MiNiFi Config Version: 2 +Flow Controller: + name: InvokeHttpMiNiFiTemplateTest2 + comment: '' +Core Properties: + flow controller graceful shutdown period: 10 sec + flow service write delay interval: 500 ms + administrative yield duration: 30 sec + bored yield duration: 10 millis + max concurrent threads: 1 +FlowFile Repository: + partitions: 256 + checkpoint interval: 2 mins + always sync: false + Swap: + threshold: 20000 + in period: 5 sec + in threads: 1 + out period: 5 sec + out threads: 4 +Content Repository: + content claim max appendable size: 10 MB + content claim max flow files: 100 + always sync: false +Provenance Repository: + provenance rollover time: 1 min +Component Status Repository: + buffer size: 1440 + snapshot frequency: 1 min +Security Properties: + keystore: '' + keystore type: '' + keystore password: '' + key password: '' + truststore: '' + truststore type: '' + truststore password: '' + ssl protocol: '' + Sensitive Props: + key: + algorithm: PBEWITHMD5AND256BITAES-CBC-OPENSSL + provider: BC +Processors: +- name: LogAttribute + id: 0a73c5e4-7216-4cdf-9008-ace353478d55 + class: org.apache.nifi.processors.standard.LogAttribute + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: + - success + Properties: + Attributes to Ignore: + Attributes to Log: + Log Level: info + Log Payload: 'false' + Log prefix: +- name: LogAttribute2 + id: 0bda7282-7a4f-4d39-83e5-ea86aa63f1dc + class: org.apache.nifi.processors.standard.LogAttribute + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: + - success + Properties: + Attributes to Ignore: + Attributes to Log: + Log Level: info + Log Payload: 'false' + Log prefix: +- name: Route On Status Code + id: dcff217a-a25c-424f-b9f1-d82c55bfa61b + class: org.apache.nifi.processors.standard.RouteOnAttribute + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: [] + Properties: + '200': ${invokehttp.status.code:equals(200)} + Routing Strategy: Route to Property name +- name: Search Google + id: 66649998-3b08-4b41-9f07-a51d999743f6 + class: org.apache.nifi.processors.standard.InvokeHTTP + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: [] + Properties: + Add Response Headers to Request: 'false' + Always Output Response: 'false' + Attributes to Send: + Basic Authentication Password: + Basic Authentication Username: + Connection Timeout: 5 secs + Content-Type: ${mime.type} + Digest Authentication: 'false' + Follow Redirects: 'True' + HTTP Method: GET + Include Date Header: 'True' + Max Length To Put In Attribute: '256' + Penalize on "No Retry": 'false' + Proxy Host: + Proxy Port: + Put Response Body In Attribute: + Read Timeout: 15 secs + Remote URL: http://www.google.com/search?q=${q}&rct=j + SSL Context Service: + Trusted Hostname: + Use Chunked Encoding: 'false' + invokehttp-proxy-password: + invokehttp-proxy-user: + send-message-body: 'true' +- name: Source + id: fcb378d7-6438-41d3-8485-7da85e870ca4 + class: org.apache.nifi.processors.standard.GenerateFlowFile + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 30 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: [] + Properties: + Batch Size: '1' + Data Format: Binary + File Size: 10 b + Unique FlowFiles: 'false' +- name: q=nifi + id: 6638cd15-0aaf-4bba-a18a-722cf85038b6 + class: org.apache.nifi.processors.attributes.UpdateAttribute + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: [] + Properties: + Delete Attributes Expression: + q: nifi +Process Groups: [] +Input Ports: [] +Output Ports: [] +Funnels: [] +Connections: +- name: Route On Status Code/200/LogAttribute + id: 3039718a-bb40-4811-9b74-ecbe926daae8 + source id: dcff217a-a25c-424f-b9f1-d82c55bfa61b + source relationship names: + - '200' + destination id: 0a73c5e4-7216-4cdf-9008-ace353478d55 + max work queue size: 0 + max work queue data size: 0 MB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- name: Route On Status Code/200/b23a4621-cf19-42e6-967c-ffd3716e6a24 + id: 4a7bdaf1-eba0-4317-92f2-93cef6240c28 + source id: dcff217a-a25c-424f-b9f1-d82c55bfa61b + source relationship names: + - '200' + destination id: b23a4621-cf19-42e6-967c-ffd3716e6a24 + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- name: Route On Status Code/unmatched/LogAttribute2 + id: 9f948a25-afb4-4598-9a01-cfb7e3aa5920 + source id: dcff217a-a25c-424f-b9f1-d82c55bfa61b + source relationship names: + - unmatched + destination id: 0bda7282-7a4f-4d39-83e5-ea86aa63f1dc + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- name: Route On Status Code/unmatched/b23a4621-cf19-42e6-967c-ffd3716e6a24 + id: 7299c143-46f6-4a94-aa24-ec47db16f73e + source id: dcff217a-a25c-424f-b9f1-d82c55bfa61b + source relationship names: + - unmatched + destination id: b23a4621-cf19-42e6-967c-ffd3716e6a24 + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- name: Search Google/Failure/Route On Status Code + id: 451b153a-8a4e-4bed-99ab-5159e499bcf1 + source id: 66649998-3b08-4b41-9f07-a51d999743f6 + source relationship names: + - Failure + destination id: dcff217a-a25c-424f-b9f1-d82c55bfa61b + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- name: Search Google/No Retry/Route On Status Code + id: ddbc1711-8cda-4418-b6ee-895763ef1a97 + source id: 66649998-3b08-4b41-9f07-a51d999743f6 + source relationship names: + - No Retry + destination id: dcff217a-a25c-424f-b9f1-d82c55bfa61b + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- name: Search Google/Original/Route On Status Code + id: b97044aa-3401-49df-86ab-bc440cd8f6c8 + source id: 66649998-3b08-4b41-9f07-a51d999743f6 + source relationship names: + - Original + destination id: dcff217a-a25c-424f-b9f1-d82c55bfa61b + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- name: Search Google/Response/Route On Status Code + id: c6a06d31-51fb-488f-bcde-495102f8c93f + source id: 66649998-3b08-4b41-9f07-a51d999743f6 + source relationship names: + - Response + destination id: dcff217a-a25c-424f-b9f1-d82c55bfa61b + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- name: Search Google/Retry/Route On Status Code + id: 3172a4b6-c35b-4d2f-a7c1-68eb16cc1b37 + source id: 66649998-3b08-4b41-9f07-a51d999743f6 + source relationship names: + - Retry + destination id: dcff217a-a25c-424f-b9f1-d82c55bfa61b + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- name: Source/success/q=nifi + id: 2ef71cd8-eb24-4653-8526-34ab84e79328 + source id: fcb378d7-6438-41d3-8485-7da85e870ca4 + source relationship names: + - success + destination id: 6638cd15-0aaf-4bba-a18a-722cf85038b6 + max work queue size: 0 + max work queue data size: 0 MB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- name: q=nifi/success/Search Google + id: ee5ee5bf-a552-43cf-9514-c14d40350bff + source id: 6638cd15-0aaf-4bba-a18a-722cf85038b6 + source relationship names: + - success + destination id: 66649998-3b08-4b41-9f07-a51d999743f6 + max work queue size: 0 + max work queue data size: 0 MB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +Remote Process Groups: +- id: d41d8cd9-8f00-3204-a980-0998ecf8427e + name: '' + url: http://nifi:8080/nifi + comment: '' + timeout: 30 sec + yield period: 10 sec + transport protocol: HTTP + Input Ports: + - name: response + id: b23a4621-cf19-42e6-967c-ffd3716e6a24 + comment: '' + max concurrent tasks: 1 + use compression: false diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/InvokeHttpMiNiFiTemplateTest.xml b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/InvokeHttpMiNiFiTemplateTest.xml index 99358baf8..9f838833d 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/InvokeHttpMiNiFiTemplateTest.xml +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/InvokeHttpMiNiFiTemplateTest.xml @@ -637,10 +637,11 @@ limitations under the License. false - - + proxy.server + 8675 + username http://nifi:8080/nifi - RAW + HTTP 10 sec diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/InvokeHttpMiNiFiTemplateTest.yml b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/InvokeHttpMiNiFiTemplateTest.yml index 6a84d7268..5fcaa4610 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/InvokeHttpMiNiFiTemplateTest.yml +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/InvokeHttpMiNiFiTemplateTest.yml @@ -289,7 +289,11 @@ Remote Process Groups: comment: '' timeout: 30 sec yield period: 10 sec - transport protocol: RAW + transport protocol: HTTP + proxy host: proxy.server + proxy port: 8675 + proxy user: username + proxy password: '' Input Ports: - name: response id: b23a4621-cf19-42e6-967c-ffd3716e6a24 diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ProcessGroupsAndRemoteProcessGroups-v2.yml b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ProcessGroupsAndRemoteProcessGroups-v2.yml new file mode 100644 index 000000000..1d5c880d7 --- /dev/null +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ProcessGroupsAndRemoteProcessGroups-v2.yml @@ -0,0 +1,287 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the \"License\"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +MiNiFi Config Version: 2 +Flow Controller: + name: ProcessGroupsAndRemoteProcessGroups + comment: '' +Core Properties: + flow controller graceful shutdown period: 10 sec + flow service write delay interval: 500 ms + administrative yield duration: 30 sec + bored yield duration: 10 millis + max concurrent threads: 1 +FlowFile Repository: + partitions: 256 + checkpoint interval: 2 mins + always sync: false + Swap: + threshold: 20000 + in period: 5 sec + in threads: 1 + out period: 5 sec + out threads: 4 +Content Repository: + content claim max appendable size: 10 MB + content claim max flow files: 100 + always sync: false +Provenance Repository: + provenance rollover time: 1 min +Component Status Repository: + buffer size: 1440 + snapshot frequency: 1 min +Security Properties: + keystore: '' + keystore type: '' + keystore password: '' + key password: '' + truststore: '' + truststore type: '' + truststore password: '' + ssl protocol: '' + Sensitive Props: + key: + algorithm: PBEWITHMD5AND256BITAES-CBC-OPENSSL + provider: BC +Processors: +- id: 207748d1-0158-1000-0000-000000000000 + name: GenerateFlowFile + class: org.apache.nifi.processors.standard.GenerateFlowFile + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: [] + Properties: + Batch Size: '1' + Data Format: Binary + File Size: 1 b + Unique FlowFiles: 'false' +- id: 2079e8bd-0158-1000-0000-000000000000 + name: LogAttribute + class: org.apache.nifi.processors.standard.LogAttribute + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: + - success + Properties: + Attributes to Ignore: + Attributes to Log: + Log Level: info + Log Payload: 'false' + Log prefix: +- id: 2077ab1e-0158-1000-0000-000000000000 + name: UpdateAttribute + class: org.apache.nifi.processors.attributes.UpdateAttribute + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: [] + Properties: + Delete Attributes Expression: + top: top +Process Groups: +- id: 207888b1-0158-1000-0000-000000000000 + name: middle + Processors: + - id: 2078f34e-0158-1000-0000-000000000000 + name: UpdateAttribute + class: org.apache.nifi.processors.attributes.UpdateAttribute + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: [] + Properties: + Delete Attributes Expression: + middle: middle + Process Groups: + - id: 20794cd4-0158-1000-0000-000000000000 + name: bottom + Processors: + - id: 207a89ba-0158-1000-0000-000000000000 + name: UpdateAttribute + class: org.apache.nifi.processors.attributes.UpdateAttribute + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: [] + Properties: + Delete Attributes Expression: + bottom: bottom + Process Groups: [] + Input Ports: + - id: 207a5f50-0158-1000-0000-000000000000 + name: input + Output Ports: + - id: 207a6d92-0158-1000-0000-000000000000 + name: output + Funnels: [] + Connections: + - id: 21a6abb9-0158-1000-0000-000000000000 + name: UpdateAttribute/success/21a39aba-0158-1000-a1a0-1b55bcddcd72 + source id: 207a89ba-0158-1000-0000-000000000000 + source relationship names: + - success + destination id: 21a39aba-0158-1000-a1a0-1b55bcddcd72 + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: '' + - id: 207ad5e9-0158-1000-0000-000000000000 + name: UpdateAttribute/success/output + source id: 207a89ba-0158-1000-0000-000000000000 + source relationship names: + - success + destination id: 207a6d92-0158-1000-0000-000000000000 + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: '' + - id: 207aca0d-0158-1000-0000-000000000000 + name: input//UpdateAttribute + source id: 207a5f50-0158-1000-0000-000000000000 + source relationship names: [] + destination id: 207a89ba-0158-1000-0000-000000000000 + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: '' + Remote Process Groups: + - id: 21a651bb-0158-1000-0000-000000000000 + name: '' + url: http://localhost:9091/nifi + comment: '' + timeout: 30 sec + yield period: 10 sec + transport protocol: RAW + proxy host: '' + proxy port: '' + proxy user: '' + proxy password: '' + Input Ports: + - id: 21a39aba-0158-1000-a1a0-1b55bcddcd72 + name: input2 + comment: '' + max concurrent tasks: 1 + use compression: false + Input Ports: + - id: 2078c936-0158-1000-0000-000000000000 + name: input + Output Ports: + - id: 2079b327-0158-1000-0000-000000000000 + name: output + Funnels: [] + Connections: + - id: 21a5b1f1-0158-1000-0000-000000000000 + name: UpdateAttribute/success/21a2fb5e-0158-1000-3b5e-5a7d3aaee01b + source id: 2078f34e-0158-1000-0000-000000000000 + source relationship names: + - success + destination id: 21a2fb5e-0158-1000-3b5e-5a7d3aaee01b + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: '' + - id: 207b0eb1-0158-1000-0000-000000000000 + name: UpdateAttribute/success/input + source id: 2078f34e-0158-1000-0000-000000000000 + source relationship names: + - success + destination id: 207a5f50-0158-1000-0000-000000000000 + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: '' + - id: 20792ec2-0158-1000-0000-000000000000 + name: input//UpdateAttribute + source id: 2078c936-0158-1000-0000-000000000000 + source relationship names: [] + destination id: 2078f34e-0158-1000-0000-000000000000 + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: '' + - id: 207b1880-0158-1000-0000-000000000000 + name: output//output + source id: 207a6d92-0158-1000-0000-000000000000 + source relationship names: [] + destination id: 2079b327-0158-1000-0000-000000000000 + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: '' + Remote Process Groups: + - id: 21a4e2f0-0158-1000-0000-000000000000 + name: '' + url: http://localhost:9090/nifi + comment: '' + timeout: 30 sec + yield period: 10 sec + transport protocol: HTTP + Input Ports: + - id: 21a2fb5e-0158-1000-3b5e-5a7d3aaee01b + name: input + comment: '' + max concurrent tasks: 1 + use compression: false +Input Ports: [] +Output Ports: [] +Funnels: [] +Connections: +- id: 2077bf8f-0158-1000-0000-000000000000 + name: GenerateFlowFile/success/UpdateAttribute + source id: 207748d1-0158-1000-0000-000000000000 + source relationship names: + - success + destination id: 2077ab1e-0158-1000-0000-000000000000 + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: '' +- id: 2079cf6f-0158-1000-0000-000000000000 + name: UpdateAttribute/success/input + source id: 2077ab1e-0158-1000-0000-000000000000 + source relationship names: + - success + destination id: 2078c936-0158-1000-0000-000000000000 + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: '' +- id: 2079faa0-0158-1000-0000-000000000000 + name: output//LogAttribute + source id: 2079b327-0158-1000-0000-000000000000 + source relationship names: [] + destination id: 2079e8bd-0158-1000-0000-000000000000 + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: '' +Remote Process Groups: [] diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ProcessGroupsAndRemoteProcessGroups.yml b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ProcessGroupsAndRemoteProcessGroups.yml index a62ab4aa4..2a071ae81 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ProcessGroupsAndRemoteProcessGroups.yml +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ProcessGroupsAndRemoteProcessGroups.yml @@ -184,6 +184,10 @@ Process Groups: timeout: 30 sec yield period: 10 sec transport protocol: RAW + proxy host: '' + proxy port: '' + proxy user: '' + proxy password: '' Input Ports: - id: 21a39aba-0158-1000-a1a0-1b55bcddcd72 name: input2 @@ -244,6 +248,10 @@ Process Groups: timeout: 30 sec yield period: 10 sec transport protocol: HTTP + proxy host: '' + proxy port: '' + proxy user: '' + proxy password: '' Input Ports: - id: 21a2fb5e-0158-1000-3b5e-5a7d3aaee01b name: input diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/SimpleTailFileToRPG-v2.yml b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/SimpleTailFileToRPG-v2.yml new file mode 100644 index 000000000..c70a01849 --- /dev/null +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/SimpleTailFileToRPG-v2.yml @@ -0,0 +1,113 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the \"License\"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +MiNiFi Config Version: 2 +Flow Controller: + name: Simple TailFile To RPG + comment: '' +Core Properties: + flow controller graceful shutdown period: 10 sec + flow service write delay interval: 500 ms + administrative yield duration: 30 sec + bored yield duration: 10 millis + max concurrent threads: 1 +FlowFile Repository: + partitions: 256 + checkpoint interval: 2 mins + always sync: false + Swap: + threshold: 20000 + in period: 5 sec + in threads: 1 + out period: 5 sec + out threads: 4 +Content Repository: + content claim max appendable size: 10 MB + content claim max flow files: 100 + always sync: false +Provenance Repository: + provenance rollover time: 1 min +Component Status Repository: + buffer size: 1440 + snapshot frequency: 1 min +Security Properties: + keystore: '' + keystore type: '' + keystore password: '' + key password: '' + truststore: '' + truststore type: '' + truststore password: '' + ssl protocol: '' + Sensitive Props: + key: + algorithm: PBEWITHMD5AND256BITAES-CBC-OPENSSL + provider: BC +Processors: +- id: b0c04f28-0158-1000-0000-000000000000 + name: TailFile + class: org.apache.nifi.processors.standard.TailFile + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 1 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: [] + Properties: + File Location: Local + File to Tail: ./logs/minifi-app.log + Initial Start Position: Beginning of File + Rolling Filename Pattern: + tail-base-directory: + tail-mode: Single file + tailfile-lookup-frequency: 10 minutes + tailfile-maximum-age: 24 hours + tailfile-recursive-lookup: 'false' + tailfile-rolling-strategy: Fixed name +Process Groups: [] +Input Ports: [] +Output Ports: [] +Funnels: [] +Connections: +- id: b0c0c3cc-0158-1000-0000-000000000000 + name: TailFile/success/ac0e798c-0158-1000-0588-cda9b944e011 + source id: b0c04f28-0158-1000-0000-000000000000 + source relationship names: + - success + destination id: ac0e798c-0158-1000-0588-cda9b944e011 + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: '' +Remote Process Groups: +- id: b0c09ff0-0158-1000-0000-000000000000 + name: '' + url: http://localhost:8080/nifi + comment: '' + timeout: 30 sec + yield period: 10 sec + transport protocol: RAW + Input Ports: + - id: aca664f8-0158-1000-a139-92485891d349 + name: test2 + comment: '' + max concurrent tasks: 1 + use compression: false + - id: ac0e798c-0158-1000-0588-cda9b944e011 + name: test + comment: '' + max concurrent tasks: 1 + use compression: false diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/SimpleTailFileToRPG.yml b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/SimpleTailFileToRPG.yml index 9c4155aac..6946a8bb5 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/SimpleTailFileToRPG.yml +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/SimpleTailFileToRPG.yml @@ -101,6 +101,10 @@ Remote Process Groups: timeout: 30 sec yield period: 10 sec transport protocol: RAW + proxy host: '' + proxy port: '' + proxy user: '' + proxy password: '' Input Ports: - id: aca664f8-0158-1000-a139-92485891d349 name: test2 From f23f5a22c3360776892b9c36ed41fffb699a698a Mon Sep 17 00:00:00 2001 From: Bryan Rosander Date: Mon, 12 Dec 2016 15:22:01 -0500 Subject: [PATCH 2/4] MINIFI-153 - Validating proxy settings --- .../schema/RemoteProcessGroupSchema.java | 17 +++++++++++++ .../schema/RemoteProcessGroupSchemaTest.java | 25 +++++++++++++++++-- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/RemoteProcessGroupSchema.java b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/RemoteProcessGroupSchema.java index 53705f244..d4972f08a 100644 --- a/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/RemoteProcessGroupSchema.java +++ b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/RemoteProcessGroupSchema.java @@ -18,6 +18,7 @@ package org.apache.nifi.minifi.commons.schema; import org.apache.nifi.minifi.commons.schema.common.BaseSchemaWithIdAndName; +import org.apache.nifi.minifi.commons.schema.common.StringUtil; import java.util.List; import java.util.Map; @@ -35,6 +36,10 @@ public class RemoteProcessGroupSchema extends BaseSchemaWithIdAndName { public static final String PROXY_USER_KEY = "proxy user"; public static final String PROXY_PASSWORD_KEY = "proxy password"; + public static final String EXPECTED_PROXY_HOST_IF_PROXY_PORT = "expected " + PROXY_HOST_KEY + " to be set if " + PROXY_PORT_KEY + " is"; + public static final String EXPECTED_PROXY_HOST_IF_PROXY_USER = "expected " + PROXY_HOST_KEY + " to be set if " + PROXY_USER_KEY + " is"; + public static final String EXPECTED_PROXY_USER_IF_PROXY_PASSWORD = "expected " + PROXY_USER_KEY + " to be set if " + PROXY_USER_KEY + " is"; + public enum TransportProtocolOptions { RAW, HTTP; @@ -88,6 +93,18 @@ public RemoteProcessGroupSchema(Map map) { proxyPort = getOptionalKeyAsType(map, PROXY_PORT_KEY, Integer.class, wrapperName, DEFAULT_PROXY_PORT); proxyUser = getOptionalKeyAsType(map, PROXY_USER_KEY, String.class, wrapperName, DEFAULT_PROXY_USER); proxyPassword = getOptionalKeyAsType(map, PROXY_PASSWORD_KEY, String.class, wrapperName, DEFAULT_PROXY_PASSWORD); + + if (StringUtil.isNullOrEmpty(proxyHost)) { + if (proxyPort != null) { + addValidationIssue(PROXY_PORT_KEY, wrapperName, EXPECTED_PROXY_HOST_IF_PROXY_PORT); + } + if (!StringUtil.isNullOrEmpty(proxyUser)) { + addValidationIssue(PROXY_USER_KEY, wrapperName, EXPECTED_PROXY_HOST_IF_PROXY_USER); + } + } + if (!StringUtil.isNullOrEmpty(proxyPassword) && StringUtil.isNullOrEmpty(proxyUser)) { + addValidationIssue(PROXY_PASSWORD_KEY, wrapperName, EXPECTED_PROXY_USER_IF_PROXY_PASSWORD); + } } @Override diff --git a/minifi-commons/minifi-commons-schema/src/test/java/org/apache/nifi/minifi/commons/schema/RemoteProcessGroupSchemaTest.java b/minifi-commons/minifi-commons-schema/src/test/java/org/apache/nifi/minifi/commons/schema/RemoteProcessGroupSchemaTest.java index 4d4630771..5d1b8835d 100644 --- a/minifi-commons/minifi-commons-schema/src/test/java/org/apache/nifi/minifi/commons/schema/RemoteProcessGroupSchemaTest.java +++ b/minifi-commons/minifi-commons-schema/src/test/java/org/apache/nifi/minifi/commons/schema/RemoteProcessGroupSchemaTest.java @@ -19,6 +19,7 @@ package org.apache.nifi.minifi.commons.schema; +import org.apache.nifi.minifi.commons.schema.common.BaseSchema; import org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys; import org.junit.Test; @@ -28,6 +29,8 @@ import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; public class RemoteProcessGroupSchemaTest { @Test @@ -72,11 +75,29 @@ public void testTransportProtocol() { @Test public void testProxySettings() { Map map = new HashMap<>(); - map.put(RemoteProcessGroupSchema.PROXY_HOST_KEY, "host"); map.put(RemoteProcessGroupSchema.PROXY_PORT_KEY, 1234); map.put(RemoteProcessGroupSchema.PROXY_USER_KEY, "user"); - map.put(RemoteProcessGroupSchema.PROXY_PASSWORD_KEY, "password"); RemoteProcessGroupSchema remoteProcessGroupSchema = new RemoteProcessGroupSchema(map); + assertTrue(remoteProcessGroupSchema.getValidationIssues().contains(BaseSchema.getIssueText(RemoteProcessGroupSchema.PROXY_PORT_KEY, remoteProcessGroupSchema.getWrapperName(), + RemoteProcessGroupSchema.EXPECTED_PROXY_HOST_IF_PROXY_PORT))); + assertTrue(remoteProcessGroupSchema.getValidationIssues().contains(BaseSchema.getIssueText(RemoteProcessGroupSchema.PROXY_USER_KEY, remoteProcessGroupSchema.getWrapperName(), + RemoteProcessGroupSchema.EXPECTED_PROXY_HOST_IF_PROXY_USER))); + map.remove(RemoteProcessGroupSchema.PROXY_USER_KEY); + + map.put(RemoteProcessGroupSchema.PROXY_HOST_KEY, "host"); + remoteProcessGroupSchema = new RemoteProcessGroupSchema(map); + assertFalse(remoteProcessGroupSchema.getValidationIssues().contains(BaseSchema.getIssueText(RemoteProcessGroupSchema.PROXY_PORT_KEY, remoteProcessGroupSchema.getWrapperName(), + RemoteProcessGroupSchema.EXPECTED_PROXY_HOST_IF_PROXY_PORT))); + + map.put(RemoteProcessGroupSchema.PROXY_PASSWORD_KEY, "password"); + remoteProcessGroupSchema = new RemoteProcessGroupSchema(map); + assertTrue(remoteProcessGroupSchema.getValidationIssues().contains(BaseSchema.getIssueText(RemoteProcessGroupSchema.PROXY_PASSWORD_KEY, remoteProcessGroupSchema.getWrapperName(), + RemoteProcessGroupSchema.EXPECTED_PROXY_USER_IF_PROXY_PASSWORD))); + + map.put(RemoteProcessGroupSchema.PROXY_USER_KEY, "user"); + remoteProcessGroupSchema = new RemoteProcessGroupSchema(map); + assertFalse(remoteProcessGroupSchema.getValidationIssues().contains(BaseSchema.getIssueText(RemoteProcessGroupSchema.PROXY_PASSWORD_KEY, remoteProcessGroupSchema.getWrapperName(), + RemoteProcessGroupSchema.EXPECTED_PROXY_USER_IF_PROXY_PASSWORD))); assertEquals("host", remoteProcessGroupSchema.getProxyHost()); assertEquals(Integer.valueOf(1234), remoteProcessGroupSchema.getProxyPort()); From 769d3f10b9019c4ab9618b6f37e729990fc0b8a2 Mon Sep 17 00:00:00 2001 From: Bryan Rosander Date: Mon, 12 Dec 2016 16:40:28 -0500 Subject: [PATCH 3/4] MINIFI-153 - Fixing validation --- .../commons/schema/RemoteProcessGroupSchema.java | 11 ++++++++--- .../commons/schema/RemoteProcessGroupSchemaTest.java | 8 ++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/RemoteProcessGroupSchema.java b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/RemoteProcessGroupSchema.java index d4972f08a..32506f7a6 100644 --- a/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/RemoteProcessGroupSchema.java +++ b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/RemoteProcessGroupSchema.java @@ -38,7 +38,8 @@ public class RemoteProcessGroupSchema extends BaseSchemaWithIdAndName { public static final String EXPECTED_PROXY_HOST_IF_PROXY_PORT = "expected " + PROXY_HOST_KEY + " to be set if " + PROXY_PORT_KEY + " is"; public static final String EXPECTED_PROXY_HOST_IF_PROXY_USER = "expected " + PROXY_HOST_KEY + " to be set if " + PROXY_USER_KEY + " is"; - public static final String EXPECTED_PROXY_USER_IF_PROXY_PASSWORD = "expected " + PROXY_USER_KEY + " to be set if " + PROXY_USER_KEY + " is"; + public static final String EXPECTED_PROXY_USER_IF_PROXY_PASSWORD = "expected " + PROXY_USER_KEY + " to be set if " + PROXY_PASSWORD_KEY + " is"; + public static final String EXPECTED_PROXY_PASSWORD_IF_PROXY_USER = "expected " + PROXY_PASSWORD_KEY + " to be set if " + PROXY_USER_KEY + " is"; public enum TransportProtocolOptions { RAW, HTTP; @@ -102,8 +103,12 @@ public RemoteProcessGroupSchema(Map map) { addValidationIssue(PROXY_USER_KEY, wrapperName, EXPECTED_PROXY_HOST_IF_PROXY_USER); } } - if (!StringUtil.isNullOrEmpty(proxyPassword) && StringUtil.isNullOrEmpty(proxyUser)) { - addValidationIssue(PROXY_PASSWORD_KEY, wrapperName, EXPECTED_PROXY_USER_IF_PROXY_PASSWORD); + if (StringUtil.isNullOrEmpty(proxyUser)) { + if (!StringUtil.isNullOrEmpty(proxyPassword)) { + addValidationIssue(PROXY_PASSWORD_KEY, wrapperName, EXPECTED_PROXY_USER_IF_PROXY_PASSWORD); + } + } else if (StringUtil.isNullOrEmpty(proxyPassword)) { + addValidationIssue(PROXY_USER_KEY, wrapperName, EXPECTED_PROXY_PASSWORD_IF_PROXY_USER); } } diff --git a/minifi-commons/minifi-commons-schema/src/test/java/org/apache/nifi/minifi/commons/schema/RemoteProcessGroupSchemaTest.java b/minifi-commons/minifi-commons-schema/src/test/java/org/apache/nifi/minifi/commons/schema/RemoteProcessGroupSchemaTest.java index 5d1b8835d..754f91ee1 100644 --- a/minifi-commons/minifi-commons-schema/src/test/java/org/apache/nifi/minifi/commons/schema/RemoteProcessGroupSchemaTest.java +++ b/minifi-commons/minifi-commons-schema/src/test/java/org/apache/nifi/minifi/commons/schema/RemoteProcessGroupSchemaTest.java @@ -94,10 +94,18 @@ public void testProxySettings() { assertTrue(remoteProcessGroupSchema.getValidationIssues().contains(BaseSchema.getIssueText(RemoteProcessGroupSchema.PROXY_PASSWORD_KEY, remoteProcessGroupSchema.getWrapperName(), RemoteProcessGroupSchema.EXPECTED_PROXY_USER_IF_PROXY_PASSWORD))); + map.remove(RemoteProcessGroupSchema.PROXY_PASSWORD_KEY); map.put(RemoteProcessGroupSchema.PROXY_USER_KEY, "user"); remoteProcessGroupSchema = new RemoteProcessGroupSchema(map); + assertTrue(remoteProcessGroupSchema.getValidationIssues().contains(BaseSchema.getIssueText(RemoteProcessGroupSchema.PROXY_USER_KEY, remoteProcessGroupSchema.getWrapperName(), + RemoteProcessGroupSchema.EXPECTED_PROXY_PASSWORD_IF_PROXY_USER))); + + map.put(RemoteProcessGroupSchema.PROXY_PASSWORD_KEY, "password"); + remoteProcessGroupSchema = new RemoteProcessGroupSchema(map); assertFalse(remoteProcessGroupSchema.getValidationIssues().contains(BaseSchema.getIssueText(RemoteProcessGroupSchema.PROXY_PASSWORD_KEY, remoteProcessGroupSchema.getWrapperName(), RemoteProcessGroupSchema.EXPECTED_PROXY_USER_IF_PROXY_PASSWORD))); + assertFalse(remoteProcessGroupSchema.getValidationIssues().contains(BaseSchema.getIssueText(RemoteProcessGroupSchema.PROXY_USER_KEY, remoteProcessGroupSchema.getWrapperName(), + RemoteProcessGroupSchema.EXPECTED_PROXY_PASSWORD_IF_PROXY_USER))); assertEquals("host", remoteProcessGroupSchema.getProxyHost()); assertEquals(Integer.valueOf(1234), remoteProcessGroupSchema.getProxyPort()); From fe83927c14ca9b136ae9eb92246a08ab47f0f5b8 Mon Sep 17 00:00:00 2001 From: Bryan Rosander Date: Mon, 12 Dec 2016 21:31:38 -0500 Subject: [PATCH 4/4] MINIFI-153 - Validating htttp transport protocol for proxy --- .../nifi/minifi/commons/schema/RemoteProcessGroupSchema.java | 4 ++++ .../minifi/commons/schema/RemoteProcessGroupSchemaTest.java | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/RemoteProcessGroupSchema.java b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/RemoteProcessGroupSchema.java index 32506f7a6..e73ff7337 100644 --- a/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/RemoteProcessGroupSchema.java +++ b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/RemoteProcessGroupSchema.java @@ -31,6 +31,7 @@ public class RemoteProcessGroupSchema extends BaseSchemaWithIdAndName { public static final String URL_KEY = "url"; public static final String TIMEOUT_KEY = "timeout"; public static final String TRANSPORT_PROTOCOL_KEY = "transport protocol"; + public static final String S2S_PROXY_REQUIRES_HTTP = "Site-To-Site proxy support requires HTTP " + TRANSPORT_PROTOCOL_KEY; public static final String PROXY_HOST_KEY = "proxy host"; public static final String PROXY_PORT_KEY = "proxy port"; public static final String PROXY_USER_KEY = "proxy user"; @@ -102,7 +103,10 @@ public RemoteProcessGroupSchema(Map map) { if (!StringUtil.isNullOrEmpty(proxyUser)) { addValidationIssue(PROXY_USER_KEY, wrapperName, EXPECTED_PROXY_HOST_IF_PROXY_USER); } + } else if (!TransportProtocolOptions.HTTP.name().equals(transportProtocol)) { + addValidationIssue(PROXY_HOST_KEY, wrapperName, S2S_PROXY_REQUIRES_HTTP); } + if (StringUtil.isNullOrEmpty(proxyUser)) { if (!StringUtil.isNullOrEmpty(proxyPassword)) { addValidationIssue(PROXY_PASSWORD_KEY, wrapperName, EXPECTED_PROXY_USER_IF_PROXY_PASSWORD); diff --git a/minifi-commons/minifi-commons-schema/src/test/java/org/apache/nifi/minifi/commons/schema/RemoteProcessGroupSchemaTest.java b/minifi-commons/minifi-commons-schema/src/test/java/org/apache/nifi/minifi/commons/schema/RemoteProcessGroupSchemaTest.java index 754f91ee1..2339f4008 100644 --- a/minifi-commons/minifi-commons-schema/src/test/java/org/apache/nifi/minifi/commons/schema/RemoteProcessGroupSchemaTest.java +++ b/minifi-commons/minifi-commons-schema/src/test/java/org/apache/nifi/minifi/commons/schema/RemoteProcessGroupSchemaTest.java @@ -88,6 +88,8 @@ public void testProxySettings() { remoteProcessGroupSchema = new RemoteProcessGroupSchema(map); assertFalse(remoteProcessGroupSchema.getValidationIssues().contains(BaseSchema.getIssueText(RemoteProcessGroupSchema.PROXY_PORT_KEY, remoteProcessGroupSchema.getWrapperName(), RemoteProcessGroupSchema.EXPECTED_PROXY_HOST_IF_PROXY_PORT))); + assertTrue(remoteProcessGroupSchema.getValidationIssues().contains(BaseSchema.getIssueText(RemoteProcessGroupSchema.PROXY_HOST_KEY, remoteProcessGroupSchema.getWrapperName(), + RemoteProcessGroupSchema.S2S_PROXY_REQUIRES_HTTP))); map.put(RemoteProcessGroupSchema.PROXY_PASSWORD_KEY, "password"); remoteProcessGroupSchema = new RemoteProcessGroupSchema(map); @@ -101,11 +103,14 @@ public void testProxySettings() { RemoteProcessGroupSchema.EXPECTED_PROXY_PASSWORD_IF_PROXY_USER))); map.put(RemoteProcessGroupSchema.PROXY_PASSWORD_KEY, "password"); + map.put(RemoteProcessGroupSchema.TRANSPORT_PROTOCOL_KEY, RemoteProcessGroupSchema.TransportProtocolOptions.HTTP.name()); remoteProcessGroupSchema = new RemoteProcessGroupSchema(map); assertFalse(remoteProcessGroupSchema.getValidationIssues().contains(BaseSchema.getIssueText(RemoteProcessGroupSchema.PROXY_PASSWORD_KEY, remoteProcessGroupSchema.getWrapperName(), RemoteProcessGroupSchema.EXPECTED_PROXY_USER_IF_PROXY_PASSWORD))); assertFalse(remoteProcessGroupSchema.getValidationIssues().contains(BaseSchema.getIssueText(RemoteProcessGroupSchema.PROXY_USER_KEY, remoteProcessGroupSchema.getWrapperName(), RemoteProcessGroupSchema.EXPECTED_PROXY_PASSWORD_IF_PROXY_USER))); + assertFalse(remoteProcessGroupSchema.getValidationIssues().contains(BaseSchema.getIssueText(RemoteProcessGroupSchema.PROXY_HOST_KEY, remoteProcessGroupSchema.getWrapperName(), + RemoteProcessGroupSchema.S2S_PROXY_REQUIRES_HTTP))); assertEquals("host", remoteProcessGroupSchema.getProxyHost()); assertEquals(Integer.valueOf(1234), remoteProcessGroupSchema.getProxyPort());